Release v3.0.2 (#2617)

- bump version
- update release notes
- update links to Postgres Slack
- simplify /sync health-check endpoint code
- update unit-tests to cover missing lines

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,48 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Environment**
-- Patroni version:
-- PostgreSQL version:
-- DCS (and its version):
-
-**Patroni configuration file**
-```
-Please copy&paste your Patroni configuration file here
-```
-
-**patronictl show-config**
-```
-Please copy&paste the output of "patronictl show-config" command here
-```
-
-**Have you checked Patroni logs?**
-Please provide a snippet of Patroni log files here
-
-**Have you checked PostgreSQL logs?**
-Please provide a snippet here
-
-**Have you tried to use GitHub issue search?**
-Maybe there is already a similar issue solved.
-
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,97 @@
+name: Bug Report
+description: Create a report to help us improve
+labels:
+- bug
+body:
+- type: markdown
+  attributes:
+    value: |
+      If you have a question please post it on channel [#patroni](https://postgresteam.slack.com/archives/C9XPYG92A) in the [PostgreSQL Slack](https://join.slack.com/t/postgresteam/shared_invite/zt-1qj14i9sj-E9WqIFlvcOiHsEk2yFEMjA).
+      Before reporting a bug please make sure to **reproduce it with the latest Patroni version**!
+      Please fill the form below and provide as much information as possible.
+      Not doing so may result in your bug not being addressed in a timely manner.
+
+- type: textarea
+  id: problem
+  attributes:
+    label: What happened?
+  validations:
+    required: true
+
+- type: textarea
+  id: repro
+  attributes:
+    label: How can we reproduce it (as minimally and precisely as possible)?
+  validations:
+    required: true
+
+- type: textarea
+  id: expected
+  attributes:
+    label: What did you expect to happen?
+  validations:
+    required: true
+
+- type: textarea
+  id: environment
+  attributes:
+    label: Patroni/PostgreSQL/DCS version
+    value: |
+      -   Patroni version: 
+      -   PostgreSQL version: 
+      -   DCS (and its version): 
+  validations:
+    required: true
+
+- type: textarea
+  id: patroniConfig
+  attributes:
+    label: Patroni configuration file
+    description: Please copy and paste Patroni configuration file here. This will be automatically formatted into code, so no need for backticks.
+    render: yaml
+  validations:
+    required: true
+
+- type: textarea
+  id: globalConfig
+  attributes:
+    label: patronictl show-config
+    description: Please copy and paste `patronictl show-config` output here. This will be automatically formatted into code, so no need for backticks.
+    render: yaml
+  validations:
+    required: true
+
+- type: textarea
+  id: patroniLogs
+  attributes:
+    label: Patroni log files
+    description: Please copy and paste any relevant Patroni log output. This will be automatically formatted into code, so no need for backticks.
+    render: shell
+  validations:
+    required: true
+
+- type: textarea
+  id: postgresLogs
+  attributes:
+    label: PostgreSQL log files
+    description: Please copy and paste any relevant PostgreSQL log output. This will be automatically formatted into code, so no need for backticks.
+    render: shell
+  validations:
+    required: true
+
+- type: checkboxes
+  id: issueSearch
+  attributes:
+    label: Have you tried to use GitHub issue search?
+    description: Maybe there is already a similar issue solved.
+    options:
+    - label: 'Yes'
+      required: true
+  validations:
+    required: true
+
+- type: textarea
+  id: additional
+  attributes:
+    label: Anything else we need to know?
+    description: Add any other context about the problem here.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+- name: Question
+  url: https://join.slack.com/t/postgresteam/shared_invite/zt-1qj14i9sj-E9WqIFlvcOiHsEk2yFEMjA
+  about: "Please ask questions on channel #patroni in the PostgreSQL Slack"

.github/workflows/install_deps.py

@@ -5,7 +5,6 @@ import subprocess
 import stat
 import sys
 import tarfile
-import time
 import zipfile
 
 
@@ -20,7 +19,7 @@ def install_requirements(what):
     requirements = ['mock>=2.0.0', 'flake8', 'pytest', 'pytest-cov'] if what == 'all' else ['behave']
     requirements += ['coverage']
     # try to split tests between psycopg2 and psycopg3
-    requirements += ['psycopg[binary]'] if sys.version_info >= (3, 6, 0) and\
+    requirements += ['psycopg[binary]'] if sys.version_info > (3, 7, 0) and\
         (sys.platform != 'darwin' or what == 'etcd3') else ['psycopg2-binary']
     for r in read('requirements.txt').split('\n'):
         r = r.strip()
@@ -30,6 +29,7 @@ def install_requirements(what):
                 requirements.append(r)
 
     subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'pip'])
+    subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'wheel'])
     r = subprocess.call([sys.executable, '-m', 'pip', 'install'] + requirements)
     s = subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'setuptools'])
     return s | r
@@ -45,10 +45,10 @@ def install_packages(what):
     packages['exhibitor'] = packages['zookeeper']
     packages = packages.get(what, [])
     ver = versions.get(what)
-    subprocess.call(['sudo', 'sed', '-i', 's/pgdg main.*$/pgdg main {0}/'.format(ver),
-                     '/etc/apt/sources.list.d/pgdg.list'])
+    if float(ver) >= 15:
+        packages += ['postgresql-{0}-citus-11.2'.format(ver)]
     subprocess.call(['sudo', 'apt-get', 'update', '-y'])
-    return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev', 'wget'] + packages)
+    return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev'] + packages)
 
 
 def get_file(url, name):
@@ -98,7 +98,7 @@ def unpack(archive, name):
 
 
 def install_etcd():
-    version = os.environ.get('ETCDVERSION', '3.3.13')
+    version = os.environ.get('ETCDVERSION', '3.4.23')
     platform = {'linux2': 'linux', 'win32': 'windows', 'cygwin': 'windows'}.get(sys.platform, sys.platform)
     dirname = 'etcd-v{0}-{1}-amd64'.format(version, platform)
     ext = 'tar.gz' if platform == 'linux' else 'zip'
@@ -110,59 +110,17 @@ def install_etcd():
 
 
 def install_postgres():
-    version = os.environ.get('PGVERSION', '14.1-1')
+    version = os.environ.get('PGVERSION', '15.1-1')
     platform = {'darwin': 'osx', 'win32': 'windows-x64', 'cygwin': 'windows-x64'}[sys.platform]
+    if platform == 'osx':
+        return subprocess.call(['brew', 'install', 'expect', 'postgresql@{0}'.format(version.split('.')[0])])
     name = 'postgresql-{0}-{1}-binaries.zip'.format(version, platform)
     get_file('http://get.enterprisedb.com/postgresql/' + name, name)
     unzip_all(name)
     bin_dir = os.path.join('pgsql', 'bin')
     for f in os.listdir(bin_dir):
         chmod_755(os.path.join(bin_dir, f))
-    subprocess.call(['pgsql/bin/postgres', '-V'])
-    return 0
-
-
-def setup_kubernetes():
-    get_file('https://storage.googleapis.com/minikube/k8sReleases/v1.7.0/localkube-linux-amd64', 'localkube')
-    chmod_755('localkube')
-
-    devnull = open(os.devnull, 'w')
-    subprocess.Popen(['sudo', 'nohup', './localkube', '--logtostderr=true', '--enable-dns=false'],
-                     stdout=devnull, stderr=devnull)
-    for _ in range(0, 120):
-        if subprocess.call(['wget', '-qO', '-', 'http://127.0.0.1:8080/'], stdout=devnull, stderr=devnull) == 0:
-            break
-        time.sleep(1)
-    else:
-        print('localkube did not start')
-        return 1
-
-    subprocess.call('sudo chmod 644 /var/lib/localkube/certs/*', shell=True)
-    print('Set up .kube/config')
-    kube = os.path.join(os.path.expanduser('~'), '.kube')
-    os.makedirs(kube)
-    with open(os.path.join(kube, 'config'), 'w') as f:
-        f.write("""apiVersion: v1
-clusters:
-- cluster:
-    certificate-authority: /var/lib/localkube/certs/ca.crt
-    server: https://127.0.0.1:8443
-  name: local
-contexts:
-- context:
-    cluster: local
-    user: myself
-  name: local
-current-context: local
-kind: Config
-preferences: {}
-users:
-- name: myself
-  user:
-    client-certificate: /var/lib/localkube/certs/apiserver.crt
-    client-key: /var/lib/localkube/certs/apiserver.key
-""")
-    return 0
+    return subprocess.call(['pgsql/bin/postgres', '-V'])
 
 
 def main():
@@ -171,8 +129,6 @@ def main():
     if what != 'all':
         if sys.platform.startswith('linux'):
             r = install_packages(what)
-            if r == 0 and what == 'kubernetes':
-                r = setup_kubernetes()
         else:
             r = install_postgres()
 

.github/workflows/mapping.py

@@ -1 +1 @@
-versions = {'etcd': '9.6', 'etcd3': '14', 'consul': '13', 'exhibitor': '12', 'raft': '11', 'kubernetes': '14'}
+versions = {'etcd': '9.6', 'etcd3': '14', 'consul': '13', 'exhibitor': '12', 'raft': '11', 'kubernetes': '15'}

.github/workflows/release.yaml

@@ -0,0 +1,41 @@
+name: Publish Patroni distributions to PyPI and TestPyPI
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+  release:
+    types:
+      - published
+jobs:
+  build-n-publish:
+    name: Build and publish Patroni distributions to PyPI and TestPyPI
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+
+    - name: Run tests and flake8
+      run: python .github/workflows/run_tests.py
+
+    - name: Build a binary wheel and a source tarball
+      run: python setup.py sdist bdist_wheel
+
+    - name: Publish distribution to Test PyPI
+      if: github.event_name == 'push'
+      uses: pypa/[email protected]
+      with:
+        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        repository_url: https://test.pypi.org/legacy/
+
+    - name: Publish distribution to PyPI
+      if: github.event_name == 'release'
+      uses: pypa/[email protected]
+      with:
+        password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/run_tests.py

@@ -28,22 +28,20 @@ def main():
         version = versions.get(what)
         path = '/usr/lib/postgresql/{0}/bin:.'.format(version)
         unbuffer = ['timeout', '900', 'unbuffer']
-        args = ['--tags=-skip'] if what == 'etcd' else []
     else:
-        path = os.path.abspath(os.path.join('pgsql', 'bin'))
         if sys.platform == 'darwin':
-            path += ':.'
-        args = unbuffer = []
+            version = os.environ.get('PGVERSION', '15.1-1')
+            path = '/usr/local/opt/postgresql@{0}/bin:.'.format(version.split('.')[0])
+            unbuffer = ['unbuffer']
+        else:
+            path = os.path.abspath(os.path.join('pgsql', 'bin'))
+            unbuffer = []
     env['PATH'] = path + os.pathsep + env['PATH']
     env['DCS'] = what
+    if what == 'kubernetes':
+        env['PATRONI_KUBERNETES_CONTEXT'] = 'k3d-k3s-default'
 
-    ret = subprocess.call(unbuffer + [sys.executable, '-m', 'behave'] + args, env=env)
-
-    if ret != 0:
-        if subprocess.call('grep . features/output/*_failed/*postgres?.*', shell=True) != 0:
-            subprocess.call('grep . features/output/*/*postgres?.*', shell=True)
-        return 1
-    return 0
+    return subprocess.call(unbuffer + [sys.executable, '-m', 'behave'], env=env)
 
 
 if __name__ == '__main__':

.github/workflows/tests.yaml

@@ -5,8 +5,10 @@ on:
   push:
     branches:
     - master
-    tags:
-    - v.*
+
+env:
+  CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
+  SECRETS_AVAILABLE: ${{ secrets.CODACY_PROJECT_TOKEN != '' }}
 
 jobs:
   unit:
@@ -17,30 +19,10 @@ jobs:
         os: [ubuntu, windows, macos]
 
     steps:
-    - uses: actions/checkout@v1
-    - name: Set up Python 2.7
-      uses: actions/setup-python@v2
-      with:
-        python-version: 2.7
-      if: matrix.os != 'windows'
-    - name: Install dependencies
-      run: python .github/workflows/install_deps.py
-      if: matrix.os != 'windows'
-    - name: Run tests and flake8
-      run: python .github/workflows/run_tests.py
-      if: matrix.os != 'windows'
-
-    - name: Set up Python 3.6
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.6
-    - name: Install dependencies
-      run: python .github/workflows/install_deps.py
-    - name: Run tests and flake8
-      run: python .github/workflows/run_tests.py
+    - uses: actions/checkout@v3
 
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.7
     - name: Install dependencies
@@ -49,7 +31,7 @@ jobs:
       run: python .github/workflows/run_tests.py
 
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.8
     - name: Install dependencies
@@ -58,7 +40,7 @@ jobs:
       run: python .github/workflows/run_tests.py
 
     - name: Set up Python 3.9
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - name: Install dependencies
@@ -67,7 +49,7 @@ jobs:
       run: python .github/workflows/run_tests.py
 
     - name: Set up Python 3.10
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: '3.10'
     - name: Install dependencies
@@ -75,6 +57,15 @@ jobs:
     - name: Run tests and flake8
       run: python .github/workflows/run_tests.py
 
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.11
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+    - name: Run tests and flake8
+      run: python .github/workflows/run_tests.py
+
     - name: Combine coverage
       run: python .github/workflows/run_tests.py combine
 
@@ -92,60 +83,77 @@ jobs:
     runs-on: ${{ matrix.os }}-latest
     env:
       DCS: ${{ matrix.dcs }}
-      ETCDVERSION: 3.3.13
-      PGVERSION: 12.1-1  # for windows and macos
+      ETCDVERSION: 3.4.23
+      PGVERSION: 15.1-1  # for windows and macos
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu]
-        python-version: [2.7, 3.6, 3.9]
+        python-version: [3.7, '3.10']
         dcs: [etcd, etcd3, consul, exhibitor, kubernetes, raft]
-        exclude:
-        - dcs: kubernetes
-          python-version: 2.7
         include:
         - os: macos
-          python-version: 3.7
+          python-version: 3.8
           dcs: raft
         - os: macos
-          python-version: 3.8
+          python-version: 3.9
           dcs: etcd
         - os: macos
-          python-version: '3.10'
+          python-version: 3.11
           dcs: etcd3
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Add postgresql apt repo
-      run: sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+    - uses: nolar/setup-k3d-k3s@v1
+      if: matrix.dcs == 'kubernetes'
+    - name: Add postgresql and citus apt repo
+      run: |
+        sudo apt-get update -y
+        sudo apt-get install -y wget ca-certificates gnupg debian-archive-keyring apt-transport-https
+        sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+        sudo sh -c 'wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg'
+        sudo sh -c 'echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://repos.citusdata.com/community/ubuntu/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list'
+        sudo sh -c 'wget -qO - https://repos.citusdata.com/community/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg'
       if: matrix.os == 'ubuntu'
     - name: Install dependencies
       run: python .github/workflows/install_deps.py
     - name: Run behave tests
       run: python .github/workflows/run_tests.py
-    - uses: actions/setup-python@v2
+    - name: Upload logs if behave failed
+      uses: actions/upload-artifact@v3
+      if: failure()
       with:
-        python-version: '3.10'
-    - name: Install coveralls
-      run: python -m pip install coveralls
-    - name: Upload Coverage
-      env:
-        COVERALLS_FLAG_NAME: behave-${{ matrix.os }}-${{ matrix.dcs }}-${{ matrix.python-version }}
-        COVERALLS_PARALLEL: 'true'
-        GITHUB_TOKEN: ${{ secrets.github_token }}
-      run: python -m coveralls --service=github
+        name: behave-${{ matrix.os }}-${{ matrix.dcs }}-${{ matrix.python-version }}-logs
+        path: |
+          features/output/*_failed/*postgres?.*
+          features/output/*.log
+        if-no-files-found: error
+        retention-days: 5
+    - name: Generate coverage xml report
+      run: python -m coverage xml -o cobertura.xml
+    - name: Upload coverage to Codacy
+      run: bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r cobertura.xml -l Python --partial
+      if: ${{ env.SECRETS_AVAILABLE == 'true' }}
 
   coveralls-finish:
     name: Finalize coveralls.io
-    needs: [unit, behave]
+    needs: unit
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
     - run: python -m pip install coveralls
     - run: python -m coveralls --service=github --finish
       env:
         GITHUB_TOKEN: ${{ secrets.github_token }}
+
+  codacy-final:
+    name: Finalize Codacy
+    needs: behave
+    runs-on: ubuntu-latest
+    steps:
+    - run: bash <(curl -Ls https://coverage.codacy.com/get.sh) final
+      if: ${{ env.SECRETS_AVAILABLE == 'true' }}

CODEOWNERS

@@ -0,0 +1,2 @@
+# global owners
+*        @CyberDem0n @hughcapet

Dockerfile

@@ -1,6 +1,6 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
-ARG PG_MAJOR=10
+ARG PG_MAJOR=15
 ARG COMPRESS=false
 ARG PGHOME=/home/postgres
 ARG PGDATA=$PGHOME/data
@@ -43,18 +43,18 @@ RUN set -ex \
     && echo 'syntax on\nfiletype plugin indent on\nset mouse-=a\nautocmd FileType yaml setlocal ts=2 sts=2 sw=2 expandtab' > /etc/vim/vimrc.local \
 \
     # Prepare postgres/patroni/haproxy environment
-    && mkdir -p $PGHOME/.config/patroni /patroni /run/haproxy \
-    && ln -s ../../postgres0.yml $PGHOME/.config/patroni/patronictl.yaml \
+    && mkdir -p "$PGHOME/.config/patroni" /patroni /run/haproxy \
+    && ln -s ../../postgres0.yml "$PGHOME/.config/patroni/patronictl.yaml" \
     && ln -s /patronictl.py /usr/local/bin/patronictl \
     && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \
     && chown -R postgres:postgres /var/log \
 \
     # Download etcd
-    && curl -sL https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
+    && curl -sL "https://github.com/coreos/etcd/releases/download/v$ETCDVERSION/etcd-v$ETCDVERSION-linux-$(dpkg --print-architecture).tar.gz" \
             | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl \
 \
     # Download confd
-    && curl -sL https://github.com/kelseyhightower/confd/releases/download/v${CONFDVERSION}/confd-${CONFDVERSION}-linux-amd64 \
+    && curl -sL "https://github.com/kelseyhightower/confd/releases/download/v$CONFDVERSION/confd-$CONFDVERSION-linux-$(dpkg --print-architecture)" \
             > /usr/local/bin/confd && chmod +x /usr/local/bin/confd \
 \
     # Clean up all useless packages and some files
@@ -90,7 +90,7 @@ RUN set -ex \
     && find /usr/bin -xtype l -delete \
     && find /var/log -type f -exec truncate --size 0 {} \; \
     && find /usr/lib/python3/dist-packages -name '*test*' | xargs rm -fr \
-    && find /lib/x86_64-linux-gnu/security -type f ! -name pam_env.so ! -name pam_permit.so ! -name pam_unix.so -delete
+    && find /lib/$(uname -m)-linux-gnu/security -type f ! -name pam_env.so ! -name pam_permit.so ! -name pam_unix.so -delete
 
 # perform compression if it is necessary
 ARG COMPRESS
@@ -99,8 +99,10 @@ RUN if [ "$COMPRESS" = "true" ]; then \
         # Allow certain sudo commands from postgres
         && echo 'postgres ALL=(ALL) NOPASSWD: /bin/tar xpJf /a.tar.xz -C /, /bin/rm /a.tar.xz, /bin/ln -snf dash /bin/sh' >> /etc/sudoers \
         && ln -snf busybox /bin/sh \
-        && files="/bin/sh /usr/bin/sudo /usr/lib/sudo/sudoers.so /lib/x86_64-linux-gnu/security/pam_*.so" \
-        && libs="$(ldd $files | awk '{print $3;}' | grep '^/' | sort -u) /lib/x86_64-linux-gnu/ld-linux-x86-64.so.* /lib/x86_64-linux-gnu/libnsl.so.* /lib/x86_64-linux-gnu/libnss_compat.so.*" \
+        && arch=$(uname -m) \
+        && darch=$(uname -m | sed 's/_/-/') \
+        && files="/bin/sh /usr/bin/sudo /usr/lib/sudo/sudoers.so /lib/$arch-linux-gnu/security/pam_*.so" \
+        && libs="$(ldd $files | awk '{print $3;}' | grep '^/' | sort -u) /lib/ld-linux-$darch.so.* /lib/$arch-linux-gnu/ld-linux-$darch.so.* /lib/$arch-linux-gnu/libnsl.so.* /lib/$arch-linux-gnu/libnss_compat.so.* /lib/$arch-linux-gnu/libnss_files.so.*" \
         && (echo /var/run $files $libs | tr ' ' '\n' && realpath $files $libs) | sort -u | sed 's/^\///' > /exclude \
         && find /etc/alternatives -xtype l -delete \
         && save_dirs="usr lib var bin sbin etc/ssl etc/init.d etc/alternatives etc/apt" \
@@ -117,7 +119,7 @@ RUN if [ "$COMPRESS" = "true" ]; then \
 FROM scratch
 COPY --from=builder / /
 
-LABEL maintainer="Alexander Kukushkin <alexander.kukushkin@zalando.de>"
+LABEL maintainer="Alexander Kukushkin <akukushkin@microsoft.com>"
 
 ARG PG_MAJOR
 ARG COMPRESS
@@ -151,7 +153,7 @@ RUN sed -i 's/env python/&3/' /patroni*.py \
     && sed -i 's/^      parameters:/      pg_hba:\n      - local all all trust\n      - host replication all all md5\n      - host all all all md5\n&\n        max_connections: 100/'  postgres?.yml \
     && if [ "$COMPRESS" = "true" ]; then chmod u+s /usr/bin/sudo; fi \
     && chmod +s /bin/ping \
-    && chown -R postgres:postgres $PGHOME /run /etc/haproxy
+    && chown -R postgres:postgres "$PGHOME" /run /etc/haproxy
 
 USER postgres
 

Dockerfile.citus

@@ -0,0 +1,173 @@
+## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
+## It has all the necessary components to play/debug with a single node appliance, running etcd
+ARG PG_MAJOR=15
+ARG COMPRESS=false
+ARG PGHOME=/home/postgres
+ARG PGDATA=$PGHOME/data
+ARG LC_ALL=C.UTF-8
+ARG LANG=C.UTF-8
+
+FROM postgres:$PG_MAJOR as builder
+
+ARG PGHOME
+ARG PGDATA
+ARG LC_ALL
+ARG LANG
+
+ENV ETCDVERSION=3.3.13 CONFDVERSION=0.16.0
+
+RUN set -ex \
+    && export DEBIAN_FRONTEND=noninteractive \
+    && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && apt-get update -y \
+    # postgres:10 is based on debian, which has the patroni package. We will install all required dependencies
+    && apt-cache depends patroni | sed -n -e 's/.*Depends: \(python3-.\+\)$/\1/p' \
+            | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
+            | xargs apt-get install -y vim curl less jq locales haproxy sudo \
+                            python3-etcd python3-kazoo python3-pip busybox \
+                            net-tools iputils-ping --fix-missing \
+    && curl https://install.citusdata.com/community/deb.sh | bash \
+    && apt-get -y install postgresql-$PG_MAJOR-citus-11.2 \
+    && pip3 install dumb-init \
+\
+    # Cleanup all locales but en_US.UTF-8
+    && find /usr/share/i18n/charmaps/ -type f ! -name UTF-8.gz -delete \
+    && find /usr/share/i18n/locales/ -type f ! -name en_US ! -name en_GB ! -name i18n* ! -name iso14651_t1 ! -name iso14651_t1_common ! -name 'translit_*' -delete \
+    && echo 'en_US.UTF-8 UTF-8' > /usr/share/i18n/SUPPORTED \
+\
+    # Make sure we have a en_US.UTF-8 locale available
+    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
+\
+    # haproxy dummy config
+    && echo 'global\n        stats socket /run/haproxy/admin.sock mode 660 level admin' > /etc/haproxy/haproxy.cfg \
+\
+    # vim config
+    && echo 'syntax on\nfiletype plugin indent on\nset mouse-=a\nautocmd FileType yaml setlocal ts=2 sts=2 sw=2 expandtab' > /etc/vim/vimrc.local \
+\
+    # Prepare postgres/patroni/haproxy environment
+    && mkdir -p $PGHOME/.config/patroni /patroni /run/haproxy \
+    && ln -s ../../postgres0.yml $PGHOME/.config/patroni/patronictl.yaml \
+    && ln -s /patronictl.py /usr/local/bin/patronictl \
+    && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \
+    && chown -R postgres:postgres /var/log \
+\
+    # Download etcd
+    && curl -sL https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-$(dpkg --print-architecture).tar.gz \
+            | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl \
+\
+    # Download confd
+    && curl -sL https://github.com/kelseyhightower/confd/releases/download/v${CONFDVERSION}/confd-${CONFDVERSION}-linux-$(dpkg --print-architecture) \
+            > /usr/local/bin/confd && chmod +x /usr/local/bin/confd \
+    # Prepare client cert for HAProxy
+    && cat /etc/ssl/private/ssl-cert-snakeoil.key  /etc/ssl/certs/ssl-cert-snakeoil.pem > /etc/ssl/private/ssl-cert-snakeoil.crt \
+\
+    # Clean up all useless packages and some files
+    && apt-get purge -y --allow-remove-essential python3-pip gzip bzip2 util-linux e2fsprogs \
+                libmagic1 bsdmainutils login ncurses-bin libmagic-mgc e2fslibs bsdutils \
+                exim4-config gnupg-agent dirmngr libpython2.7-stdlib libpython2.7-minimal \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* \
+        /root/.cache \
+        /var/cache/debconf/* \
+        /etc/rc?.d \
+        /etc/systemd \
+        /docker-entrypoint* \
+        /sbin/pam* \
+        /sbin/swap* \
+        /sbin/unix* \
+        /usr/local/bin/gosu \
+        /usr/sbin/[acgipr]* \
+        /usr/sbin/*user* \
+        /usr/share/doc* \
+        /usr/share/man \
+        /usr/share/info \
+        /usr/share/i18n/locales/translit_hangul \
+        /usr/share/locale/?? \
+        /usr/share/locale/??_?? \
+        /usr/share/postgresql/*/man \
+        /usr/share/postgresql-common/pg_wrapper \
+        /usr/share/vim/vim80/doc \
+        /usr/share/vim/vim80/lang \
+        /usr/share/vim/vim80/tutor \
+#        /var/lib/dpkg/info/* \
+    && find /usr/bin -xtype l -delete \
+    && find /var/log -type f -exec truncate --size 0 {} \; \
+    && find /usr/lib/python3/dist-packages -name '*test*' | xargs rm -fr \
+    && find /lib/$(uname -m)-linux-gnu/security -type f ! -name pam_env.so ! -name pam_permit.so ! -name pam_unix.so -delete
+
+# perform compression if it is necessary
+ARG COMPRESS
+RUN if [ "$COMPRESS" = "true" ]; then \
+        set -ex \
+        # Allow certain sudo commands from postgres
+        && echo 'postgres ALL=(ALL) NOPASSWD: /bin/tar xpJf /a.tar.xz -C /, /bin/rm /a.tar.xz, /bin/ln -snf dash /bin/sh' >> /etc/sudoers \
+        && ln -snf busybox /bin/sh \
+        && arch=$(uname -m) \
+        && darch=$(uname -m | sed 's/_/-/') \
+        && files="/bin/sh /usr/bin/sudo /usr/lib/sudo/sudoers.so /lib/$arch-linux-gnu/security/pam_*.so" \
+        && libs="$(ldd $files | awk '{print $3;}' | grep '^/' | sort -u) /lib/ld-linux-$darch.so.* /lib/$arch-linux-gnu/ld-linux-$darch.so.* /lib/$arch-linux-gnu/libnsl.so.* /lib/$arch-linux-gnu/libnss_compat.so.* /lib/$arch-linux-gnu/libnss_files.so.*" \
+        && (echo /var/run $files $libs | tr ' ' '\n' && realpath $files $libs) | sort -u | sed 's/^\///' > /exclude \
+        && find /etc/alternatives -xtype l -delete \
+        && save_dirs="usr lib var bin sbin etc/ssl etc/init.d etc/alternatives etc/apt" \
+        && XZ_OPT=-e9v tar -X /exclude -cpJf a.tar.xz $save_dirs \
+        # we call "cat /exclude" to avoid including files from the $save_dirs that are also among
+        # the exceptions listed in the /exclude, as "uniq -u" eliminates all non-unique lines.
+        # By calling "cat /exclude" a second time we guarantee that there will be at least two lines
+        # for each exception and therefore they will be excluded from the output passed to 'rm'.
+        && /bin/busybox sh -c "(find $save_dirs -not -type d && cat /exclude /exclude && echo exclude) | sort | uniq -u | xargs /bin/busybox rm" \
+        && /bin/busybox --install -s \
+        && /bin/busybox sh -c "find $save_dirs -type d -depth -exec rmdir -p {} \; 2> /dev/null"; \
+    else \
+        /bin/busybox --install -s; \
+    fi
+
+FROM scratch
+COPY --from=builder / /
+
+LABEL maintainer="Alexander Kukushkin <[email protected]>"
+
+ARG PG_MAJOR
+ARG COMPRESS
+ARG PGHOME
+ARG PGDATA
+ARG LC_ALL
+ARG LANG
+
+ARG PGBIN=/usr/lib/postgresql/$PG_MAJOR/bin
+
+ENV LC_ALL=$LC_ALL LANG=$LANG EDITOR=/usr/bin/editor
+ENV PGDATA=$PGDATA PATH=$PATH:$PGBIN
+
+COPY patroni /patroni/
+COPY extras/confd/conf.d/haproxy.toml /etc/confd/conf.d/
+COPY extras/confd/templates/haproxy-citus.tmpl /etc/confd/templates/haproxy.tmpl
+COPY patroni*.py docker/entrypoint.sh /
+COPY postgres?.yml $PGHOME/
+
+WORKDIR $PGHOME
+
+RUN sed -i 's/env python/&3/' /patroni*.py \
+    # "fix" patroni configs
+    && sed -i 's/^\(  connect_address:\|  - host\)/#&/' postgres?.yml \
+    && sed -i 's/^  listen: 127.0.0.1/  listen: 0.0.0.0/' postgres?.yml \
+    && sed -i "s|^\(  data_dir: \).*|\1$PGDATA|" postgres?.yml \
+    && sed -i "s|^#\(  bin_dir: \).*|\1$PGBIN|" postgres?.yml \
+    && sed -i 's/^  - encoding: UTF8/  - locale: en_US.UTF-8\n&/' postgres?.yml \
+    && sed -i 's/^scope:/log:\n  loggers:\n    patroni.postgresql.citus: DEBUG\n#&/' postgres?.yml \
+    && sed -i 's/^\(name\|etcd\|  host\|  authentication\|  pg_hba\|  parameters\):/#&/' postgres?.yml \
+    && sed -i 's/^    \(replication\|superuser\|rewind\|unix_socket_directories\|\(\(  \)\{0,1\}\(username\|password\)\)\):/#&/' postgres?.yml \
+    && sed -i 's/^postgresql:/&\n  basebackup:\n    checkpoint: fast/' postgres?.yml \
+    && sed -i 's|^      parameters:|      pg_hba:\n      - local all all trust\n      - hostssl replication all all md5 clientcert=verify-ca\n      - hostssl all all all md5 clientcert=verify-ca\n&\n        max_connections: 100\n        shared_buffers: 16MB\n        ssl: "on"\n        ssl_ca_file: /etc/ssl/certs/ssl-cert-snakeoil.pem\n        ssl_cert_file: /etc/ssl/certs/ssl-cert-snakeoil.pem\n        ssl_key_file: /etc/ssl/private/ssl-cert-snakeoil.key\n        citus.node_conninfo: "sslrootcert=/etc/ssl/certs/ssl-cert-snakeoil.pem sslkey=/etc/ssl/private/ssl-cert-snakeoil.key sslcert=/etc/ssl/certs/ssl-cert-snakeoil.pem sslmode=verify-ca"|'  postgres?.yml \
+    && sed -i 's/^#\(ctl\|  certfile\|  keyfile\)/\1/' postgres?.yml \
+    && sed -i 's|^#  cafile: .*$|  verify_client: required\n  cafile: /etc/ssl/certs/ssl-cert-snakeoil.pem|' postgres?.yml \
+    && sed -i 's|^#  cacert: .*$|  cacert: /etc/ssl/certs/ssl-cert-snakeoil.pem|' postgres?.yml \
+    && sed -i 's/^#  insecure: .*/  insecure: on/' postgres?.yml \
+    # client cert for HAProxy to access Patroni REST API
+    && if [ "$COMPRESS" = "true" ]; then chmod u+s /usr/bin/sudo; fi \
+    && chmod +s /bin/ping \
+    && chown -R postgres:postgres $PGHOME /run /etc/haproxy
+
+USER postgres
+
+ENTRYPOINT ["/bin/sh", "/entrypoint.sh"]

MAINTAINERS

@@ -1,3 +1,2 @@
-Alexander Kukushkin <alexander.kukushkin@zalando.de>
-Feike Steenbergen <feike.steenbergen@zalando.de>
-Oleksii Kliukin <[email protected]>
+Alexander Kukushkin <akukushkin@microsoft.com>
+Polina Bungina <polina.bungina@zalando.de>

README.rst

@@ -12,7 +12,9 @@ Patroni is a template for you to create your own customized, high-availability s
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
 
-Currently supported PostgreSQL versions: 9.3 to 14.
+Currently supported PostgreSQL versions: 9.3 to 15.
+
+**Note to Citus users**: Starting from 3.0 Patroni nicely integrates with the `Citus <https://github.com/citusdata/citus>`__ database extension to Postgres. Please check the `Citus support page <https://github.com/zalando/patroni/blob/master/docs/citus.rst>`__ in the Patroni documentation for more info about how to use Patroni high availability together with a Citus distributed cluster.
 
 **Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the `Kubernetes <https://github.com/zalando/patroni/blob/master/docs/kubernetes.rst>`__ chapter of the Patroni documentation.
 
@@ -47,7 +49,7 @@ We report new releases information `here <https://github.com/zalando/patroni/rel
 Community
 =========
 
-There are two places to connect with the Patroni community: `on github <https://github.com/zalando/patroni>`__, via Issues and PRs, and on channel #patroni in the `PostgreSQL Slack <https://postgres-slack.herokuapp.com/>`__.  If you're using Patroni, or just interested, please join us.
+There are two places to connect with the Patroni community: `on github <https://github.com/zalando/patroni>`__, via Issues and PRs, and on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://join.slack.com/t/postgresteam/shared_invite/zt-1qj14i9sj-E9WqIFlvcOiHsEk2yFEMjA>`__.  If you're using Patroni, or just interested, please join us.
 
 ===================================
 Technical Requirements/Installation

TODO.md

@@ -1,12 +0,0 @@
-Failover
-========
-- When determining who should become master, include the minor version of PostgreSQL in the decision.
-
-Configuration
-==============
-- Provide a way to change pg_hba.conf of a running cluster on the Patroni level, without changing individual nodes.
-- Provide hooks to store and retrieve cluster-wide passwords without exposing them in a plain-text form to unauthorized users.
-
-Documentation
-==============
-- Document how to run cascading replication and possibly initialize the cluster without an access to the master node.

docker-compose-citus.yml

@@ -0,0 +1,139 @@
+# docker compose file for running a Citus cluster
+# with 3-node etcd v3 cluster as the DCS and one haproxy node.
+# The Citus cluster has a coordinator (3 nodes)
+# and two worker clusters (2 nodes).
+#
+# Before starting it up you need to build the docker image:
+# $ docker build -f Dockerfile.citus -t patroni-citus .
+# The cluster could be started as:
+# $ docker-compose -f docker-compose-citus.yml up -d
+# You can read more about it in the:
+# https://github.com/zalando/patroni/blob/master/docker/README.md#citus-cluster
+version: "2"
+
+networks:
+    demo:
+
+services:
+    etcd1: &etcd
+        image: patroni-citus
+        networks: [ demo ]
+        environment:
+            ETCDCTL_API: 3
+            ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380
+            ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379
+            ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380
+            ETCD_INITIAL_CLUSTER_STATE: new
+            ETCD_INITIAL_CLUSTER_TOKEN: tutorial
+        container_name: demo-etcd1
+        hostname: etcd1
+        command: etcd -name etcd1 -initial-advertise-peer-urls http://etcd1:2380
+
+    etcd2:
+        <<: *etcd
+        container_name: demo-etcd2
+        hostname: etcd2
+        command: etcd -name etcd2 -initial-advertise-peer-urls http://etcd2:2380
+
+    etcd3:
+        <<: *etcd
+        container_name: demo-etcd3
+        hostname: etcd3
+        command: etcd -name etcd3 -initial-advertise-peer-urls http://etcd3:2380
+
+    haproxy:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: haproxy
+        container_name: demo-haproxy
+        ports:
+            - "5000:5000"  # Access to the coorinator primary
+            - "5001:5001"  # Load-balancing across workers primaries
+        command: haproxy
+        environment: &haproxy_env
+            ETCDCTL_API: 3
+            ETCDCTL_ENDPOINTS: http://etcd1:2379,http://etcd2:2379,http://etcd3:2379
+            PATRONI_ETCD3_HOSTS: "'etcd1:2379','etcd2:2379','etcd3:2379'"
+            PATRONI_SCOPE: demo
+            PATRONI_CITUS_GROUP: 0
+            PATRONI_CITUS_DATABASE: citus
+            PGSSLMODE: verify-ca
+            PGSSLKEY: /etc/ssl/private/ssl-cert-snakeoil.key
+            PGSSLCERT: /etc/ssl/certs/ssl-cert-snakeoil.pem
+            PGSSLROOTCERT: /etc/ssl/certs/ssl-cert-snakeoil.pem
+
+    coord1:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: coord1
+        container_name: demo-coord1
+        environment: &coord_env
+            <<: *haproxy_env
+            PATRONI_NAME: coord1
+            PATRONI_CITUS_GROUP: 0
+
+    coord2:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: coord2
+        container_name: demo-coord2
+        environment:
+            <<: *coord_env
+            PATRONI_NAME: coord2
+
+    coord3:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: coord3
+        container_name: demo-coord3
+        environment:
+            <<: *coord_env
+            PATRONI_NAME: coord3
+
+
+    work1-1:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: work1-1
+        container_name: demo-work1-1
+        environment: &work1_env
+            <<: *haproxy_env
+            PATRONI_NAME: work1-1
+            PATRONI_CITUS_GROUP: 1
+
+    work1-2:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: work1-2
+        container_name: demo-work1-2
+        environment:
+            <<: *work1_env
+            PATRONI_NAME: work1-2
+
+
+    work2-1:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: work2-1
+        container_name: demo-work2-1
+        environment: &work2_env
+            <<: *haproxy_env
+            PATRONI_NAME: work2-1
+            PATRONI_CITUS_GROUP: 2
+
+    work2-2:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: work2-2
+        container_name: demo-work2-2
+        environment:
+            <<: *work2_env
+            PATRONI_NAME: work2-2

docker-compose.yml

@@ -1,5 +1,12 @@
 # docker compose file for running a 3-node PostgreSQL cluster
 # with 3-node etcd cluster as the DCS and one haproxy node
+#
+# requires a patroni image build from the Dockerfile:
+# $ docker build -t patroni .
+# The cluster could be started as:
+# $ docker-compose up -d
+# You can read more about it in the:
+# https://github.com/zalando/patroni/blob/master/docker/README.md
 version: "2"
 
 networks:

docker/README.md

@@ -1,10 +1,10 @@
-# Patroni Dockerfile
-You can run Patroni in a docker container using this Dockerfile
+# Dockerfile and Dockerfile.citus
+You can run Patroni in a docker container using these Dockerfiles
 
-This Dockerfile is meant in aiding development of Patroni and quick testing of features. It is not a production-worthy
-Dockerfile
+They are meant in aiding development of Patroni and quick testing of features and not a production-worthy!
 
     docker build -t patroni .
+    docker build -f Dockerfile.citus -t patroni-citus .
 
 # Examples
 
@@ -12,7 +12,10 @@ Dockerfile
 
     docker run -d patroni
 
-## Three-node Patroni cluster with three-node etcd cluster and one haproxy container using docker-compose
+## Three-node Patroni cluster
+
+In addition to three Patroni containers the stack starts three containers with etcd (forming a three-node cluster), and one container with haproxy.
+The haproxy listens on ports 5000 (connects to the primary) and 5001 (does load-balancing between healthy standbys).
 
 Example session:
 
@@ -92,7 +95,8 @@ Example session:
     b2e169fcb8a34028: name=etcd1 peerURLs=http://etcd1:2380 clientURLs=http://etcd1:2379 isLeader=false
     postgres@patroni1:~$ exit
 
-    $ psql -h localhost -p 5000 -U postgres -W
+    $ docker exec -ti demo-haproxy bash
+    postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -W
     Password: postgres
     psql (11.2 (Ubuntu 11.2-1.pgdg18.04+1), server 10.7 (Debian 10.7-1.pgdg90+1))
     Type "help" for help.
@@ -105,7 +109,7 @@ Example session:
 
     localhost/postgres=# \q
 
-    $ psql -h localhost -p 5001 -U postgres -W
+    $postgres@haproxy:~ psql -h localhost -p 5001 -U postgres -W
     Password: postgres
     psql (11.2 (Ubuntu 11.2-1.pgdg18.04+1), server 10.7 (Debian 10.7-1.pgdg90+1))
     Type "help" for help.
@@ -115,3 +119,188 @@ Example session:
     ───────────────────
      t
     (1 row)
+
+## Citus cluster
+
+The stack starts three containers with etcd (forming a three-node etcd cluster), seven containers with Patroni+PostgreSQL+Citus (three coordinator nodes, and two worker clusters with two nodes each), and one container with haproxy.
+The haproxy listens on ports 5000 (connects to the coordinator primary) and 5001 (does load-balancing between worker primary nodes).
+
+Example session:
+
+    $ docker-compose -f docker-compose-citus.yml up -d
+    Creating demo-work2-1 ... done
+    Creating demo-work1-1 ... done
+    Creating demo-etcd2   ... done
+    Creating demo-etcd1   ... done
+    Creating demo-coord3  ... done
+    Creating demo-etcd3   ... done
+    Creating demo-coord1  ... done
+    Creating demo-haproxy ... done
+    Creating demo-work2-2 ... done
+    Creating demo-coord2  ... done
+    Creating demo-work1-2 ... done
+
+    $ docker ps
+    CONTAINER ID   IMAGE                  COMMAND                  CREATED         STATUS         PORTS                              NAMES
+    852d8885a612   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-coord3
+    cdd692f947ab   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-work1-2
+    9f4e340b36da   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-etcd3
+    d69c129a960a   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-etcd1
+    c5849689b8cd   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-coord1
+    c9d72bd6217d   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-work2-1
+    24b1b43efa05   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-coord2
+    cb0cc2b4ca0a   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-work2-2
+    9796c6b8aad5   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 5 seconds                                      demo-work1-1
+    8baccd74dcae   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-etcd2
+    353ec62a0187   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds   0.0.0.0:5000-5001->5000-5001/tcp   demo-haproxy
+
+    $ docker logs demo-coord1
+    2023-01-05 15:09:31,295 INFO: Selected new etcd server http://172.27.0.4:2379
+    2023-01-05 15:09:31,388 INFO: Lock owner: None; I am coord1
+    2023-01-05 15:09:31,501 INFO: trying to bootstrap a new cluster
+    ...
+    2023-01-05 15:09:45,096 INFO: postmaster pid=39
+    localhost:5432 - no response
+    2023-01-05 15:09:45.137 UTC [39] LOG:  starting PostgreSQL 15.1 (Debian 15.1-1.pgdg110+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
+    2023-01-05 15:09:45.137 UTC [39] LOG:  listening on IPv4 address "0.0.0.0", port 5432
+    2023-01-05 15:09:45.152 UTC [39] LOG:  listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
+    2023-01-05 15:09:45.177 UTC [43] LOG:  database system was shut down at 2023-01-05 15:09:32 UTC
+    2023-01-05 15:09:45.193 UTC [39] LOG:  database system is ready to accept connections
+    localhost:5432 - accepting connections
+    localhost:5432 - accepting connections
+    2023-01-05 15:09:46,139 INFO: establishing a new patroni connection to the postgres cluster
+    2023-01-05 15:09:46,208 INFO: running post_bootstrap
+    2023-01-05 15:09:47.209 UTC [55] LOG:  starting maintenance daemon on database 16386 user 10
+    2023-01-05 15:09:47.209 UTC [55] CONTEXT:  Citus maintenance daemon for database 16386 user 10
+    2023-01-05 15:09:47,215 WARNING: Could not activate Linux watchdog device: "Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'"
+    2023-01-05 15:09:47.446 UTC [41] LOG:  checkpoint starting: immediate force wait
+    2023-01-05 15:09:47,466 INFO: initialized a new cluster
+    2023-01-05 15:09:47,594 DEBUG: query(SELECT nodeid, groupid, nodename, nodeport, noderole FROM pg_catalog.pg_dist_node WHERE noderole = 'primary', ())
+    2023-01-05 15:09:47,594 INFO: establishing a new patroni connection to the postgres cluster
+    2023-01-05 15:09:47,467 INFO: Lock owner: coord1; I am coord1
+    2023-01-05 15:09:47,613 DEBUG: query(SELECT pg_catalog.citus_set_coordinator_host(%s, %s, 'primary', 'default'), ('172.27.0.6', 5432))
+    2023-01-05 15:09:47,924 INFO: no action. I am (coord1), the leader with the lock
+    2023-01-05 15:09:51.282 UTC [41] LOG:  checkpoint complete: wrote 1086 buffers (53.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.029 s, sync=3.746 s, total=3.837 s; sync files=280, longest=0.028 s, average=0.014 s; distance=8965 kB, estimate=8965 kB
+    2023-01-05 15:09:51.283 UTC [41] LOG:  checkpoint starting: immediate force wait
+    2023-01-05 15:09:51.495 UTC [41] LOG:  checkpoint complete: wrote 18 buffers (0.9%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.044 s, sync=0.091 s, total=0.212 s; sync files=15, longest=0.015 s, average=0.007 s; distance=67 kB, estimate=8076 kB
+    2023-01-05 15:09:57,467 INFO: Lock owner: coord1; I am coord1
+    2023-01-05 15:09:57,569 INFO: Assigning synchronous standby status to ['coord3']
+    server signaled
+    2023-01-05 15:09:57.574 UTC [39] LOG:  received SIGHUP, reloading configuration files
+    2023-01-05 15:09:57.580 UTC [39] LOG:  parameter "synchronous_standby_names" changed to "coord3"
+    2023-01-05 15:09:59,637 INFO: Synchronous standby status assigned to ['coord3']
+    2023-01-05 15:09:59,638 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.27.0.2', 5432, 1))
+    2023-01-05 15:09:59.690 UTC [67] LOG:  standby "coord3" is now a synchronous standby with priority 1
+    2023-01-05 15:09:59.690 UTC [67] STATEMENT:  START_REPLICATION SLOT "coord3" 0/3000000 TIMELINE 1
+    2023-01-05 15:09:59,694 INFO: no action. I am (coord1), the leader with the lock
+    2023-01-05 15:09:59,704 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.27.0.8', 5432, 2))
+    2023-01-05 15:10:07,625 INFO: no action. I am (coord1), the leader with the lock
+    2023-01-05 15:10:17,579 INFO: no action. I am (coord1), the leader with the lock
+
+    $ docker exec -ti demo-haproxy bash
+    postgres@haproxy:~$ etcdctl member list
+    1bab629f01fa9065, started, etcd3, http://etcd3:2380, http://172.27.0.10:2379
+    8ecb6af518d241cc, started, etcd2, http://etcd2:2380, http://172.27.0.4:2379
+    b2e169fcb8a34028, started, etcd1, http://etcd1:2380, http://172.27.0.7:2379
+
+    postgres@haproxy:~$ etcdctl get --keys-only --prefix /service/demo
+    /service/demo/0/config
+    /service/demo/0/initialize
+    /service/demo/0/leader
+    /service/demo/0/members/coord1
+    /service/demo/0/members/coord2
+    /service/demo/0/members/coord3
+    /service/demo/0/status
+    /service/demo/0/sync
+    /service/demo/1/config
+    /service/demo/1/initialize
+    /service/demo/1/leader
+    /service/demo/1/members/work1-1
+    /service/demo/1/members/work1-2
+    /service/demo/1/status
+    /service/demo/1/sync
+    /service/demo/2/config
+    /service/demo/2/initialize
+    /service/demo/2/leader
+    /service/demo/2/members/work2-1
+    /service/demo/2/members/work2-2
+    /service/demo/2/status
+    /service/demo/2/sync
+
+    postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus
+    Password for user postgres: postgres
+    psql (15.1 (Debian 15.1-1.pgdg110+1))
+    SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off)
+    Type "help" for help.
+
+    citus=# select pg_is_in_recovery();
+     pg_is_in_recovery
+    -------------------
+     f
+    (1 row)
+
+    citus=# table pg_dist_node;
+     nodeid | groupid |  nodename  | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards
+    --------+---------+------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------
+          1 |       0 | 172.27.0.6 |     5432 | default  | t           | t        | primary  | default     | t              | f
+          2 |       1 | 172.27.0.2 |     5432 | default  | t           | t        | primary  | default     | t              | t
+          3 |       2 | 172.27.0.8 |     5432 | default  | t           | t        | primary  | default     | t              | t
+    (3 rows)
+
+    citus=# \q
+
+    postgres@haproxy:~$ patronictl list
+    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+    |     0 | coord1  | 172.27.0.6  | Leader       | running |  1 |           |
+    |     0 | coord2  | 172.27.0.5  | Replica      | running |  1 |         0 |
+    |     0 | coord3  | 172.27.0.9  | Sync Standby | running |  1 |         0 |
+    |     1 | work1-1 | 172.27.0.2  | Leader       | running |  1 |           |
+    |     1 | work1-2 | 172.27.0.12 | Sync Standby | running |  1 |         0 |
+    |     2 | work2-1 | 172.27.0.11 | Sync Standby | running |  1 |         0 |
+    |     2 | work2-2 | 172.27.0.8  | Leader       | running |  1 |           |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+
+    postgres@haproxy:~$ patronictl switchover --group 2 --force
+    Current cluster topology
+    + Citus cluster: demo (group: 2, 7185185529556963355) +-----------+
+    | Member  | Host        | Role         | State   | TL | Lag in MB |
+    +---------+-------------+--------------+---------+----+-----------+
+    | work2-1 | 172.27.0.11 | Sync Standby | running |  1 |         0 |
+    | work2-2 | 172.27.0.8  | Leader       | running |  1 |           |
+    +---------+-------------+--------------+---------+----+-----------+
+    2023-01-05 15:29:29.54204 Successfully switched over to "work2-1"
+    + Citus cluster: demo (group: 2, 7185185529556963355) -------+
+    | Member  | Host        | Role    | State   | TL | Lag in MB |
+    +---------+-------------+---------+---------+----+-----------+
+    | work2-1 | 172.27.0.11 | Leader  | running |  1 |           |
+    | work2-2 | 172.27.0.8  | Replica | stopped |    |   unknown |
+    +---------+-------------+---------+---------+----+-----------+
+
+    postgres@haproxy:~$ patronictl list
+    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+    |     0 | coord1  | 172.27.0.6  | Leader       | running |  1 |           |
+    |     0 | coord2  | 172.27.0.5  | Replica      | running |  1 |         0 |
+    |     0 | coord3  | 172.27.0.9  | Sync Standby | running |  1 |         0 |
+    |     1 | work1-1 | 172.27.0.2  | Leader       | running |  1 |           |
+    |     1 | work1-2 | 172.27.0.12 | Sync Standby | running |  1 |         0 |
+    |     2 | work2-1 | 172.27.0.11 | Leader       | running |  2 |           |
+    |     2 | work2-2 | 172.27.0.8  | Sync Standby | running |  2 |         0 |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+
+    postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus
+    Password for user postgres: postgres
+    psql (15.1 (Debian 15.1-1.pgdg110+1))
+    SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off)
+    Type "help" for help.
+
+    citus=# table pg_dist_node;
+     nodeid | groupid |  nodename   | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards
+    --------+---------+-------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------
+          1 |       0 | 172.27.0.6  |     5432 | default  | t           | t        | primary  | default     | t              | f
+          3 |       2 | 172.27.0.11 |     5432 | default  | t           | t        | primary  | default     | t              | t
+          2 |       1 | 172.27.0.2  |     5432 | default  | t           | t        | primary  | default     | t              | t
+    (3 rows)

docker/entrypoint.sh

@@ -7,29 +7,36 @@ if [ -f /a.tar.xz ]; then
     sudo ln -snf dash /bin/sh
 fi
 
-readonly PATRONI_SCOPE=${PATRONI_SCOPE:-batman}
-PATRONI_NAMESPACE=${PATRONI_NAMESPACE:-/service}
-readonly PATRONI_NAMESPACE=${PATRONI_NAMESPACE%/}
-readonly DOCKER_IP=$(hostname --ip-address)
+readonly PATRONI_SCOPE="${PATRONI_SCOPE:-batman}"
+PATRONI_NAMESPACE="${PATRONI_NAMESPACE:-/service}"
+readonly PATRONI_NAMESPACE="${PATRONI_NAMESPACE%/}"
+DOCKER_IP=$(hostname --ip-address)
+readonly DOCKER_IP
 
 case "$1" in
     haproxy)
         haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D
-        CONFD="confd -prefix=$PATRONI_NAMESPACE/$PATRONI_SCOPE -interval=10 -backend"
-        if [ ! -z "$PATRONI_ZOOKEEPER_HOSTS" ]; then
-            while ! /usr/share/zookeeper/bin/zkCli.sh -server $PATRONI_ZOOKEEPER_HOSTS ls /; do
+        set -- confd "-prefix=$PATRONI_NAMESPACE/$PATRONI_SCOPE" -interval=10 -backend
+        if [ -n "$PATRONI_ZOOKEEPER_HOSTS" ]; then
+            while ! /usr/share/zookeeper/bin/zkCli.sh -server "$PATRONI_ZOOKEEPER_HOSTS" ls /; do
                 sleep 1
             done
-            exec dumb-init $CONFD zookeeper -node $PATRONI_ZOOKEEPER_HOSTS
+            set -- "$@" zookeeper -node "$PATRONI_ZOOKEEPER_HOSTS"
         else
-            while ! etcdctl cluster-health 2> /dev/null; do
+            while ! etcdctl member list 2> /dev/null; do
                 sleep 1
             done
-            exec dumb-init $CONFD etcdv3 -node $(echo $ETCDCTL_ENDPOINTS | sed 's/,/ -node /g')
+            set -- "$@" etcdv3
+            while IFS='' read -r line; do
+                set -- "$@" -node "$line"
+            done <<-EOT
+$(echo "$ETCDCTL_ENDPOINTS" | sed 's/,/\n/g')
+EOT
         fi
+        exec dumb-init "$@"
         ;;
     etcd)
-        exec "$@" -advertise-client-urls http://$DOCKER_IP:2379
+        exec "$@" -advertise-client-urls "http://$DOCKER_IP:2379"
         ;;
     zookeeper)
         exec /usr/share/zookeeper/bin/zkServer.sh start-foreground
@@ -56,5 +63,13 @@ export PATRONI_REPLICATION_USERNAME="${PATRONI_REPLICATION_USERNAME:-replicator}
 export PATRONI_REPLICATION_PASSWORD="${PATRONI_REPLICATION_PASSWORD:-replicate}"
 export PATRONI_SUPERUSER_USERNAME="${PATRONI_SUPERUSER_USERNAME:-postgres}"
 export PATRONI_SUPERUSER_PASSWORD="${PATRONI_SUPERUSER_PASSWORD:-postgres}"
+export PATRONI_REPLICATION_SSLMODE="${PATRONI_REPLICATION_SSLMODE:-$PGSSLMODE}"
+export PATRONI_REPLICATION_SSLKEY="${PATRONI_REPLICATION_SSLKEY:-$PGSSLKEY}"
+export PATRONI_REPLICATION_SSLCERT="${PATRONI_REPLICATION_SSLCERT:-$PGSSLCERT}"
+export PATRONI_REPLICATION_SSLROOTCERT="${PATRONI_REPLICATION_SSLROOTCERT:-$PGSSLROOTCERT}"
+export PATRONI_SUPERUSER_SSLMODE="${PATRONI_SUPERUSER_SSLMODE:-$PGSSLMODE}"
+export PATRONI_SUPERUSER_SSLKEY="${PATRONI_SUPERUSER_SSLKEY:-$PGSSLKEY}"
+export PATRONI_SUPERUSER_SSLCERT="${PATRONI_SUPERUSER_SSLCERT:-$PGSSLCERT}"
+export PATRONI_SUPERUSER_SSLROOTCERT="${PATRONI_SUPERUSER_SSLROOTCERT:-$PGSSLROOTCERT}"
 
 exec python3 /patroni.py postgres0.yml

docs/CONTRIBUTING.rst

@@ -8,7 +8,7 @@ Wanna contribute to Patroni? Yay - here is how!
 Chatting
 --------
 
-Just want to chat with other Patroni users?  Looking for interactive troubleshooting help? Join us on channel #patroni in the `PostgreSQL Slack <https://postgres-slack.herokuapp.com/>`__.
+Just want to chat with other Patroni users?  Looking for interactive troubleshooting help? Join us on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://join.slack.com/t/postgresteam/shared_invite/zt-1qj14i9sj-E9WqIFlvcOiHsEk2yFEMjA>`__.
 
 Running tests
 -------------

docs/ENVIRONMENT.rst

@@ -33,6 +33,13 @@ It is possible to create new database users right after the successful initializ
 
 Example: defining ``PATRONI_admin_PASSWORD=strongpasswd`` and ``PATRONI_admin_OPTIONS='createrole,createdb'`` will cause creation of the user **admin** with the password **strongpasswd** that is allowed to create other users and databases.
 
+Citus
+-----
+Enables integration Patroni with `Citus <https://docs.citusdata.com>`__. If configured, Patroni will take care of registering Citus worker nodes on the coordinator. You can find more information about Citus support :ref:`here <citus>`.
+
+-  **PATRONI\_CITUS\_GROUP**: the Citus group id, integer. Use ``0`` for coordinator and ``1``, ``2``, etc... for workers
+-  **PATRONI\_CITUS\_DATABASE**: the database where ``citus`` extension should be created. Must be the same on the coordinator and all workers. Currently only one database is supported.
+
 Consul
 ------
 -  **PATRONI\_CONSUL\_HOST**: the host:port for the Consul local agent.
@@ -47,7 +54,8 @@ Consul
 -  **PATRONI\_CONSUL\_DC**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
 -  **PATRONI\_CONSUL\_CONSISTENCY**: (optional) Select consul consistency mode. Possible values are ``default``, ``consistent``, or ``stale`` (more details in `consul API reference <https://www.consul.io/api/features/consistency.html/>`__)
 -  **PATRONI\_CONSUL\_CHECKS**: (optional) list of Consul health checks used for the session. By default an empty list is used.
--  **PATRONI\_CONSUL\_REGISTER\_SERVICE**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, replica or standby-leader depending on the node's role. Defaults to **false**
+-  **PATRONI\_CONSUL\_REGISTER\_SERVICE**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, primary, replica, or standby-leader depending on the node's role. Defaults to **false**
+-  **PATRONI\_CONSUL\_SERVICE\_TAGS**: (optional) additional static tags to add to the Consul service apart from the role (``master``/``primary``/``replica``/``standby-leader``). By default an empty list is used.
 -  **PATRONI\_CONSUL\_SERVICE\_CHECK\_INTERVAL**: (optional) how often to perform health check against registered url
 -  **PATRONI\_CONSUL\_SERVICE\_CHECK\_TLS\_SERVER\_NAME**: (optional) overide SNI host when connecting via TLS, see also `consul agent check API reference <https://www.consul.io/api-docs/agent/check#tlsservername>`__.
 
@@ -109,9 +117,10 @@ Kubernetes
 -  **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
 -  **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='[{"name": "postgresql", "port": 5432}]'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
 -  **PATRONI\_KUBERNETES\_CACERT**: (optional) Specifies the file with the CA_BUNDLE file with certificates of trusted CAs to use while verifying Kubernetes API SSL certs. If not provided, patroni will use the value provided by the ServiceAccount secret.
+-  **PATRONI\_RETRIABLE\_HTTP\_CODES**: (optional) list of HTTP status codes from K8s API to retry on. By default Patroni is retrying on ``500``, ``503``, and ``504``, or if K8s API response has ``retry-after`` HTTP header.
 
-Raft
-----
+Raft (deprecated)
+-----------------
 
 -  **PATRONI\_RAFT\_SELF\_ADDR**: ``ip:port`` to listen on for Raft connections. The ``self_addr`` must be accessible from other nodes of the cluster. If not set, the node will not participate in consensus.
 -  **PATRONI\_RAFT\_BIND\_ADDR**: (optional) ``ip:port`` to listen on for Raft connections. If not specified the ``self_addr`` will be used.
@@ -123,11 +132,12 @@ PostgreSQL
 ----------
 -  **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
 -  **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications.
+-  **PATRONI\_POSTGRESQL\_PROXY\_ADDRESS**: IP address + port through which a connection pool (e.g. pgbouncer) running next to Postgres is accessible. The value is written to the member key in DCS as ``proxy_url`` and could be used/useful for service discovery.
 -  **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
 -  **PATRONI\_POSTGRESQL\_CONFIG\_DIR**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni.
 -  **PATRONI\_POSTGRESQL\_BIN_DIR**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
 -  **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
--  **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
+-  **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access the replication source via streaming replication
 -  **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization.
 -  **PATRONI\_REPLICATION\_SSLMODE**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
 -  **PATRONI\_REPLICATION\_SSLKEY**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.

docs/README.rst

@@ -109,7 +109,7 @@ Planning the Number of PostgreSQL Nodes
 ---------------------------------------
 
 Patroni/PostgreSQL nodes are decoupled from DCS nodes (except when Patroni implements RAFT on its own) and therefore
-there is no requirement on the minimal number of nodes. Running a cluster consisting of one master and one standby is 
+there is no requirement on the minimal number of nodes. Running a cluster consisting of one primary and one standby is
 perfectly fine. You can add more standby nodes later.
 
 Running and Configuring
@@ -177,7 +177,7 @@ Testing an HA solution is a time consuming process, with many variables. This is
 That said, here are some pieces of your infrastructure you should be sure to test:
 
 * Network (the network in front of your system as well as the NICs [physical or virtual] themselves)
-* Disk IO 
+* Disk IO
 * file limits (nofile in Linux)
 * RAM. Even if you have oomkiller turned off as suggested, the unavailability of RAM could cause issues.
 * CPU

docs/SETTINGS.rst

@@ -17,21 +17,22 @@ Dynamic configuration is stored in the DCS (Distributed Configuration Store) and
 -  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
 -  **maximum\_lag\_on\_syncnode**: the maximum bytes a synchronous follower may lag before it is considered as an unhealthy candidate and swapped by healthy asynchronous follower. Patroni utilize the max replica lsn if there is more than one follower, otherwise it will use leader's current wal lsn. Default is -1, Patroni will not take action to swap synchronous unhealthy follower when the value is set to 0 or below. Please set the value high enough so Patroni won't swap synchrounous follower fequently during high transaction volume.
 -  **max\_timelines\_history**: maximum number of timeline history items kept in DCS.  Default value: 0. When set to 0, it keeps the full history in DCS.
--  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered (in seconds). Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
-- **master\_stop\_timeout**: The number of seconds Patroni is allowed to wait when stopping Postgres and effective only when synchronous_mode is enabled. When set to > 0 and the synchronous_mode is enabled, Patroni sends SIGKILL to the postmaster if the stop operation is running for more than the value set by master_stop_timeout. Set the value according to your durability/availability tradeoff. If the parameter is not set or set <= 0, master_stop_timeout does not apply.
+-  **primary\_start\_timeout**: the amount of time a primary is allowed to recover from failures before failover is triggered (in seconds). Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Worst case failover time for primary failure is: loop\_wait + primary\_start\_timeout + loop\_wait, unless primary\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
+- **primary\_stop\_timeout**: The number of seconds Patroni is allowed to wait when stopping Postgres and effective only when synchronous_mode is enabled. When set to > 0 and the synchronous_mode is enabled, Patroni sends SIGKILL to the postmaster if the stop operation is running for more than the value set by primary\_stop\_timeout. Set the value according to your durability/availability tradeoff. If the parameter is not set or set <= 0, primary\_stop\_timeout does not apply.
 -  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
--  **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the master. See :ref:`replication modes documentation <replication_modes>` for details.
+-  **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the primary. See :ref:`replication modes documentation <replication_modes>` for details.
+-  **failsafe\_mode**: Enables :ref:`DCS Failsafe Mode <dcs_failsafe_mode>`. Defaults to `false`.
 -  **postgresql**:
     -  **use\_pg\_rewind**: whether or not to use pg_rewind. Defaults to `false`.
     -  **use\_slots**: whether or not to use replication slots. Defaults to `true` on PostgreSQL 9.4+.
     -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. There is no recovery.conf anymore in PostgreSQL 12, but you may continue using this section, because Patroni handles it transparently.
     -  **parameters**: list of configuration settings for Postgres.
 -  **standby\_cluster**: if this section is defined, we want to bootstrap a standby cluster.
-    -  **host**: an address of remote master
-    -  **port**: a port of remote master
-    -  **primary\_slot\_name**: which slot on the remote master to use for replication. This parameter is optional, the default value is derived from the instance name (see function `slot_name_from_member_name`).
-    -  **create\_replica\_methods**: an ordered list of methods that can be used to bootstrap standby leader from the remote master, can be different from the list defined in :ref:`postgresql_settings`
-    -  **restore\_command**: command to restore WAL records from the remote master to standby leader, can be different from the list defined in :ref:`postgresql_settings`
+    -  **host**: an address of remote node
+    -  **port**: a port of remote node
+    -  **primary\_slot\_name**: which slot on the remote node to use for replication. This parameter is optional, the default value is derived from the instance name (see function `slot_name_from_member_name`).
+    -  **create\_replica\_methods**: an ordered list of methods that can be used to bootstrap standby leader from the remote primary, can be different from the list defined in :ref:`postgresql_settings`
+    -  **restore\_command**: command to restore WAL records from the remote primary to nodes in a standby cluster, can be different from the list defined in :ref:`postgresql_settings`
     -  **archive\_cleanup\_command**: cleanup command for standby leader
     -  **recovery\_min\_apply\_delay**: how long to wait before actually apply WAL records on a standby leader
 -  **slots**: define permanent replication slots. These slots will be preserved during switchover/failover. The logical slots are copied from the primary to a standby with restart, and after that their position advanced every **loop_wait** seconds (if necessary). Copying logical slot files performed via ``libpq`` connection and using either rewind or superuser credentials (see **postgresql.authentication** section). There is always a chance that the logical slot position on the replica is a bit behind the former primary, therefore application should be prepared that some messages could be received the second time after the failover. The easiest way of doing so - tracking ``confirmed_flush_lsn``. Enabling permanent logical replication slots requires **postgresql.use_slots** to be set and will also automatically enable the ``hot_standby_feedback``. Since the failover of logical replication slots is unsafe on PostgreSQL 9.6 and older and PostgreSQL version 10 is missing some important functions, the feature only works with PostgreSQL 11+.
@@ -111,6 +112,15 @@ Bootstrap configuration
                 -  **- createdb**
     -  **post\_bootstrap** or **post\_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
 
+.. _citus_settings:
+
+Citus
+-----
+Enables integration Patroni with `Citus <https://docs.citusdata.com>`__. If configured, Patroni will take care of registering Citus worker nodes on the coordinator. You can find more information about Citus support :ref:`here <citus>`.
+
+-  **group**: the Citus group id, integer. Use ``0`` for coordinator and ``1``, ``2``, etc... for workers
+-  **database**: the database where ``citus`` extension should be created. Must be the same on the coordinator and all workers. Currently only one database is supported.
+
 .. _consul_settings:
 
 Consul
@@ -129,9 +139,9 @@ Most of the parameters are optional, but you have to specify one of the **host**
 -  **dc**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
 -  **consistency**: (optional) Select consul consistency mode. Possible values are ``default``, ``consistent``, or ``stale`` (more details in `consul API reference <https://www.consul.io/api/features/consistency.html/>`__)
 -  **checks**: (optional) list of Consul health checks used for the session. By default an empty list is used.
--  **register\_service**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, replica or standby-leader depending on the node's role. Defaults to **false**.
--  **service\_tags**: (optional) additional static tags to add to the Consul service apart from the role (``master``/``replica``/``standby-leader``).  By default an empty list is used.
--  **service\_check\_interval**: (optional) how often to perform health check against registered url.
+-  **register\_service**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, primary, replica, or standby-leader depending on the node's role. Defaults to **false**.
+-  **service\_tags**: (optional) additional static tags to add to the Consul service apart from the role (``master``/``primary``/``replica``/``standby-leader``). By default an empty list is used.
+-  **service\_check\_interval**: (optional) how often to perform health check against registered url. Defaults to '5s'.
 -  **service\_check\_tls\_server\_name**: (optional) overide SNI host when connecting via TLS, see also `consul agent check API reference <https://www.consul.io/api-docs/agent/check#tlsservername>`__.
 
 The ``token`` needs to have the following ACL permissions:
@@ -183,7 +193,7 @@ ZooKeeper
 -  **key**: (optional) File with the client key.
 -  **key_password**: (optional) The client key password.
 -  **verify**: (optional) Whether to verify certificate or not. Defaults to ``true``.
--  **set_acls**: (optional) If set, configure Kazoo to apply a default ACL to each ZNode that it creates. ACLs will assume 'x509' schema and should be specified as a dictionary with the principal as the key and one or more permissions as a list in the value.  Permissions may be one of ``CREATE``, ``READ``, ``WRITE``, ``DELETE`` or ``ADMIN``.  For example, ``set_acls: {CN=principal1: [CREATE, READ], CN=principal2: [ALL]}``. 
+-  **set_acls**: (optional) If set, configure Kazoo to apply a default ACL to each ZNode that it creates. ACLs will assume 'x509' schema and should be specified as a dictionary with the principal as the key and one or more permissions as a list in the value.  Permissions may be one of ``CREATE``, ``READ``, ``WRITE``, ``DELETE`` or ``ADMIN``.  For example, ``set_acls: {CN=principal1: [CREATE, READ], CN=principal2: [ALL]}``.
 
 .. note::
     It is required to install ``kazoo>=2.6.0`` to support SSL.
@@ -208,12 +218,13 @@ Kubernetes
 -  **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
 -  **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: [{"name": "postgresql", "port": 5432}]`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set.
 -  **cacert**: (optional) Specifies the file with the CA_BUNDLE file with certificates of trusted CAs to use while verifying Kubernetes API SSL certs. If not provided, patroni will use the value provided by the ServiceAccount secret.
+-  **retriable\_http\_codes**: (optional) list of HTTP status codes from K8s API to retry on. By default Patroni is retrying on ``500``, ``503``, and ``504``, or if K8s API response has ``retry-after`` HTTP header.
 
 
 .. _raft_settings:
 
-Raft
-----
+Raft (deprecated)
+-----------------
 -  **self\_addr**: ``ip:port`` to listen on for Raft connections. The ``self_addr`` must be accessible from other nodes of the cluster. If not set, the node will not participate in consensus.
 -  **bind\_addr**: (optional) ``ip:port`` to listen on for Raft connections. If not specified the ``self_addr`` will be used.
 -  **partner\_addrs**: list of other Patroni nodes in the cluster in format: ['ip1:port', 'ip2:port', 'etc...']
@@ -224,7 +235,7 @@ Raft
 
   - Q: How to list all the nodes providing consensus?
 
-    A: ``syncobj_admin -conn host:port`` -status where the host:port is the address of one of the cluster nodes
+    A: ``syncobj_admin -conn host:port -status`` where the host:port is the address of one of the cluster nodes
 
   - Q: Node that was a part of consensus and has gone and I can't reuse the same IP for other node. How to remove this node from the consensus?
 
@@ -262,7 +273,7 @@ PostgreSQL
             -  **gssencmode**: (optional) maps to the `gssencmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-GSSENCMODE>`__ connection parameter, which determines whether or with what priority a secure GSS TCP/IP connection will be negotiated with the server
             -  **channel_binding**: (optional) maps to the `channel_binding <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-CHANNEL-BINDING>`__ connection parameter, which controls the client's use of channel binding.
         -  **replication**:
-            -  **username**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
+            -  **username**: replication username; the user will be created during initialization. Replicas will use this user to access the replication source via streaming replication
             -  **password**: replication password; the user will be created during initialization.
             -  **sslmode**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
             -  **sslkey**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
@@ -292,6 +303,7 @@ PostgreSQL
             -  **on\_start**: run this script when the postgres starts.
             -  **on\_stop**: run this script when the postgres stops.
     -  **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications.
+    -  **proxy\_address**: IP address + port through which a connection pool (e.g. pgbouncer) running next to Postgres is accessible. The value is written to the member key in DCS as ``proxy_url`` and could be used/useful for service discovery.
     -  **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica.
        "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its
        own config item. See :ref:`custom replica creation methods documentation <custom_replica_creation>` for further explanation.
@@ -314,14 +326,16 @@ PostgreSQL
     -  **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds.
     -  **use\_pg\_rewind**: try to use pg\_rewind on the former leader when it joins cluster as a replica.
     -  **remove\_data\_directory\_on\_rewind\_failure**: If this option is enabled, Patroni will remove the PostgreSQL data directory and recreate the replica. Otherwise it will try to follow the new leader. Default value is **false**.
-    -  **remove\_data\_directory\_on\_diverged\_timelines**: Patroni will remove the PostgreSQL data directory and recreate the replica if it notices that timelines are diverging and the former master can not start streaming from the new master. This option is useful when ``pg_rewind`` can not be used. While performing timelines divergence check on PostgreSQL v10 and older Patroni will try to connect with replication credential to the "postgres" database. Hence, such access should be allowed in the  pg_hba.conf. Default value is **false**.
+    -  **remove\_data\_directory\_on\_diverged\_timelines**: Patroni will remove the PostgreSQL data directory and recreate the replica if it notices that timelines are diverging and the former primary can not start streaming from the new primary. This option is useful when ``pg_rewind`` can not be used. While performing timelines divergence check on PostgreSQL v10 and older Patroni will try to connect with replication credential to the "postgres" database. Hence, such access should be allowed in the  pg_hba.conf. Default value is **false**.
     -  **replica\_method**: for each create_replica_methods other than basebackup, you would add a configuration section of the same name. At a minimum, this should include "command" with a full path to the actual script to be executed. Other configuration parameters will be passed along to the script in the form "parameter=value".
     -  **pre\_promote**: a fencing script that executes during a failover after acquiring the leader lock but before promoting the replica. If the script exits with a non-zero code, Patroni does not promote the replica and removes the leader key from DCS.
 
+.. _restapi_settings:
+
 REST API
 --------
 -  **restapi**:
-        -  **connect\_address**: IP address (or hostname) and port, to access the Patroni's :ref:`REST API <rest_api>`. All the members of the cluster must be able to connect to this address, so unless the Patroni setup is intended for a demo inside the localhost, this address must be a non "localhost" or loopback address (ie: "localhost" or "127.0.0.1"). It can serve as an endpoint for HTTP health checks (read below about the "listen" REST API parameter), and also for user queries (either directly or via the REST API), as well as for the health checks done by the cluster members during leader elections (for example, to determine whether the master is still running, or if there is a node which has a WAL position that is ahead of the one doing the query; etc.) The connect_address is put in the member key in DCS, making it possible to translate the member name into the address to connect to its REST API.
+        -  **connect\_address**: IP address (or hostname) and port, to access the Patroni's :ref:`REST API <rest_api>`. All the members of the cluster must be able to connect to this address, so unless the Patroni setup is intended for a demo inside the localhost, this address must be a non "localhost" or loopback address (ie: "localhost" or "127.0.0.1"). It can serve as an endpoint for HTTP health checks (read below about the "listen" REST API parameter), and also for user queries (either directly or via the REST API), as well as for the health checks done by the cluster members during leader elections (for example, to determine whether the leader is still running, or if there is a node which has a WAL position that is ahead of the one doing the query; etc.) The connect_address is put in the member key in DCS, making it possible to translate the member name into the address to connect to its REST API.
 
         -  **listen**: IP address (or hostname) and port that Patroni will listen to for the REST API - to provide also the same health checks and cluster messaging between the participating nodes, as described above. to provide health-check information for HAProxy (or any other load balancer capable of doing a HTTP "OPTION" or "GET" checks).
 

docs/_static/multi-dc-asynchronous-replication.drawio

@@ -0,0 +1 @@
+<mxfile host="app.diagrams.net" modified="2023-03-13T14:29:21.924Z" agent="5.0 (X11; Ubuntu)" etag="sukwsRuBYbiX8e-LLYnw" version="21.0.6" type="device"><diagram name="Page-1" id="Xu3tU9JEMeQEUPilRV_D">7Vxtb9s2EP41BrYPNfTmt4+Jk2bFOixrihXYF4O2aEkNLaoUZTv99SMlUhZF+i2RE8dVEsDiiTxKd88deXeMO+54sb4jIAn/wj5EHcfy1x33puM4tud67INTngrKkLc4ISCRLzptCA/RTyiIlqBmkQ9TpSPFGNEoUYkzHMdwRhUaIASv1G5zjNRZExBAjfAwA0infot8Ggqq3R9tbvwBoyCk8v0GxY0FkJ3Fm6Qh8PGqQnJvO+6YYEyLq8V6DBEXnpRLMe7jlrvlgxEY00MG/L3474v99d/Hb+jnn596d99vsjj7ILgsAcrEC9+MhYJS+iSFkOAoprkge9fsj80ztjo9dmfMW12nVyPU2wOVYOstzkMl1NsDlWDX2du1+e36A1YIWkthb9XmtyoPyP7ca5xRFMVwXELOYsSAAD9iqhhjhAmjxThm0rsO6QKxls0uV2FE4UMCZlyqK2YujDbHMRWgtx3ZFoLnXBmsKWBzEcEj1wQkt0tYKKTogxBI0mhajiJwlpE0WsIvMC2YcyoDYMKvF+uA22oXrFKvGxCcJfnjf2JzGe9O2OVkhnDmcyaU4EcoX7LjuOz3Iwfc9TxCqPbyS0hoxGzpCkUB500xnwqIFoJzyjkyiURx8Dlv3biWkIJpCh+kIfTF6+j4l2Bms8J1hSTs4Q7iBaTkiXURd3vSNoVz8kRztbF0T9LCipF7chwQ3iUoWW8MkF0IGzzCHh3NHu8Bk3gcaTZpELemm95VfzzsVwVnb9VKHXk1HZSsTCiugFzXyk6/c7CqbHvAzXq3shyrpyur7Ni4slxdWXd8TJxSEDP5OH3EAT4l7CrIoc7o/vRJ02X6COksFII3epdtFrHF6xxmpYw+z3/qpiUR8hlMIbrHaUSj3DdMMaV4sdewZ5D7KBUX+xwdSJPibefRGvrbvBWBKc7IDBa+ivm51OS1/OlE6mAiRX5CZI5UJzLQcdk3+JD+qVDpHYtKAoHPOhCYIKbTFpyvB04u+YmU+wkR2lMRar81RHstRFuIKhB13DODaF+D6O3X8Q0PNFGWcu04lh4nKTisKE8BR76BSmthgIoqK/8x4bDEWz0k61pWHmR1+24t+BLxVY06MlKLOK3Wc7SF8SAfze4bmNg1mjOs9c0Dqb12olmE2XDqWH/MppDEkIm5GxVIT2R4wxTkn8zPDlUQu44O4qEpnBieCMQDDcQtZFvIViHbPzPEDjWAQj+AcqHDhIY4wDFAtxvqNcFZ7JdY3fT5jLmkczR/h5Q+ieUTZBS/JGYtVtAd/URmkAISwF38xBLDX3CnqghEgEZLNSFpkrwYes/trLK01r2S56ksigcVo2r6Kx/j+SodtU6odUI7nRDT23l5IVl8eNduaPBWbuhlorcvQPT9A0U/Oi/R6475ckU/OC/R66nkG74WFHkQtujFil76PzJeNSyWxA9iTbziYiQwF6nsIPMn7JMtgqPiChWUjwVb2aEt+bUlv03Jb4ZJggmgcOIDCiblPmJ7iel05T+9itVQ+c9Ttx1vX/2zDbn7yyz/ucfqyrbPrfpnt1nsS89iH49Sr16lfvNEtq0nAVuY/uIwdZzzg+lQg6lWcNFDwzZxdFGJo+P97blVXOw229mC9p3VXJyzznYK8e5N/JSnw/dlfuRKc+l1lzIebl1R64reS+XFgNF36IvkLuANfNHLpO9ehPSHB0pfyvFcpO/9UtKXRvL60n+kftr/Z/jtGi7DW39l/0juoeEwv8yM+NGyUkepJUsq9RRDv5xkKNzwhMKHIk/Pyzb9ZN3ZUrY5fjqVxGc65BFsd88zHMzIa4pRrylG+8R7MKNBU4yGTTEaNcSIeYCGGNlNMXKaYtQUsp1tyL6vWGXBTDPWX5qsuKV6BJIHDPJfax11bapvk+cIr2YhILTLq5JTkPIV0FSROtmG2altmAc9bb/slrnV6o6510Di1Lhu6QfVj1x87KYMbSsjY73hQLic2as0xigh0QJw9e+UhnHhf4aVMXRT1bT2BqqLyPeLHSY/UAA2Jw3UQJ5HxXxTKQ4d5Far5ABEkcdQr65UWdjW95RVuUGt2OHqUe7IFOWeymbPZKNvOFO1a2u8d0fvntWGXi/PS2MJ3WearjVDIE2VQTSiCL6Cw9l6BixntBKo5axiTBYAGZk9UALBIooD1u1LUWbME1iO9RtIn+JZSHCMs/T3ildRD4nt80FcsltcEEdFnjY7nR/SEb7L+jT/UX6JiJikU/2eDpNfsbqW5wwV1yJt4Lm5Y9kFz+cpPDItzJqbbxMpum++k8W9/R8=</diagram></mxfile>

docs/_static/multi-dc-synchronous-replication.drawio

@@ -0,0 +1 @@
+<mxfile host="app.diagrams.net" modified="2023-03-13T14:25:32.295Z" agent="5.0 (X11; Ubuntu)" etag="EcVEbU6F-AyuIGXoP3hl" version="21.0.6" type="device"><diagram id="SVgELWPNXIlR7V7eDs_m" name="Page-1">7Vtbc5s4FP41frQHgY3tx9hO2u5kZ9Km7c70xSODDGxlxAoRO/31K4FkgwS2k+DGTZxmpuggjsS5fOci0nGmq80HCpPwb+Ij3LEtf9NxZh3bBn2nz/8TlMeCMhyOCkJAI19O2hHuo19IEi1JzSIfpZWJjBDMoqRK9EgcI49VaJBSsq5OWxJcXTWBATII9x7EJvWfyGehpAJ3vLvxEUVBKJce2cPixgqqyfJN0hD6ZF0iOdcdZ0oJYcXVajNFWAhPyaV47qbh7nZjFMXsmAf8b7fdj9cAuzc/wOTz92/W3Y+vXafg8gBxJl94NrXlftmjEkJCopjlghxM+C9fZ2p1BvzOVIx69kAj6ONhlQDMkeBRJejjYZUAdPZAWx/oGywRjFGFvaWtb5U2yH+dCckYjmI03ZqcxYkBhX7EVTElmFBOi0nMpTcJ2QrzEeCX6zBi6D6BnpDqmrsLpy1JzKTRA1uNpeAFV27WDPK1qOSRawLR6wdUKKSYgzFM0mixfYoiL6Np9IC+oLRgLqjcABNxvdoEwld7cJ32ewElWZJv/xNfq/bunF/OPUwyXzBhlPxE6iU7tsP/3QiDmywjjLWXf0CURdyXrnAUCN6MiKWgHGG0ZIIjl0gUB7f5aOZYUgp1S/gwDZEvX8e0f+kSYlW0KZGkP3xAZIUYfeRT5N2+JX3zUY2L4Xrn6Y5y37Dk5I56Dkp0Cbasdw7IL6QPPsEf3b7hkHeQizyODKeskbehnMGVOx25ZcmBRrXopqcpYcuqzoxLVm6qZS/wHK0roOvKNnQF6nQFHPtEunKAoavrr9OZCEY4S7mX8itgqC39iZgXShmX5Fax7VzGqQYVJX13hAmKnzqlL/MfBRYl2O5ZVg7EPdfRAFpisEYd11ILLNdmjhsYD/On+f0aJkCj2SNtbg62ylhv4QLhO5JGLMpxakEYI6sSnHhIIGOjeevo9zNbIBojLuZelCPfJFEQyBXkn8yM7aoZ25aJOaMaM+6PXm7F6ch2HA96nxfjq1/hp+zT9C+vaxpxx3axAGY/euCXAcuduiAtqE7ha9bMy0klq3f/y0Sak2NKtwhJV3yCm2yKh+TtFy1XJYmVjtkCcA7s4WhG/bYYDdpidEi8RzMatsVo1BajcUuMuIu3xAi0xchui1Fblm03WfZdySsLZoazvmtyBZb0NCCP2qqktKu5gB6rlpisvRBS1vMhgwuYooY8rCEaHRvImqNWvxq1RMWlJ8rD3sAMW4MWEuXasGXvD1tHQEhbftbI6O6DeIk4ZTDmOnqatZzZq7TGKKHRCgr175VGbdx/hpNxL2BVzzqYLK4i3xeP8xqavw7c1dTVZFpkpjBjRJbXudNW8nBZkdUUaaWS3+6f0mfBoOKzwDEzzXFdpnmq2tYsbWeiBig0yYuduNaycpjrylpI2FZCUZON8uJnXLWOgm2DeVzaWu+6reURmhAKGZqLqDrfQkJzF+V0LS6zUdNai+vMOlxgbHa46gLlLgZQBHk5blGUYC7p/Q2VWhtuaic22PZxLc5yo6WitIMNi/0mszXDkosdcieYJsXbLqMN8pt8gkc0klEPFR4hAlydb/iLuVLFXEh+ruR+dGuv/1Qb3UYmaaSu2dpza2zUPZWJqu1cGnvvt7H3dCN+xcZe/VGCWSK9zaOEp6vq8LHPbz5KAGYXdjZ1LgnrJWF9F+ewQ/fsstThJUt921kqeLKVnl2a6hpWhvwAKZETykISkBji6x11woEk9rdWsJtzS4R+cz3+ixh7lIoUPa5nBWUl3kKZ+95CSlFsfa8WKMKQcSytYEydUOWjdwKid9rrAg1kOMxUeTBIA8TkY5putvt4gbrMfOxSVbyzquIZoHN2ZYX7TsqKZ+jq7D5RskfnESLUl7s5wu6T+fj3BIOBFsoNDRRh63SxwEwvVUYZOnlEgGmxE3XKwSKGUQsHc219pnDwqLDxQCdntJbmL1jFhK4grmV2/xh7IccWkqV84pcix8sRvemI55lH9ULqDadEwhbzaNI52UnikYikvLnmrFB+/i5X6ZS/MK9Dqi4P7mPHqfiA+hLsua6lppDlMkUn8Rp7/Ieh2fA3odn4ldFM2WUDmh0GE/MrrBNCYFuffLXH6ALKLZD/DAhXp58vhnCO4CN3WEVw59wR3DF72q+J4IebE9aRUK/+FqA9qH9Za6j/h8n52JCqFHIuch68VTm33pVrkDMf7v4EsoCZ3R+SOtf/Aw==</diagram></mxfile>

docs/citus.rst

@@ -0,0 +1,355 @@
+.. _citus:
+
+Citus support
+=============
+
+Patroni makes it extremely simple to deploy `Multi-Node Citus`__ clusters.
+
+__ https://docs.citusdata.com/en/stable/installation/multi_node.html
+
+TL;DR
+-----
+
+There are only a few simple rules you need to follow:
+
+1. `Citus <https://github.com/citusdata/citus>`__ database extension to
+   PostgreSQL must be available on all nodes.  Absolute minimum supported Citus
+   version is 10.0, but, to take all benefits from transparent switchovers and
+   restarts of workers we recommend using at least Citus 11.2.
+2. Cluster name (``scope``) must be the same for all Citus nodes!
+3. Superuser credentials must be the same on coordinator and all worker
+   nodes, and ``pg_hba.conf`` should allow superuser access between all nodes.
+4. :ref:`REST API <restapi_settings>` access should be allowed from worker
+   nodes to the coordinator. E.g., credentials should be the same and if
+   configured, client certificates from worker nodes must be accepted by the
+   coordinator.
+5. Add the following section to the ``patroni.yaml``:
+
+.. code:: YAML
+
+        citus:
+          group: X  # 0 for coordinator and 1, 2, 3, etc for workers
+          database: citus  # must be the same on all nodes
+
+
+After that you just need to start Patroni and it will handle the rest:
+
+1. ``citus`` extension will be automatically added to ``shared_preload_libraries``.
+2. If ``max_prepared_transactions`` isn't explicitly set in the global
+   :ref:`dynamic configuration <dynamic_configuration>` Patroni will
+   automatically set it to ``2*max_connections``.
+3. The ``citus.database`` will be automatically created followed by ``CREATE EXTENSION citus``.
+4. Current superuser :ref:`credentials <postgresql_settings>` will be added to the ``pg_dist_authinfo``
+   table to allow cross-node communication. Don't forget to update them if
+   later you decide to change superuser username/password/sslcert/sslkey!
+5. The coordinator primary node will automatically discover worker primary
+   nodes and add them to the ``pg_dist_node`` table using the
+   ``citus_add_node()`` function.
+6. Patroni will also maintain ``pg_dist_node`` in case failover/switchover
+   on the coordinator or worker clusters occurs.
+
+patronictl
+----------
+
+Coordinator and worker clusters are physically different PostgreSQL/Patroni
+clusters that are just logically groupped together using the
+`Citus <https://github.com/citusdata/citus>`__ database extension to
+PostgreSQL. Therefore in most cases it is not possible to manage them as a
+single entity.
+
+It results in two major differences in ``patronictl`` behaviour when
+``patroni.yaml`` has the ``citus`` section comparing with the usual:
+
+1. The ``list`` and the ``topology`` by default output all members of the Citus
+   formation (coordinators and workers). The new column ``Group`` indicates
+   which Citus group they belong to.
+2. For all ``patronictl`` commands the new option is introduced, named
+   ``--group``. For some commands the default value for the group might be
+   taken from the ``patroni.yaml``. For example, ``patronictl pause`` will
+   enable the maintenance mode by default for the ``group`` that is set in the
+   ``citus`` section, but for example for ``patronictl  switchover`` or
+   ``patronictl remove`` the group must be explicitly specified.
+
+An example of ``patronictl list`` output for the Citus cluster::
+
+    postgres@coord1:~$ patronictl list demo
+    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+    |     0 | coord1  | 172.27.0.10 | Replica      | running |  1 |         0 |
+    |     0 | coord2  | 172.27.0.6  | Sync Standby | running |  1 |         0 |
+    |     0 | coord3  | 172.27.0.4  | Leader       | running |  1 |           |
+    |     1 | work1-1 | 172.27.0.8  | Sync Standby | running |  1 |         0 |
+    |     1 | work1-2 | 172.27.0.2  | Leader       | running |  1 |           |
+    |     2 | work2-1 | 172.27.0.5  | Sync Standby | running |  1 |         0 |
+    |     2 | work2-2 | 172.27.0.7  | Leader       | running |  1 |           |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+
+If we add the ``--group`` option, the output will change to::
+
+    postgres@coord1:~$ patronictl list demo --group 0
+    + Citus cluster: demo (group: 0, 7179854923829112860) -----------+
+    | Member | Host        | Role         | State   | TL | Lag in MB |
+    +--------+-------------+--------------+---------+----+-----------+
+    | coord1 | 172.27.0.10 | Replica      | running |  1 |         0 |
+    | coord2 | 172.27.0.6  | Sync Standby | running |  1 |         0 |
+    | coord3 | 172.27.0.4  | Leader       | running |  1 |           |
+    +--------+-------------+--------------+---------+----+-----------+
+
+    postgres@coord1:~$ patronictl list demo --group 1
+    + Citus cluster: demo (group: 1, 7179854923881963547) -----------+
+    | Member  | Host       | Role         | State   | TL | Lag in MB |
+    +---------+------------+--------------+---------+----+-----------+
+    | work1-1 | 172.27.0.8 | Sync Standby | running |  1 |         0 |
+    | work1-2 | 172.27.0.2 | Leader       | running |  1 |           |
+    +---------+------------+--------------+---------+----+-----------+
+
+Citus worker switchover
+-----------------------
+
+When a switchover is orchestrated for a Citus worker node, Citus offers the
+opportunity to make the switchover close to transparent for an application.
+Because the application connects to the coordinator, which in turn connects to
+the worker nodes, then it is possible with Citus to `pause` the SQL traffic on
+the coordinator for the shards hosted on a worker node. The switchover then
+happens while the traffic is kept on the coordinator, and resumes as soon as a
+new primary worker node is ready to accept read-write queries.
+
+An example of ``patronictl switchover`` on the worker cluster::
+
+    postgres@coord1:~$ patronictl switchover demo
+    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+    |     0 | coord1  | 172.27.0.10 | Replica      | running |  1 |         0 |
+    |     0 | coord2  | 172.27.0.6  | Sync Standby | running |  1 |         0 |
+    |     0 | coord3  | 172.27.0.4  | Leader       | running |  1 |           |
+    |     1 | work1-1 | 172.27.0.8  | Leader       | running |  1 |           |
+    |     1 | work1-2 | 172.27.0.2  | Sync Standby | running |  1 |         0 |
+    |     2 | work2-1 | 172.27.0.5  | Sync Standby | running |  1 |         0 |
+    |     2 | work2-2 | 172.27.0.7  | Leader       | running |  1 |           |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+    Citus group: 2
+    Primary [work2-2]:
+    Candidate ['work2-1'] []:
+    When should the switchover take place (e.g. 2022-12-22T08:02 )  [now]:
+    Current cluster topology
+    + Citus cluster: demo (group: 2, 7179854924063375386) -----------+
+    | Member  | Host       | Role         | State   | TL | Lag in MB |
+    +---------+------------+--------------+---------+----+-----------+
+    | work2-1 | 172.27.0.5 | Sync Standby | running |  1 |         0 |
+    | work2-2 | 172.27.0.7 | Leader       | running |  1 |           |
+    +---------+------------+--------------+---------+----+-----------+
+    Are you sure you want to switchover cluster demo, demoting current primary work2-2? [y/N]: y
+    2022-12-22 07:02:40.33003 Successfully switched over to "work2-1"
+    + Citus cluster: demo (group: 2, 7179854924063375386) ------+
+    | Member  | Host       | Role    | State   | TL | Lag in MB |
+    +---------+------------+---------+---------+----+-----------+
+    | work2-1 | 172.27.0.5 | Leader  | running |  1 |           |
+    | work2-2 | 172.27.0.7 | Replica | stopped |    |   unknown |
+    +---------+------------+---------+---------+----+-----------+
+
+    postgres@coord1:~$ patronictl list demo
+    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+    |     0 | coord1  | 172.27.0.10 | Replica      | running |  1 |         0 |
+    |     0 | coord2  | 172.27.0.6  | Sync Standby | running |  1 |         0 |
+    |     0 | coord3  | 172.27.0.4  | Leader       | running |  1 |           |
+    |     1 | work1-1 | 172.27.0.8  | Leader       | running |  1 |           |
+    |     1 | work1-2 | 172.27.0.2  | Sync Standby | running |  1 |         0 |
+    |     2 | work2-1 | 172.27.0.5  | Leader       | running |  2 |           |
+    |     2 | work2-2 | 172.27.0.7  | Sync Standby | running |  2 |         0 |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+
+And this is how it looks on the coordinator side::
+
+    # The worker primary notifies the coordinator that it is going to execute "pg_ctl stop".
+    2022-12-22 07:02:38,636 DEBUG: query("BEGIN")
+    2022-12-22 07:02:38,636 DEBUG: query("SELECT pg_catalog.citus_update_node(3, '172.27.0.7-demoted', 5432, true, 10000)")
+    # From this moment all application traffic on the coordinator to the worker group 2 is paused.
+
+    # The future worker primary notifies the coordinator that it acquired the leader lock in DCS and about to run "pg_ctl promote".
+    2022-12-22 07:02:40,085 DEBUG: query("SELECT pg_catalog.citus_update_node(3, '172.27.0.5', 5432)")
+
+    # The new worker primary just finished promote and notifies coordinator that it is ready to accept read-write traffic.
+    2022-12-22 07:02:41,485 DEBUG: query("COMMIT")
+    # From this moment the application traffic on the coordinator to the worker group 2 is unblocked.
+
+Peek into DCS
+-------------
+
+The Citus cluster (coordinator and workers) are stored in DCS as a fleet of
+Patroni clusters logically grouped together::
+
+    /service/batman/              # scope=batman
+    /service/batman/0/            # citus.group=0, coordinator
+    /service/batman/0/initialize
+    /service/batman/0/leader
+    /service/batman/0/members/
+    /service/batman/0/members/m1
+    /service/batman/0/members/m2
+    /service/batman/1/            # citus.group=1, worker
+    /service/batman/1/initialize
+    /service/batman/1/leader
+    /service/batman/1/members/
+    /service/batman/1/members/m3
+    /service/batman/1/members/m4
+    ...
+
+Such an approach was chosen because for most DCS it becomes possible to fetch
+the entire Citus cluster with a single recursive read request. Only Citus
+coordinator nodes are reading the whole tree, because they have to discover
+worker nodes. Worker nodes are reading only the subtree for their own group and
+in some cases they could read the subtree of the coordinator group.
+
+Citus on Kubernetes
+-------------------
+
+Since Kubernetes doesn't support hierarchical structures we had to include the
+citus group to all K8s objects Patroni creates::
+
+    batman-0-leader  # the leader config map for the coordinator
+    batman-0-config  # the config map holding initialize, config, and history "keys"
+    ...
+    batman-1-leader  # the leader config map for worker group 1
+    batman-1-config
+    ...
+
+I.e., the naming pattern is: ``${scope}-${citus.group}-${type}``.
+
+All Kubernetes objects are discovered by Patroni using the `label selector`__,
+therefore all Pods with Patroni&Citus and Endpoints/ConfigMaps must have
+similar labels, and Patroni must be configured to use them using Kubernetes
+:ref:`settings <kubernetes_settings>` or :ref:`environment variables
+<kubernetes_environment>`.
+
+__ https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors
+
+A couple of examples of Patroni configuration using Pods environment variables:
+
+1. for the coordinator cluster
+
+.. code:: YAML
+
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          labels:
+            application: patroni
+            citus-group: "0"
+            citus-type: coordinator
+            cluster-name: citusdemo
+          name: citusdemo-0-0
+          namespace: default
+        spec:
+          containers:
+          - env:
+            - name: PATRONI_SCOPE
+              value: citusdemo
+            - name: PATRONI_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
+            - name: PATRONI_KUBERNETES_POD_IP
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: status.podIP
+            - name: PATRONI_KUBERNETES_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: PATRONI_KUBERNETES_LABELS
+              value: '{application: patroni}'
+            - name: PATRONI_CITUS_DATABASE
+              value: citus
+            - name: PATRONI_CITUS_GROUP
+              value: "0"
+
+2. for the worker cluster from the group 2
+
+.. code:: YAML
+
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          labels:
+            application: patroni
+            citus-group: "2"
+            citus-type: worker
+            cluster-name: citusdemo
+          name: citusdemo-2-0
+          namespace: default
+        spec:
+          containers:
+          - env:
+            - name: PATRONI_SCOPE
+              value: citusdemo
+            - name: PATRONI_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
+            - name: PATRONI_KUBERNETES_POD_IP
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: status.podIP
+            - name: PATRONI_KUBERNETES_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: PATRONI_KUBERNETES_LABELS
+              value: '{application: patroni}'
+            - name: PATRONI_CITUS_DATABASE
+              value: citus
+            - name: PATRONI_CITUS_GROUP
+              value: "2"
+
+As you may noticed, both examples have ``citus-group`` label set. This label
+allows Patroni to identify object as belonging to a certain Citus group. In
+addition to that, there is also ``PATRONI_CITUS_GROUP`` environment variable,
+which has the same value as the ``citus-group`` label. When Patroni creates
+new Kubernetes objects ConfigMaps or Endpoints, it automatically puts the
+``citus-group: ${env.PATRONI_CITUS_GROUP}`` label on them:
+
+.. code:: YAML
+
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: citusdemo-0-leader  # Is generated as ${env.PATRONI_SCOPE}-${env.PATRONI_CITUS_GROUP}-leader
+          labels:
+            application: patroni    # Is set from the ${env.PATRONI_KUBERNETES_LABELS}
+            cluster-name: citusdemo # Is automatically set from the ${env.PATRONI_SCOPE}
+            citus-group: '0'        # Is automatically set from the ${env.PATRONI_CITUS_GROUP}
+
+You can find a complete example of Patroni deployment on Kubernetes with Citus
+support in the `kubernetes`__ folder of the Patroni repository.
+
+__ https://github.com/zalando/patroni/tree/master/kubernetes
+
+There are two important files for you:
+
+1. Dockerfile.citus
+2. citus_k8s.yaml
+
+Citus upgrades and PostgreSQL major upgrades
+--------------------------------------------
+
+First, please read about upgrading Citus version in the `documentation`__.
+There is one minor change in the process. When executing upgrade, you have to
+use ``patronictl restart`` instead of ``systemctl restart`` to restart
+PostgreSQL.
+
+__ https://docs.citusdata.com/en/latest/admin_guide/upgrading_citus.html
+
+The PostgreSQL major upgrade with Citus is a bit more complex. You will have to
+combine techniques used in the Citus documentation about major upgrades and
+Patroni documentation about :ref:`PostgreSQL major upgrade<major_upgrade>`.
+Please keep in mind that Citus cluster consists of many Patroni clusters
+(coordinator and workers) and they all have to be upgraded independently.

docs/dcs_failsafe_mode.rst

@@ -0,0 +1,63 @@
+.. _dcs_failsafe_mode:
+
+DCS Failsafe Mode
+=================
+
+The problem
+-----------
+
+Patroni is heavily relying on Distributed Configuration Store (DCS) to solve the task of leader elections and detect network partitioning. That is, the node is allowed to run Postgres as the primary only if it can update the leader lock in DCS. In case the update of the leader lock fails, Postgres is immediately demoted and started as read-only. Depending on which DCS is used, the chances of hitting the "problem" differ. For example, with Etcd which is only used for Patroni, chances are close to zero, while with K8s API (backed by Etcd) it could be observed more frequently.
+
+
+Reasons for the current implementation
+---------------------------------------
+
+The leader lock update failure could be caused by two main reasons:
+
+1. Network partitioning
+2. DCS being down
+
+In general, it is impossible to distinguish between these two from a single node, and therefore Patroni assumes the worst case - network partitioning. In the case of a partitioned network, other nodes of the Patroni cluster may successfully grab the leader lock and promote Postgres to primary. In order to avoid a split-brain, the old primary is demoted before the leader lock expires.
+
+
+DCS Failsafe Mode
+-----------------
+
+We introduce a new special option, the ``failsafe_mode``. It could be enabled only via global configuration stored in the DCS ``/config`` key. If the failsafe mode is enabled and the leader lock update in DCS failed due to reasons different from the version/value/index mismatch, Postgres may continue to run as a primary if it can access all known members of the cluster via Patroni REST API.
+
+
+Low-level implementation details
+--------------------------------
+
+- We introduce a new, permanent key in DCS, named ``/failsafe``.
+- The ``/failsafe`` key contains all known members of the given Patroni cluster at a given time.
+- The current leader maintains the ``/failsafe`` key.
+- The member is allowed to participate in the leader race and become the new leader only if it is present in the ``/failsafe`` key.
+- If the cluster consists of a single node the ``/failsafe`` key will contain a single member.
+- In the case of DCS "outage" the existing primary connects to all members presented in the ``/failsafe`` key via the ``POST /failsafe`` REST API and may continue to run as the primary if all replicas acknowledge it.
+- If one of the members doesn't respond, the primary is demoted.
+- Replicas are using incoming ``POST /failsafe`` REST API requests as an indicator that the primary is still alive. This information is cached for ``ttl`` seconds.
+
+
+F.A.Q.
+------
+
+- Why MUST the current primary see ALL other members? Can’t we rely on quorum here?
+
+  This is a great question! The problem is that the view on the quorum might be different from the perspective of DCS and Patroni. While DCS nodes must be evenly distributed across availability zones, there is no such rule for Patroni, and more importantly, there is no mechanism for introducing and enforcing such a rule. If the majority of Patroni nodes ends up in the losing part of the partitioned network (including primary) while minority nodes are in the winning part, the primary must be demoted. Only checking ALL other members allows detecting such a situation.
+
+- What if node/pod gets terminated while DCS is down?
+
+  If DCS isn’t accessible, the check “are ALL other cluster members accessible?” is executed every cycle of the heartbeat loop (every ``loop_wait`` seconds). If pod/node is terminated, the check will fail and Postgres will be demoted to a read-only and will not recover until DCS is restored.
+
+- What if all members of the Patroni cluster are lost while DCS is down?
+
+  Patroni could be configured to create the new replica from the backup even when the cluster doesn't have a leader. But, if the new member isn't present in the ``/failsafe`` key, it will not be able to grab the leader lock and promote.
+  
+- What will happen if the primary lost access to DCS while replicas didn't?
+
+  The primary will execute the failsafe code and contact all known replicas. These replicas will use this information as an indicator that the primary is alive and will not start the leader race even if the leader lock in DCS has expired.
+
+- How to enable the Failsafe Mode?
+
+  Before enabling the ``failsafe_mode`` please make sure that Patroni version on all members is up-to-date. After that, you can use either the ``PATCH /config`` :ref:`REST API <rest_api>` or ``patronictl edit-config -s failsafe_mode=true``

docs/dynamic_configuration.rst

@@ -22,7 +22,7 @@ Patroni configuration is stored in the DCS (Distributed Configuration Store). Th
 
 The local configuration can be either a single YAML file or a directory. When it is a directory, all YAML files in that directory are loaded one by one in sorted order. In case a key is defined in multiple files, the occurrence in the last file takes precedence.
 
-Some of the PostgreSQL parameters must hold the same values on the master and the replicas. For those, values set either in the local patroni configuration files or via the environment variables take no effect. To alter or set their values one must change the shared configuration in the DCS. Below is the actual list of such parameters together with the default values:
+Some of the PostgreSQL parameters must hold the same values on the primary and the replicas. For those, values set either in the local patroni configuration files or via the environment variables take no effect. To alter or set their values one must change the shared configuration in the DCS. Below is the actual list of such parameters together with the default values:
 
 - max_connections: 100
 - max_locks_per_transaction: 64
@@ -32,7 +32,7 @@ Some of the PostgreSQL parameters must hold the same values on the master and th
 - wal_log_hints: on
 - track_commit_timestamp: off
 
-For the parameters below, PostgreSQL does not require equal values among the master and all the replicas. However, considering the possibility of a replica to become the master at any time, it doesn't really make sense to set them differently; therefore, Patroni restricts setting their values to the Dynamic configuration
+For the parameters below, PostgreSQL does not require equal values among the primary and all the replicas. However, considering the possibility of a replica to become the primary at any time, it doesn't really make sense to set them differently; therefore, Patroni restricts setting their values to the Dynamic configuration
 
 - max_wal_senders: 5
 - max_replication_slots: 5
@@ -86,4 +86,4 @@ Also, the following Patroni configuration options can be changed only dynamicall
 Upon changing these options, Patroni will read the relevant section of the configuration stored in DCS and change its
 run-time values.
 
-Patroni nodes are dumping the state of the DCS options to disk upon for every change of the configuration into the file ``patroni.dynamic.json`` located in the Postgres data directory. Only the master is allowed to restore these options from the on-disk dump if these are completely absent from the DCS or if they are invalid.
+Patroni nodes are dumping the state of the DCS options to disk upon for every change of the configuration into the file ``patroni.dynamic.json`` located in the Postgres data directory. Only the leader is allowed to restore these options from the on-disk dump if these are completely absent from the DCS or if they are invalid.

docs/existing_data.rst

@@ -23,17 +23,19 @@ A Patroni cluster can be started with a data directory from a single-node Postgr
 3. Start Patroni (e.g. ``patroni /etc/patroni/patroni.yml``). It automatically detects that PostgreSQL daemon is already running but its configuration might be out-of-date.
 4. Ask Patroni to restart the node with ``patronictl restart cluster-name node-name``. This step is only required if PostgreSQL configuration is out-of-date.
 
+.. _major_upgrade:
+
 Major Upgrade of PostgreSQL Version
 ===================================
 
 The only possible way to do a major upgrade currently is:
 
 1. Stop Patroni
-2. Upgrade PostgreSQL binaries and perform `pg_upgrade <https://www.postgresql.org/docs/current/pgupgrade.html>`_ on the master node
+2. Upgrade PostgreSQL binaries and perform `pg_upgrade <https://www.postgresql.org/docs/current/pgupgrade.html>`_ on the primary node
 3. Update patroni.yml
 4. Remove the initialize key from DCS or wipe complete cluster state from DCS. The second one could be achieved by running ``patronictl remove <cluster-name>``. It is necessary because pg_upgrade runs initdb which actually creates a new database with a new PostgreSQL system identifier.
 5. If you wiped the cluster state in the previous step, you may wish to copy patroni.dynamic.json from old data dir to the new one.  It will help you to retain some PostgreSQL parameters you had set before.
-6. Start Patroni on the master node.
+6. Start Patroni on the primary node.
 7. Upgrade PostgreSQL binaries, update patroni.yml and wipe the data_dir on standby nodes.
 8. Start Patroni on the standby nodes and wait for the replication to complete.
 

docs/ha_multi_dc.rst

@@ -0,0 +1,54 @@
+.. _ha_multi_dc:
+
+=================
+HA multi datacenter
+=================
+
+The high availability of a PostgreSQL cluster deployed in multiple data centers is based on replication, which can be synchronous or asynchronous (`replication_modes <replication_modes.rst>`_).
+
+In both cases, it is important to be clear about the following concepts:
+
+- Postgres can run as primary or standby leader only when it owns the leading key and can update the leading key.
+- You should run the odd number of etcd, ZooKeeper or Consul nodes: 3 or 5!
+
+Synchronous Replication
+----------------------------
+
+To have a multi DC cluster that can automatically tolerate a zone drop, a minimum of 3 is required.
+
+The architecture diagram would be the following:
+
+.. image:: _static/multi-dc-synchronous-replication.png
+
+We must deploy a cluster of etcd, ZooKeeper or Consul through the different DC, with a minimum of 3 nodes, one in each zone.
+
+Regarding postgres, we must deploy at least 2 nodes, in different DC. Then you have to set ``synchronous_mode: true`` in the global configuration (``patronictl edit-config``).
+
+This enables sync replication and the primary node will choose one of the nodes as synchronous.
+
+Asynchronous Replication
+----------------------------------
+
+With only two data centers it would be better to have two independent etcd clusters and run Patroni :ref:`standby cluster <standby_cluster>` in the second data center. If the first site is down, you can MANUALLY promote the ``standby_cluster``.
+
+The architecture diagram would be the following:
+
+.. image:: _static/multi-dc-asynchronous-replication.png
+
+Automatic promotion is not possible, because DC2 will never able to figure out the state of DC1.
+
+You should not use ``pg_ctl promote`` in this scenario, you need "manually promote" the healthy cluster with ``patronictl edit-config`` and remove ``standby_cluster`` section from there.
+
+.. warning::
+    If the source cluster is still up and running and you promote the standby cluster you create a split-brain.
+
+In case you want to return to the "initial" state, there are only two ways of resolving it:
+
+- Add the standby_cluster section back and it will trigger pg_rewind, but there are chances that pg_rewind will fail.
+- Rebuild the standby cluster from scratch.
+
+Before promoting standby cluster one have to manually ensure that the source cluster is down (STONITH). When DC1 recovers, the cluster has to be converted to a standby cluster.
+
+Before doing that you may manually examine the database and extract all changes that happened between the time when network between DC1 and DC2 has stopped working and the time when you manually stopped the cluster in DC1.
+
+Once extracted, you may also manually apply these changes to the cluster in DC2.

docs/index.rst

@@ -10,7 +10,9 @@ Patroni is a template for you to create your own customized, high-availability s
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
-Currently supported PostgreSQL versions: 9.3 to 14.
+Currently supported PostgreSQL versions: 9.3 to 15.
+
+**Note to Citus users**: Starting from 3.0 Patroni nicely integrates with the `Citus <https://github.com/citusdata/citus>`__ database extension to Postgres. Please check the :ref:`Citus support page <citus>` in the Patroni documentation for more info about how to use Patroni high availability together with a Citus distributed cluster.
 
 **Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the :ref:`Kubernetes <kubernetes>` chapter of the Patroni documentation.
 
@@ -20,7 +22,9 @@ Currently supported PostgreSQL versions: 9.3 to 14.
    :caption: Contents:
 
    README
+   citus
    dynamic_configuration
+   dcs_failsafe_mode
    rest_api
    existing_data
    ENVIRONMENT
@@ -28,6 +32,7 @@ Currently supported PostgreSQL versions: 9.3 to 14.
    security
    replica_bootstrap
    replication_modes
+   ha_multi_dc
    pause
    kubernetes
    watchdog

docs/kubernetes.rst

@@ -23,10 +23,7 @@ Use ConfigMaps
 In this mode, Patroni will create ConfigMaps instead of Endpoints and store keys inside meta-data of those ConfigMaps.
 Changing the leader takes at least two updates, one to the leader ConfigMap and another to the respective Endpoint.
 
-There are two ways to direct the traffic to the Postgres master:
-
-- use the `callback script <https://github.com/zalando/patroni/blob/master/kubernetes/callback.py>`_ provided by Patroni
-- configure the Kubernetes Postgres service to use the label selector with the `role_label` (configured in patroni configuration).
+To direct the traffic to the Postgres leader you need to configure the Kubernetes Postgres service to use the label selector with the `role_label` (configured in patroni configuration).
 
 Note that in some cases, for instance, when running on OpenShift, there is no alternative to using ConfigMaps.
 
@@ -39,7 +36,7 @@ Examples
 --------
 
 - The `kubernetes <https://github.com/zalando/patroni/tree/master/kubernetes>`__ folder of the Patroni repository contains
-  examples of the Docker image, the Kubernetes manifest and the callback script in order to test Patroni Kubernetes setup.
+  examples of the Docker image, and the Kubernetes manifest to test Patroni Kubernetes setup.
   Note that in the current state it will not be able to use PersistentVolumes because of permission issues.
 
 - You can find the full-featured Docker image that can use Persistent Volumes in the

docs/pause.rst

@@ -6,7 +6,7 @@ Pause/Resume mode for the cluster
 The goal
 --------
 
-Under certain circumstances Patroni needs to temporary step down from managing the cluster, while still retaining the cluster state in DCS. Possible use cases are uncommon activities on the cluster, such as major version upgrades or corruption recovery. During those activities nodes are often started and stopped for the reason unknown to Patroni, some nodes can be even temporary promoted, violating the assumption of running only one master. Therefore, Patroni needs to be able to "detach" from the running cluster, implementing an equivalent of the maintenance mode in Pacemaker.
+Under certain circumstances Patroni needs to temporarily step down from managing the cluster, while still retaining the cluster state in DCS. Possible use cases are uncommon activities on the cluster, such as major version upgrades or corruption recovery. During those activities nodes are often started and stopped for reasons unknown to Patroni, some nodes can be even temporarily promoted, violating the assumption of running only one primary. Therefore, Patroni needs to be able to "detach" from the running cluster, implementing an equivalent of the maintenance mode in Pacemaker.
 
 
 
@@ -17,16 +17,18 @@ When Patroni runs in a paused mode, it does not change the state of PostgreSQL,
 
 - For each node, the member key in DCS is updated with the current information about the cluster. This causes Patroni to run read-only queries on a member node if the member is running.
 
-- For the Postgres master with the leader lock Patroni updates the lock. If the node with the leader lock stops being the master (i.e. is demoted manually), Patroni will release the lock instead of promoting the node back.
+- For the Postgres primary with the leader lock Patroni updates the lock. If the node with the leader lock stops being the primary (i.e. is demoted manually), Patroni will release the lock instead of promoting the node back.
 
-- Manual unscheduled restart, reinitialize and manual failover are allowed. Manual failover is only allowed if the node to failover to is specified. In the paused mode, manual failover does not require a running master node.
+- Manual unscheduled restart, reinitialize and manual failover are allowed. Manual failover is only allowed if the node to failover to is specified. In the paused mode, manual failover does not require a running primary node.
 
-- If 'parallel' masters are detected by Patroni, it emits a warning, but does not demote the masters without the leader lock.
+- If 'parallel' primaries are detected by Patroni, it emits a warning, but does not demote the primary without the leader lock.
 
-- If there is no leader lock in the cluster, the running master acquires the lock. If there is more than one master node, then the first master to acquire the lock wins. If there are no masters altogether, Patroni does not try to promote any replicas. There is an exception in this rule: if there is no leader lock because the old master has demoted itself due to the manual promotion, then only the candidate node mentioned in the promotion request may take the leader lock. When the new leader lock is granted (i.e. after promoting a replica manually), Patroni makes sure the replicas that were streaming from the previous leader will switch to the new one.
+- If there is no leader lock in the cluster, the running primary acquires the lock. If there is more than one primary node, then the first primary to acquire the lock wins. If there are no primary altogether, Patroni does not try to promote any replicas. There is an exception in this rule: if there is no leader lock because the old primary has demoted itself due to the manual promotion, then only the candidate node mentioned in the promotion request may take the leader lock. When the new leader lock is granted (i.e. after promoting a replica manually), Patroni makes sure the replicas that were streaming from the previous leader will switch to the new one.
 
 - When Postgres is stopped, Patroni does not try to start it. When Patroni is stopped, it does not try to stop the Postgres instance it is managing.
 
+- Patroni will not try to remove replication slots that don't represent the other cluster member or are not listed in the configuration of the permanent slots.
+
 User guide
 ----------
 

docs/releases.rst

@@ -3,6 +3,285 @@
 Release notes
 =============
 
+Version 3.0.2
+-------------
+
+.. warning::
+    Version 3.0.2 dropped support of Python older than 3.6.
+
+
+**New features**
+
+- Added sync standby replica status to ``/metrics`` endpoint (Thomas von Dein, Alexander Kukushkin)
+
+  Before were only reporting ``primary``/``standby_leader``/``replica``.
+
+- User-friendly handling of ``PAGER`` in ``patronictl`` (Israel Barth Rubio)
+
+  It makes pager configurable via ``PAGER`` environment variable, which overrides default ``less`` and ``more``.
+
+- Make K8s retriable HTTP status code configurable (Alexander)
+
+  On some managed platforms it is possible to get status code ``401 Unauthorized``, which sometimes gets resolved after a few retries.
+
+
+**Improvements**
+
+- Set ``hot_standby`` to ``off`` during custom bootstrap only if ``recovery_target_action`` is set to ``promote`` (Alexander)
+
+  It was necessary to make ``recovery_target_action=pause`` work correctly.
+
+- Don't allow ``on_reload`` callback to kill other callbacks (Alexander)
+
+  ``on_start``/``on_stop``/``on_role_change`` are usually used to add/remove Virtual IP and ``on_reload`` should not interfere with them.
+
+- Switched to ``IMDSFetcher`` in aws callback example script (Polina Bungina)
+
+  The ``IMDSv2`` requires a token to work with and the ``IMDSFetcher`` handles it transparently.
+
+
+**Bugfixes**
+
+- Fixed ``patronictl switchover`` on Citus cluster running on Kubernetes (Lukáš Lalinský)
+
+  It didn't work for namespaces different from ``default``.
+
+- Don't write to ``PGDATA`` if major version is not known (Alexander)
+
+  If right after the start ``PGDATA`` was empty (maybe wasn't yet mounted), Patroni was making a false assumption about PostgreSQL version and falsely creating ``recovery.conf`` file even if the actual major version is v10+.
+
+- Fixed bug with Citus metadata after coordinator failover (Alexander)
+
+  The ``citus_set_coordinator_host()`` call doesn't cause metadata sync and the change was invisible on worker nodes. The issue is solved by switching to ``citus_update_node()``.
+
+- Use etcd hosts listed in the config file as a fallback when all etcd nodes "failed" (Alexander)
+
+  The etcd cluster may change topology over time and Patroni tries to follow it. If at some point all nodes became unreachable Patroni will use a combination of nodes from the config plus the last known topology when trying to reconnect.
+
+
+Version 3.0.1
+-------------
+
+**Bugfixes**
+
+- Pass proper role name to an ``on_role_change`` callback script'. (Alexander Kukushkin, Polina Bungina)
+
+  Patroni used to erroneously pass ``promoted`` role to an ``on_role_change`` callback script on promotion. The passed role name changed back to ``master``. This regression was introduced in 3.0.0.
+
+
+Version 3.0.0
+-------------
+
+This version adds integration with `Citus <https://www.citusdata.com>`__ and makes it possible to survive temporary DCS outages without demoting primary.
+
+.. warning::
+   - Version 3.0.0 is the last release supporting Python 2.7. Upcoming release will drop support of Python versions older than 3.7.
+
+   - The RAFT support is deprecated. We will do our best to maintain it, but take neither guarantee nor responsibility for possible issues.
+
+   - This version is the first step in getting rid of the "master", in favor of "primary". Upgrading to the next major release will work reliably only if you run at least 3.0.0.
+
+
+**New features**
+
+- DCS failsafe mode (Alexander Kukushkin, Polina Bungina)
+
+  If the feature is enabled it will allow Patroni cluster to survive temporary DCS outages. You can find more details in the :ref:`documentation <dcs_failsafe_mode>`.
+
+- Citus support (Alexander, Polina, Jelte Fennema)
+
+  Patroni enables easy deployment and management of `Citus <https://www.citusdata.com>`__ clusters with HA. Please check :ref:`here <citus>` page for more information.
+
+
+**Improvements**
+
+- Suppress recurring errors when dropping unknown but active replication slots (Michael Banck)
+
+  Patroni will still write these logs, but only in DEBUG.
+
+- Run only one monitoring query per HA loop (Alexander)
+
+  It wasn't the case if synchronous replication is enabled.
+
+- Keep only latest failed data directory (William Albertus Dembo)
+
+  If bootstrap failed Patroni used to rename $PGDATA folder with timestamp suffix. From now on the suffix will be ``.failed`` and if such folder exists it is removed before renaming.
+
+- Improved check of synchronous replication connections (Alexander)
+
+  When the new host is added to the ``synchronous_standby_names`` it will be set as synchronous in DCS only when it managed to catch up with the primary in addition to ``pg_stat_replication.sync_state = 'sync'``.
+
+
+**Removed functionality**
+
+- Remove ``patronictl scaffold`` (Alexander)
+
+  The only reason for having it was a hacky way of running standby clusters.
+
+
+Version 2.1.7
+-------------
+
+**Bugfixes**
+
+- Fixed little incompatibilities with legacy python modules (Alexander Kukushkin)
+
+  They prevented from building/running Patroni on Debian buster/Ubuntu bionic.
+
+
+Version 2.1.6
+-------------
+
+**Improvements**
+
+- Fix annoying exceptions on ssl socket shutdown (Alexander Kukushkin)
+
+  The HAProxy is closing connections as soon as it got the HTTP Status code leaving no time for Patroni to properly shutdown SSL connection.
+
+- Adjust example Dockerfile for arm64 (Polina Bungina)
+
+  Remove explicit ``amd64`` and ``x86_64``, don't remove ``libnss_files.so.*``.
+
+
+**Security improvements**
+
+- Enforce ``search_path=pg_catalog`` for non-replication connections (Alexander)
+
+  Since Patroni is heavily relying on superuser connections, we want to protect it from the possible attacks carried out using user-defined functions and/or operators in ``public`` schema with the same name and signature as the corresponding objects in ``pg_catalog``. For that, ``search_path=pg_catalog`` is enforced for all connections created by Patroni (except replication connections).
+
+- Prevent passwords from being recorded in ``pg_stat_statements`` (Feike Steenbergen)
+
+  It is achieved by setting ``pg_stat_statements.track_utility=off`` when creating users.
+
+
+**Bugfixes**
+
+- Declare ``proxy_address`` as optional (Denis Laxalde)
+
+  As it is effectively a non-required option.
+
+- Improve behaviour of the insecure option (Alexander)
+
+  Ctl's ``insecure`` option didn't work properly when client certificates were used for REST API requests.
+
+- Take watchdog configuration from ``bootstrap.dcs`` when the new cluster is bootstrapped (Matt Baker)
+
+  Patroni used to initially configure watchdog with defaults when bootstrapping a new cluster rather than taking configuration used to bootstrap the DCS.
+
+- Fix the way file extensions are treated while finding executables in WIN32 (Martín Marqués)
+
+  Only add ``.exe`` to a file name if it has no extension yet.
+
+- Fix Consul TTL setup (Alexander)
+
+  We used ``ttl/2.0`` when setting the value on the HTTPClient, but forgot to multiply the current value by 2 in the class' property. It was resulting in Consul TTL off by twice.
+
+
+**Removed functionality**
+
+- Remove ``patronictl configure`` (Polina)
+
+  There is no more need for a separate ``patronictl`` config creation.
+
+
+Version 2.1.5
+-------------
+
+This version enhances compatibility with PostgreSQL 15 and declares Etcd v3 support as production ready. The Patroni on Raft remains in Beta.
+
+**New features**
+
+- Improve ``patroni --validate-config`` (Denis Laxalde)
+
+  Exit with code 1 if config is invalid and print errors to stderr.
+
+- Don't drop replication slots in pause (Alexander Kukushkin)
+
+  Patroni is automatically creating/removing physical replication slots when members are joining/leaving the cluster. In pause slots will no longer be removed.
+
+- Support the ``HEAD`` request method for monitoring endpoints (Robert Cutajar)
+
+  If used instead of ``GET`` Patroni will return only the HTTP Status Code.
+
+- Support behave tests on Windows (Alexander)
+
+  Emulate graceful Patroni shutdown (``SIGTERM``) on Windows by introduce the new REST API endpoint ``POST /sigterm``.
+
+- Introduce ``postgresql.proxy_address`` (Alexander)
+
+  It will be written to the member key in DCS as the ``proxy_url`` and could be used/useful for service discovery.
+
+
+**Stability improvements**
+
+- Call ``pg_replication_slot_advance()`` from a thread (Alexander)
+
+  On busy clusters with many logical replication slots the ``pg_replication_slot_advance()`` call was affecting the main HA loop and could result in the member key expiration.
+
+- Archive possibly missing WALs before calling ``pg_rewind`` on the old primary (Polina Bungina)
+
+  If the primary crashed and was down during considerable time, some WAL files could be missing from archive and from the new primary. There is a chance that ``pg_rewind`` could remove these WAL files from the old primary making it impossible to start it as a standby. By archiving ``ready`` WAL files we not only mitigate this problem but in general improving continues archiving experience.
+
+- Ignore ``403`` errors when trying to create Kubernetes Service (Nick Hudson, Polina)
+
+  Patroni was spamming logs by unsuccessful attempts to create the service, which in fact could already exist.
+
+- Improve liveness probe (Alexander)
+
+  The liveness problem will start failing if the heartbeat loop is running longer than `ttl` on the primary or `2*ttl` on the replica. That will allow us to use it as an alternative for :ref:`watchdog <watchdog>` on Kubernetes.
+
+- Make sure only sync node tries to grab the lock when switchover (Alexander, Polina)
+
+  Previously there was a slim chance that up-to-date async member could become the leader if the manual switchover was performed without specifying the target.
+
+- Avoid cloning while bootstrap is running (Ants Aasma)
+
+  Do not allow a create replica method that does not require a leader to be triggered while the cluster bootstrap is running.
+
+- Compatibility with kazoo-2.9.0 (Alexander)
+
+  Depending on python version the ``SequentialThreadingHandler.select()`` method may raise ``TypeError`` and ``IOError`` exceptions if ``select()`` is called on the closed socket.
+
+- Explicitly shut down SSL connection before socket shutdown (Alexander)
+
+  Not doing it resulted in ``unexpected eof while reading`` errors with OpenSSL 3.0.
+
+- Compatibility with `prettytable>=2.2.0` (Alexander)
+
+  Due to the internal API changes the cluster name header was shown on the incorrect line.
+
+
+**Bugfixes**
+
+- Handle expired token for Etcd lease_grant (monsterxx03)
+
+  In case of error get the new token and retry request.
+
+- Fix bug in the ``GET /read-only-sync`` endpoint (Alexander)
+
+  It was introduced in previous release and effectively never worked.
+
+- Handle the case when data dir storage disappeared (Alexander)
+
+  Patroni is periodically checking that the PGDATA is there and not empty, but in case of issues with storage the ``os.listdir()`` is raising the ``OSError`` exception, breaking the heart-beat loop.
+
+- Apply ``master_stop_timeout`` when waiting for user backends to close (Alexander)
+
+  Something that looks like user backend could be in fact a background worker (e.g., Citus Maintenance Daemon) that is failing to stop.
+
+- Accept ``*:<port>`` for ``postgresql.listen`` (Denis)
+
+  The ``patroni --validate-config`` was complaining about it being invalid.
+
+- Timeouts fixes in Raft (Alexander)
+
+  When Patroni or patronictl are starting they try to get Raft cluster topology from known members. These calls were made without proper timeouts.
+
+- Forcefully update consul service if token was changed (John A. Lotoski)
+
+  Not doing so results in errors "rpc error making call: rpc error making call: ACL not found".
+
+
 Version 2.1.4
 -------------
 
@@ -1108,7 +1387,7 @@ Version 1.6.1
 
 - Kill all children along with the callback process before starting the new one (Alexander Kukushkin)
 
-  Not doing so makes it hard to implement callbacks in bash and eventually can lead to the situation when two callbacks are running at the same time. 
+  Not doing so makes it hard to implement callbacks in bash and eventually can lead to the situation when two callbacks are running at the same time.
 
 - Fix 'start failed' issue (Alexander Kukushkin)
 

docs/replica_bootstrap.rst

@@ -63,7 +63,7 @@ Building replicas
 -----------------
 
 Patroni uses tried and proven ``pg_basebackup`` in order to create new replicas. One downside of it is that it requires
-a running master node. Another one is the lack of 'on-the-fly' compression for the backup data and no built-in cleanup
+a running leader node. Another one is the lack of 'on-the-fly' compression for the backup data and no built-in cleanup
 for outdated backup files. Some people prefer other backup solutions, such as ``WAL-E``, ``pgBackRest``, ``Barman`` and
 others, or simply roll their own scripts. In order to accommodate all those use-cases Patroni supports running custom
 scripts to clone a new replica. Those are configured in the ``postgresql`` configuration block:
@@ -77,7 +77,7 @@ scripts to clone a new replica. Those are configured in the ``postgresql`` confi
             command: <command name>
             keep_data: True
             no_params: True
-            no_master: 1
+            no_leader: 1
 
 example: wal_e
 
@@ -89,7 +89,7 @@ example: wal_e
             - basebackup
         wal_e:
             command: patroni_wale_restore
-            no_master: 1
+            no_leader: 1
             envdir: {{WALE_ENV_DIR}}
             use_iam: 1
         basebackup:
@@ -123,11 +123,11 @@ to execute and any custom parameters that should be passed to that command. All
 --role
     Always 'replica'
 --connstring
-    Connection string to connect to the cluster member to clone from (master or other replica). The user in the
+    Connection string to connect to the cluster member to clone from (primary or other replica). The user in the
     connection string can execute SQL and replication protocol commands.
 
-A special ``no_master`` parameter, if defined, allows Patroni to call the replica creation method even if there is no
-running master or replicas. In that case, an empty string will be passed in a connection string. This is useful for
+A special ``no_leader`` parameter, if defined, allows Patroni to call the replica creation method even if there is no
+running leader or replicas. In that case, an empty string will be passed in a connection string. This is useful for
 restoring the formerly running cluster from the binary backup.
 
 A special ``keep_data`` parameter, if defined, will instruct Patroni to  not clean PGDATA folder before calling restore.
@@ -137,7 +137,7 @@ A special ``no_params`` parameter, if defined, restricts passing parameters to c
 A ``basebackup`` method is a special case: it will be used if
 ``create_replica_methods`` is empty, although it is possible
 to list it explicitly among the ``create_replica_methods`` methods. This method initializes a new replica with the
-``pg_basebackup``, the base backup is taken from the master unless there are replicas with ``clonefrom`` tag, in which case one
+``pg_basebackup``, the base backup is taken from the leader unless there are replicas with ``clonefrom`` tag, in which case one
 of such replicas will be used as the origin for pg_basebackup. It works without any configuration; however, it is
 possible to specify a ``basebackup`` configuration section. Same rules as with the other method configuration apply,
 namely, only long (with --) options should be specified there. Not all parameters make sense, if you override a connection
@@ -176,10 +176,10 @@ Standby cluster
 ---------------
 
 Another available option is to run a "standby cluster", that contains only of
-standby nodes replicating from some remote master. This type of clusters has:
+standby nodes replicating from some remote node. This type of clusters has:
 
 * "standby leader", that behaves pretty much like a regular cluster leader,
-  except it replicates from a remote master.
+  except it replicates from a remote node.
 
 * cascade replicas, that are replicating from standby leader.
 
@@ -187,6 +187,13 @@ Standby leader holds and updates a leader lock in DCS. If the leader lock
 expires, cascade replicas will perform an election to choose another leader
 from the standbys.
 
+There is no further relationship between the standby cluster and the primary
+cluster it replicates from, in particular, they must not share the same DCS
+scope if they use the same DCS. They do not know anything else from each other
+apart from replication information. Also, the standby cluster is not being
+displayed in ``patronictl list`` or ``patronictl topology`` output on the
+primary cluster.
+
 For the sake of flexibility, you can specify methods of creating a replica and
 recovery WAL records when a cluster is in the "standby mode" by providing
 `create_replica_methods` key in `standby_cluster` section. It is distinct from
@@ -212,4 +219,9 @@ in a patroni configuration:
 Note, that these options will be applied only once during cluster bootstrap,
 and the only way to change them afterwards is through DCS.
 
+Patroni expects to find `postgresql.conf` or `postgresql.conf.backup` in PGDATA
+of the remote primary and will not start if it does not find it after a
+basebackup. If the remote primary keeps its `postgresql.conf` elsewhere, it is
+your responsibility to copy it to PGDATA.
+
 If you use replication slots on the standby cluster, you must also create the corresponding replication slot on the primary cluster.  It will not be done automatically by the standby cluster implementation.  You can use Patroni's permanent replication slots feature on the primary cluster to maintain a replication slot with the same name as ``primary_slot_name``, or its default value if ``primary_slot_name`` is not provided.

docs/replication_modes.rst

@@ -13,7 +13,7 @@ In asynchronous mode the cluster is allowed to lose some committed transactions
 
 The amount of transactions that can be lost is controlled via ``maximum_lag_on_failover`` parameter. Because the primary transaction log position is not sampled in real time, in reality the amount of lost data on failover is worst case bounded by  ``maximum_lag_on_failover`` bytes of transaction log plus the amount that is written in the last ``ttl`` seconds (``loop_wait``/2 seconds in the average case). However typical steady state replication delay is well under a second.
 
-By default, when running leader elections, Patroni does not take into account the current timeline of replicas, what in some cases could be undesirable behavior. You can prevent the node not having the same timeline as a former master become the new leader by changing the value of ``check_timeline`` parameter to ``true``.
+By default, when running leader elections, Patroni does not take into account the current timeline of replicas, what in some cases could be undesirable behavior. You can prevent the node not having the same timeline as a former primary become the new leader by changing the value of ``check_timeline`` parameter to ``true``.
 
 PostgreSQL synchronous replication
 ----------------------------------

docs/rest_api.rst

@@ -7,12 +7,11 @@ Patroni has a rich REST API, which is used by Patroni itself during the leader r
 
 Health check endpoints
 ----------------------
-For all health check ``GET`` requests Patroni returns a JSON document with the status of the node, along with the HTTP status code. If you don't want or don't need the JSON document, you might consider using the ``OPTIONS`` method instead of ``GET``.
+For all health check ``GET`` requests Patroni returns a JSON document with the status of the node, along with the HTTP status code. If you don't want or don't need the JSON document, you might consider using the ``HEAD`` or ``OPTIONS`` method instead of ``GET``.
 
 - The following requests to Patroni REST API will return HTTP status code **200** only when the Patroni node is running as the primary with leader lock:
 
   - ``GET /``
-  - ``GET /master``
   - ``GET /primary``
   - ``GET /read-write``
 
@@ -33,7 +32,6 @@ For all health check ``GET`` requests Patroni returns a JSON document with the s
 
  In the following requests, since we are checking for the leader or standby-leader status, Patroni doesn't apply any of the user defined tags and they will be ignored.
   - ``GET /?tag_key1=value1&tag_key2=value2``
-  - ``GET /master?tag_key1=value1&tag_key2=value2``
   - ``GET /leader?tag_key1=value1&tag_key2=value2``
   - ``GET /primary?tag_key1=value1&tag_key2=value2``
   - ``GET /read-write?tag_key1=value1&tag_key2=value2``
@@ -58,7 +56,7 @@ For all health check ``GET`` requests Patroni returns a JSON document with the s
 
 - ``GET /health``: returns HTTP status code **200** only when PostgreSQL is up and running.
 
-- ``GET /liveness``: always returns HTTP status code **200** what only indicates that Patroni is running. Could be used for ``livenessProbe``.
+- ``GET /liveness``: returns HTTP status code **200** if Patroni heartbeat loop is properly running and **503** if the last run was more than ``ttl`` seconds ago on the primary or ``2*ttl`` on the replica. Could be used for ``livenessProbe``.
 
 - ``GET /readiness``: returns HTTP status code **200** when the Patroni node is running as the leader or when PostgreSQL is up and running. The endpoint could be used for ``readinessProbe`` when it is not possible to use Kubernetes endpoints for leader elections (OpenShift).
 
@@ -113,6 +111,69 @@ The ``GET /patroni`` is used by Patroni during the leader race. It also could be
       }
     }
 
+
+Retrieve the Patroni metrics in Prometheus format through the ``GET /metrics`` endpoint.
+
+.. code-block:: bash
+
+	$ curl http://localhost:8008/metrics
+	
+	# HELP patroni_version Patroni semver without periods. \
+	# TYPE patroni_version gauge
+	patroni_version{scope="batman"} 020103
+	# HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise.
+	# TYPE patroni_postgres_running gauge
+	patroni_postgres_running{scope="batman"} 1
+	# HELP patroni_postmaster_start_time Epoch seconds since Postgres started.
+	# TYPE patroni_postmaster_start_time gauge
+	patroni_postmaster_start_time{scope="batman"} 1657656955.179243
+	# HELP patroni_master Value is 1 if this node is the leader, 0 otherwise.
+	# TYPE patroni_master gauge
+	patroni_master{scope="batman"} 1
+	# HELP patroni_xlog_location Current location of the Postgres transaction log, 0 if this node is not the leader.
+	# TYPE patroni_xlog_location counter
+	patroni_xlog_location{scope="batman"} 22320573386952
+	# HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise.
+	# TYPE patroni_standby_leader gauge
+	patroni_standby_leader{scope="batman"} 0
+	# HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise.
+	# TYPE patroni_replica gauge
+	patroni_replica{scope="batman"} 0
+	# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.
+        # TYPE patroni_sync_standby gauge
+        patroni_sync_standby{scope="batman"} 0
+	# HELP patroni_xlog_received_location Current location of the received Postgres transaction log, 0 if this node is not a replica.
+	# TYPE patroni_xlog_received_location counter
+	patroni_xlog_received_location{scope="batman"} 0
+	# HELP patroni_xlog_replayed_location Current location of the replayed Postgres transaction log, 0 if this node is not a replica.
+	# TYPE patroni_xlog_replayed_location counter
+	patroni_xlog_replayed_location{scope="batman"} 0
+	# HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed Postgres transaction log, 0 if null.
+	# TYPE patroni_xlog_replayed_timestamp gauge
+	patroni_xlog_replayed_timestamp{scope="batman"} 0
+	# HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise.
+	# TYPE patroni_xlog_paused gauge
+	patroni_xlog_paused{scope="batman"} 0
+	# HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise.
+	# TYPE patroni_postgres_server_version gauge
+	patroni_postgres_server_version {scope="batman"} 140004
+	# HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked.
+	# TYPE patroni_cluster_unlocked gauge
+	patroni_cluster_unlocked{scope="batman"} 0
+	# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.
+	# TYPE patroni_postgres_timeline counter
+	patroni_postgres_timeline{scope="batman"} 24
+	# HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully by Patroni.
+	# TYPE patroni_dcs_last_seen gauge
+	patroni_dcs_last_seen{scope="batman"} 1677658321
+	# HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise.
+	# TYPE patroni_pending_restart gauge
+	patroni_pending_restart{scope="batman"} 1
+	# HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise.
+	# TYPE patroni_is_paused gauge
+	patroni_is_paused{scope="batman"} 1
+
+
 Cluster status endpoints
 ------------------------
 
@@ -368,7 +429,7 @@ Restart endpoint
   - **restart_pending**: boolean, if set to ``true`` Patroni will restart PostgreSQL only when restart is pending in order to apply some changes in the PostgreSQL config.
   - **role**: perform restart only if the current role of the node matches with the role from the POST request.
   - **postgres_version**: perform restart only if the current version of postgres is smaller than specified in the POST request.
-  - **timeout**: how long we should wait before PostgreSQL starts accepting connections. Overrides ``master_start_timeout``.
+  - **timeout**: how long we should wait before PostgreSQL starts accepting connections. Overrides ``primary_start_timeout``.
   - **schedule**: timestamp with time zone, schedule the restart somewhere in the future.
 
 - ``DELETE /restart``: delete the scheduled restart

docs/security.rst

@@ -9,11 +9,11 @@ A Patroni cluster has two interfaces to be protected from unauthorized access: t
 Protecting DCS
 ==============
 
-Patroni and patronictl both store and retrieve data to/from the DCS. 
+Patroni and patronictl both store and retrieve data to/from the DCS.
 
 Despite DCS doesn't contain any sensitive information, it allows changing some of Patroni/Postgres configuration. Therefore the very first thing that should be protected is DCS itself.
 
-The details of protection depend on the type of DCS used. The authentication and encryption parameters (tokens/basic-auth/client certificates) for the supported types of DCS are covered in :ref:`SETTINGS <bootstrap_settings>` 
+The details of protection depend on the type of DCS used. The authentication and encryption parameters (tokens/basic-auth/client certificates) for the supported types of DCS are covered in :ref:`SETTINGS <bootstrap_settings>`
 
 The general recommendation is to enable TLS for all DCS communication.
 
@@ -22,7 +22,7 @@ Protecting the REST API
 
 Protecting the REST API is a more complicated task.
 
-The Patroni REST API is used by Patroni itself during the leader race, by the ``patronictl`` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring. 
+The Patroni REST API is used by Patroni itself during the leader race, by the ``patronictl`` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring.
 
 From the point of view of security, REST API contains safe (``GET`` requests, only retrieve information) and unsafe (``PUT``, ``POST``, ``PATCH`` and ``DELETE`` requests, change the state of nodes) endpoints.
 

docs/watchdog.rst

@@ -3,7 +3,7 @@
 Watchdog support
 ================
 
-Having multiple PostgreSQL servers running as master can result in transactions lost due to diverging timelines. This situation is also called a split-brain problem. To avoid split-brain Patroni needs to ensure PostgreSQL will not accept any transaction commits after leader key expires in the DCS. Under normal circumstances Patroni will try to achieve this by stopping PostgreSQL when leader lock update fails for any reason. However, this may fail to happen due to various reasons:
+Having multiple PostgreSQL servers running as primary can result in transactions lost due to diverging timelines. This situation is also called a split-brain problem. To avoid split-brain Patroni needs to ensure PostgreSQL will not accept any transaction commits after leader key expires in the DCS. Under normal circumstances Patroni will try to achieve this by stopping PostgreSQL when leader lock update fails for any reason. However, this may fail to happen due to various reasons:
 
 - Patroni has crashed due to a bug, out-of-memory condition or by being accidentally killed by a system administrator.
 
@@ -13,7 +13,7 @@ Having multiple PostgreSQL servers running as master can result in transactions
 
 To guarantee correct behavior under these conditions Patroni supports watchdog devices. Watchdog devices are software or hardware mechanisms that will reset the whole system when they do not get a keepalive heartbeat within a specified timeframe. This adds an additional layer of fail safe in case usual Patroni split-brain protection mechanisms fail.
 
-Patroni will try to activate the watchdog before promoting PostgreSQL to master. If watchdog activation fails and watchdog mode is ``required`` then the node will refuse to become master. When deciding to participate in leader election Patroni will also check that watchdog configuration will allow it to become leader at all. After demoting PostgreSQL (for example due to a manual failover) Patroni will disable the watchdog again. Watchdog will also be disabled while Patroni is in paused state.
+Patroni will try to activate the watchdog before promoting PostgreSQL to primary. If watchdog activation fails and watchdog mode is ``required`` then the node will refuse to become leader. When deciding to participate in leader election Patroni will also check that watchdog configuration will allow it to become leader at all. After demoting PostgreSQL (for example due to a manual failover) Patroni will disable the watchdog again. Watchdog will also be disabled while Patroni is in paused state.
 
 By default Patroni will set up the watchdog to expire 5 seconds before TTL expires. With the default setup of ``loop_wait=10`` and ``ttl=30`` this gives HA loop at least 15 seconds (``ttl`` - ``safety_margin`` - ``loop_wait``) to complete before the system gets forcefully reset. By default accessing DCS is configured to time out after 10 seconds. This means that when DCS is unavailable, for example due to network issues, Patroni and PostgreSQL will have at least 5 seconds (``ttl`` - ``safety_margin`` - ``loop_wait`` - ``retry_timeout``) to come to a state where all client connections are terminated.
 

extras/confd/conf.d/haproxy.toml

@@ -9,5 +9,5 @@ check_cmd = "/usr/sbin/haproxy -c -f {{ .src }}"
 reload_cmd = "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)"
 
 keys = [
-    "/members/",
+    "/",
 ]

extras/confd/templates/haproxy-citus.tmpl

@@ -0,0 +1,32 @@
+global
+	maxconn 100
+
+defaults
+	log	global
+	mode	tcp
+	retries 2
+	timeout client 30m
+	timeout connect 4s
+	timeout server 30m
+	timeout check 5s
+
+listen stats
+	mode http
+	bind *:7000
+	stats enable
+	stats uri /
+
+listen coordinator
+	bind *:5000
+	option httpchk HEAD /primary
+	http-check expect status 200
+	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
+{{range gets "/0/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check check-ssl port {{index (split (index (split $data.api_url "/") 2) ":") 1}} verify required ca-file /etc/ssl/certs/ssl-cert-snakeoil.pem crt /etc/ssl/private/ssl-cert-snakeoil.crt
+{{end}}
+listen workers
+	bind *:5001
+	option httpchk HEAD /primary
+	http-check expect status 200
+	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
+{{range gets "/*/members/*"}}{{$group := index (split .Key "/") 1}}{{if ne $group "0"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check check-ssl port {{index (split (index (split $data.api_url "/") 2) ":") 1}} verify required ca-file /etc/ssl/certs/ssl-cert-snakeoil.pem crt /etc/ssl/private/ssl-cert-snakeoil.crt
+{{end}}{{end}}

extras/confd/templates/haproxy.tmpl

@@ -16,16 +16,16 @@ listen stats
 	stats enable
 	stats uri /
 
-listen master
+listen primary
 	bind *:5000
-	option httpchk OPTIONS /master
+	option httpchk HEAD /primary
 	http-check expect status 200
 	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
 {{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
 {{end}}
 listen replicas
 	bind *:5001
-	option httpchk OPTIONS /replica
+	option httpchk HEAD /replica
 	http-check expect status 200
 	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
 {{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}

extras/startup-scripts/README.md

@@ -1,6 +1,6 @@
 # startup scripts for Patroni
 
-This directory contains sample startup scripts for various OSes 
+This directory contains sample startup scripts for various OSes
 and management tools for Patroni.
 
 Scripts supplied:
@@ -10,7 +10,7 @@ Scripts supplied:
 Upstart job for Ubuntu 12.04 or 14.04.  Requires Upstart > 1.4. Intended for systems where Patroni has been installed on a base system, rather than in Docker.
 
 ### patroni.service
-Systemd service file, to be copied to /etc/systemd/system/patroni.service, tested on Centos 7.1 with Patroni installed from pip. 
+Systemd service file, to be copied to /etc/systemd/system/patroni.service, tested on Centos 7.1 with Patroni installed from pip.
 
 ### patroni
 Init.d service file for Debian-like distributions. Copy it to /etc/init.d/, make executable:

extras/startup-scripts/patroni.service

@@ -14,7 +14,7 @@ Group=postgres
 # Read in configuration file if it exists, otherwise proceed
 EnvironmentFile=-/etc/patroni_env.conf
 
-# the default is the user's home directory, and if you want to change it, you must provide an absolute path.
+# The default is the user's home directory, and if you want to change it, you must provide an absolute path.
 # WorkingDirectory=/home/sameuser
 
 # Where to send early-startup messages from the server
@@ -32,14 +32,14 @@ ExecStart=/bin/patroni /etc/patroni.yml
 # Send HUP to reload from patroni.yml
 ExecReload=/bin/kill -s HUP $MAINPID
 
-# only kill the patroni process, not it's children, so it will gracefully stop postgres
+# Only kill the patroni process, not it's children, so it will gracefully stop postgres
 KillMode=process
 
 # Give a reasonable amount of time for the server to start up/shut down
 TimeoutSec=30
 
-# Do not restart the service if it crashes, we want to manually inspect database on failure
-Restart=no
+# Restart the service if it crashed
+Restart=on-failure
 
 [Install]
 WantedBy=multi-user.target

features/basic_replication.feature

@@ -5,7 +5,7 @@ Feature: basic replication
     Given I start postgres0
     Then postgres0 is a leader after 10 seconds
     And there is a non empty initialize key in DCS after 15 seconds
-    When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 2, "synchronous_mode": true}
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "synchronous_mode": true}
     Then I receive a response code 200
     When I start postgres1
     And I configure and start postgres2 with a tag replicatefrom postgres0
@@ -21,12 +21,9 @@ Feature: basic replication
     And I shut down postgres1
     Then "sync" key in DCS has sync_standby=postgres2 after 10 seconds
     When I start postgres1
-    And "members/postgres1" key in DCS has state=running after 10 seconds
-    And I sleep for 2 seconds
-    When I issue a GET request to http://127.0.0.1:8010/sync
-    Then I receive a response code 200
-    When I issue a GET request to http://127.0.0.1:8009/async
-    Then I receive a response code 200
+    Then "members/postgres1" key in DCS has state=running after 10 seconds
+    And Status code on GET http://127.0.0.1:8010/sync is 200 after 3 seconds
+    And Status code on GET http://127.0.0.1:8009/async is 200 after 3 seconds
 
   Scenario: check stuck sync replica
     Given I issue a PATCH request to http://127.0.0.1:8008/config with {"pause": true, "maximum_lag_on_syncnode": 15000000, "postgresql": {"parameters": {"synchronous_commit": "remote_apply"}}}
@@ -38,11 +35,8 @@ Feature: basic replication
     And I load data on postgres0
     Then "sync" key in DCS has sync_standby=postgres1 after 15 seconds
     And I resume wal replay on postgres2
-    And I sleep for 2 seconds
-    And I issue a GET request to http://127.0.0.1:8009/sync
-    Then I receive a response code 200
-    When I issue a GET request to http://127.0.0.1:8010/async
-    Then I receive a response code 200
+    And Status code on GET http://127.0.0.1:8009/sync is 200 after 3 seconds
+    And Status code on GET http://127.0.0.1:8010/async is 200 after 3 seconds
     When I issue a PATCH request to http://127.0.0.1:8008/config with {"pause": null, "maximum_lag_on_syncnode": -1, "postgresql": {"parameters": {"synchronous_commit": "on"}}}
     Then I receive a response code 200
     And I drop table on postgres0
@@ -50,23 +44,17 @@ Feature: basic replication
   Scenario: check multi sync replication
     Given I issue a PATCH request to http://127.0.0.1:8008/config with {"synchronous_node_count": 2}
     Then I receive a response code 200
-    And I sleep for 10 seconds
-    Then "sync" key in DCS has sync_standby=postgres1,postgres2 after 5 seconds
-    When I issue a GET request to http://127.0.0.1:8010/sync
-    Then I receive a response code 200
-    When I issue a GET request to http://127.0.0.1:8009/sync
-    Then I receive a response code 200
+    Then "sync" key in DCS has sync_standby=postgres1,postgres2 after 10 seconds
+    And Status code on GET http://127.0.0.1:8010/sync is 200 after 3 seconds
+    And Status code on GET http://127.0.0.1:8009/sync is 200 after 3 seconds
     When I issue a PATCH request to http://127.0.0.1:8008/config with {"synchronous_node_count": 1}
     Then I receive a response code 200
     And I shut down postgres1
-    And I sleep for 10 seconds
     Then "sync" key in DCS has sync_standby=postgres2 after 10 seconds
     When I start postgres1
-    And "members/postgres1" key in DCS has state=running after 10 seconds
-    When I issue a GET request to http://127.0.0.1:8010/sync
-    Then I receive a response code 200
-    When I issue a GET request to http://127.0.0.1:8009/async
-    Then I receive a response code 200
+    Then "members/postgres1" key in DCS has state=running after 10 seconds
+    And Status code on GET http://127.0.0.1:8010/sync is 200 after 3 seconds
+    And Status code on GET http://127.0.0.1:8009/async is 200 after 3 seconds
 
   Scenario: check the basic failover in synchronous mode
     Given I run patronictl.py pause batman
@@ -77,6 +65,7 @@ Feature: basic replication
     Then I receive a response returncode 0
     And postgres2 role is the primary after 24 seconds
     And Response on GET http://127.0.0.1:8010/history contains recovery after 10 seconds
+    And there is a postgres2_cb.log with "on_role_change master batman" in postgres2 data directory
     When I issue a PATCH request to http://127.0.0.1:8010/config with {"synchronous_mode": null, "master_start_timeout": 0}
     Then I receive a response code 200
     When I add the table bar to postgres2
@@ -88,7 +77,7 @@ Feature: basic replication
     Then postgres1 is a leader after 10 seconds
     And postgres1 role is the primary after 10 seconds
 
-  Scenario: check rejoin of the former master with pg_rewind
+  Scenario: check rejoin of the former primary with pg_rewind
     Given I add the table splitbrain to postgres0
     And I start postgres0
     Then postgres0 role is the secondary after 20 seconds

features/citus.feature

@@ -0,0 +1,72 @@
+Feature: citus
+  We should check that coordinator discovers and registers workers and clients don't have errors when worker cluster switches over
+
+  Scenario: check that worker cluster is registered in the coordinator
+    Given I start postgres0 in citus group 0
+    And I start postgres2 in citus group 1
+    Then postgres0 is a leader in a group 0 after 10 seconds
+    And postgres2 is a leader in a group 1 after 10 seconds
+    When I start postgres1 in citus group 0
+    And I start postgres3 in citus group 1
+    Then replication works from postgres0 to postgres1 after 15 seconds
+    Then replication works from postgres2 to postgres3 after 15 seconds
+    And postgres0 is registered in the postgres0 as the worker in group 0
+    And postgres2 is registered in the postgres0 as the worker in group 1
+
+  Scenario: coordinator failover updates pg_dist_node
+    Given I run patronictl.py failover batman --group 0 --candidate postgres1 --force
+    Then postgres1 role is the primary after 10 seconds
+    And replication works from postgres1 to postgres0 after 15 seconds
+    And "sync" key in a group 0 in DCS has sync_standby=postgres0 after 15 seconds
+    And postgres1 is registered in the postgres2 as the worker in group 0
+    When I run patronictl.py failover batman --group 0 --candidate postgres0 --force
+    Then postgres0 role is the primary after 10 seconds
+    And replication works from postgres0 to postgres1 after 15 seconds
+    And "sync" key in a group 0 in DCS has sync_standby=postgres1 after 15 seconds
+    And postgres0 is registered in the postgres2 as the worker in group 0
+
+  Scenario: worker switchover doesn't break client queries on the coordinator
+    Given I create a distributed table on postgres0
+    And I start a thread inserting data on postgres0
+    When I run patronictl.py switchover batman --group 1 --force
+    Then I receive a response returncode 0
+    And postgres3 role is the primary after 10 seconds
+    And replication works from postgres3 to postgres2 after 15 seconds
+    And "sync" key in a group 1 in DCS has sync_standby=postgres2 after 15 seconds
+    And postgres3 is registered in the postgres0 as the worker in group 1
+    And a thread is still alive
+    When I run patronictl.py switchover batman --group 1 --force
+    Then I receive a response returncode 0
+    And postgres2 role is the primary after 10 seconds
+    And replication works from postgres2 to postgres3 after 15 seconds
+    And "sync" key in a group 1 in DCS has sync_standby=postgres3 after 15 seconds
+    And postgres2 is registered in the postgres0 as the worker in group 1
+    And a thread is still alive
+    When I stop a thread
+    Then a distributed table on postgres0 has expected rows
+
+  Scenario: worker primary restart doesn't break client queries on the coordinator
+    Given I cleanup a distributed table on postgres0
+    And I start a thread inserting data on postgres0
+    When I run patronictl.py restart batman postgres2 --group 1 --force
+    Then I receive a response returncode 0
+    And postgres2 role is the primary after 10 seconds
+    And replication works from postgres2 to postgres3 after 15 seconds
+    And postgres2 is registered in the postgres0 as the worker in group 1
+    And a thread is still alive
+    When I stop a thread
+    Then a distributed table on postgres0 has expected rows
+
+  Scenario: check that in-flight transaction is rolled back after timeout when other workers need to change pg_dist_node
+    Given I start postgres4 in citus group 2
+    Then postgres4 is a leader in a group 2 after 10 seconds
+    And "members/postgres4" key in a group 2 in DCS has role=master after 3 seconds
+    When I run patronictl.py edit-config batman --group 2 -s ttl=20 --force
+    Then I receive a response returncode 0
+    And I receive a response output "+ttl: 20"
+    When I sleep for 2 seconds
+    Then postgres4 is registered in the postgres2 as the worker in group 2
+    When I shut down postgres4
+    Then There is a transaction in progress on postgres0 changing pg_dist_node
+    When I run patronictl.py restart batman postgres2 --group 1 --force
+    Then a transaction finishes in 20 seconds

features/dcs_failsafe_mode.feature

@@ -0,0 +1,85 @@
+Feature: dcs failsafe mode
+  We should check the basic dcs failsafe mode functioning
+
+  Scenario: check failsafe mode can be successfully enabled
+    Given I start postgres0
+    And postgres0 is a leader after 10 seconds
+    And I sleep for 3 seconds
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"loop_wait": 2, "ttl": 20, "retry_timeout": 5, "failsafe_mode": true}
+    Then I receive a response code 200
+    And Response on GET http://127.0.0.1:8008/failsafe contains postgres0 after 10 seconds
+    When I issue a GET request to http://127.0.0.1:8008/failsafe
+    Then I receive a response code 200
+    And I receive a response postgres0 http://127.0.0.1:8008/patroni
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"wal_level": "logical"}}}
+    Then I receive a response code 200
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"slots": {"dcs_slot_0": {"type": "logical", "database": "postgres", "plugin": "test_decoding"}}}
+    Then I receive a response code 200
+
+  @dcs-failsafe
+  Scenario: check one-node cluster is functioning while DCS is down
+    Given DCS is down
+    Then Response on GET http://127.0.0.1:8008/primary contains failsafe_mode_is_active after 12 seconds
+    And postgres0 role is the primary after 10 seconds
+
+  @dcs-failsafe
+  Scenario: check new replica isn't promoted when leader is down and DCS is up
+    Given DCS is up
+    When I do a backup of postgres0
+    And I shut down postgres0
+    When I start postgres1 in a cluster batman from backup with no_leader
+    And I sleep for 2 seconds
+    Then postgres1 role is the replica after 12 seconds
+
+  Scenario: check leader and replica are both in /failsafe key after leader is back
+    Given I start postgres0
+    And I start postgres1
+    Then "members/postgres0" key in DCS has state=running after 10 seconds
+    And "members/postgres1" key in DCS has state=running after 2 seconds
+    And Response on GET http://127.0.0.1:8009/failsafe contains postgres1 after 10 seconds
+    When I issue a GET request to http://127.0.0.1:8009/failsafe
+    Then I receive a response code 200
+    And I receive a response postgres0 http://127.0.0.1:8008/patroni
+    And I receive a response postgres1 http://127.0.0.1:8009/patroni
+
+  @dcs-failsafe
+  @slot-advance
+  Scenario: check leader and replica are functioning while DCS is down
+    Given logical slot dcs_slot_0 is in sync between postgres0 and postgres1 after 10 seconds
+    And DCS is down
+    Then Response on GET http://127.0.0.1:8008/primary contains failsafe_mode_is_active after 12 seconds
+    Then postgres0 role is the primary after 10 seconds
+    And postgres1 role is the replica after 2 seconds
+    And replication works from postgres0 to postgres1 after 10 seconds
+    And I get all changes from logical slot dcs_slot_0 on postgres0
+    And logical slot dcs_slot_0 is in sync between postgres0 and postgres1 after 20 seconds
+
+  @dcs-failsafe
+  Scenario: check primary is demoted when one replica is shut down and DCS is down
+    Given DCS is down
+    And I kill postgres1
+    And I kill postmaster on postgres1
+    And I sleep for 2 seconds
+    Then postgres0 role is the replica after 12 seconds
+
+  @dcs-failsafe
+  Scenario: check known replica is promoted when leader is down and DCS is up
+    Given I shut down postgres0
+    And DCS is up
+    When I start postgres1
+    Then "members/postgres1" key in DCS has state=running after 10 seconds
+    And postgres1 role is the primary after 25 seconds
+
+  @dcs-failsafe
+  Scenario: check three-node cluster is functioning while DCS is down
+    Given I start postgres0
+    And I start postgres2
+    Then "members/postgres2" key in DCS has state=running after 10 seconds
+    And "members/postgres0" key in DCS has state=running after 20 seconds
+    And Response on GET http://127.0.0.1:8008/failsafe contains postgres2 after 10 seconds
+    And replication works from postgres1 to postgres0 after 10 seconds
+    Given DCS is down
+    Then Response on GET http://127.0.0.1:8008/primary contains failsafe_mode_is_active after 12 seconds
+    Then postgres1 role is the primary after 10 seconds
+    And postgres0 role is the replica after 2 seconds
+    And postgres2 role is the replica after 2 seconds

features/environment.py

@@ -1,10 +1,12 @@
 import abc
 import datetime
+import glob
 import os
 import json
+import psutil
+import re
 import shutil
 import signal
-import six
 import subprocess
 import sys
 import tempfile
@@ -14,11 +16,11 @@ import yaml
 
 import patroni.psycopg as psycopg
 
-from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from patroni.request import PatroniRequest
 
 
-@six.add_metaclass(abc.ABCMeta)
-class AbstractController(object):
+class AbstractController(abc.ABC):
 
     def __init__(self, context, name, work_directory, output_dir):
         self._context = context
@@ -99,6 +101,7 @@ class PatroniController(AbstractController):
             self.watchdog = None
 
         self._scope = (custom_config or {}).get('scope', 'batman')
+        self._citus_group = (custom_config or {}).get('citus', {}).get('group')
         self._config = self._make_patroni_test_config(name, custom_config)
         self._closables = []
 
@@ -138,12 +141,24 @@ class PatroniController(AbstractController):
     def _start(self):
         if self.watchdog:
             self.watchdog.start()
+        env = os.environ.copy()
         if isinstance(self._context.dcs_ctl, KubernetesController):
-            self._context.dcs_ctl.create_pod(self._name[8:], self._scope)
-            os.environ['PATRONI_KUBERNETES_POD_IP'] = '10.0.0.' + self._name[-1]
-        return subprocess.Popen([sys.executable, '-m', 'coverage', 'run',
-                                '--source=patroni', '-p', 'patroni.py', self._config],
-                                stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
+            self._context.dcs_ctl.create_pod(self._name[8:], self._scope, self._citus_group)
+            env['PATRONI_KUBERNETES_POD_IP'] = '10.0.0.' + self._name[-1]
+        if os.name == 'nt':
+            env['BEHAVE_DEBUG'] = 'true'
+        patroni = subprocess.Popen([sys.executable, '-m', 'coverage', 'run',
+                                   '--source=patroni', '-p', 'patroni.py', self._config], env=env,
+                                   stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
+        if os.name == 'nt':
+            patroni.terminate = self.terminate
+        return patroni
+
+    def terminate(self):
+        try:
+            self._context.request_executor.request('POST', self._restapi_url + '/sigterm')
+        except Exception:
+            pass
 
     def stop(self, kill=False, timeout=15, postgres=False):
         if postgres:
@@ -164,29 +179,61 @@ class PatroniController(AbstractController):
         patroni_config_name = self.PATRONI_CONFIG.format(name)
         patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
 
-        with open(patroni_config_name) as f:
+        with open('postgres0.yml') as f:
             config = yaml.safe_load(f)
             config.pop('etcd', None)
 
         raft_port = os.environ.get('RAFT_PORT')
-        if raft_port:
+        # If patroni_raft_controller is suspended two Patroni members is enough to get a quorum,
+        # therefore we don't want Patroni to join as a voting member when testing dcs_failsafe_mode.
+        if raft_port and not self._output_dir.endswith('dcs_failsafe_mode'):
             os.environ['RAFT_PORT'] = str(int(raft_port) + 1)
             config['raft'] = {'data_dir': self._output_dir, 'self_addr': 'localhost:' + os.environ['RAFT_PORT']}
 
-        host = config['postgresql']['listen'].split(':')[0]
+        host = config['restapi']['listen'].rsplit(':', 1)[0]
+        config['restapi']['listen'] = config['restapi']['connect_address'] = '{0}:{1}'.format(host, 8008+int(name[-1]))
 
+        host = config['postgresql']['listen'].rsplit(':', 1)[0]
         config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT)
 
         config['name'] = name
-        config['postgresql']['data_dir'] = self._data_dir
+        config['postgresql']['data_dir'] = self._data_dir.replace('\\', '/')
         config['postgresql']['basebackup'] = [{'checkpoint': 'fast'}]
+        config['postgresql']['callbacks'] = {
+            'on_role_change': '{0} features/callback2.py {1}'.format(self._context.pctl.PYTHON, name)}
         config['postgresql']['use_unix_socket'] = os.name != 'nt'  # windows doesn't yet support unix-domain sockets
-        config['postgresql']['use_unix_socket_repl'] = os.name != 'nt'  # windows doesn't yet support unix-domain sockets
-        config['postgresql']['pgpass'] = os.path.join(tempfile.gettempdir(), 'pgpass_' + name)
+        config['postgresql']['use_unix_socket_repl'] = os.name != 'nt'
+        config['postgresql']['pgpass'] = os.path.join(tempfile.gettempdir(), 'pgpass_' + name).replace('\\', '/')
         config['postgresql']['parameters'].update({
-            'logging_collector': 'on', 'log_destination': 'csvlog', 'log_directory': self._output_dir,
+            'logging_collector': 'on', 'log_destination': 'csvlog',
+            'log_directory': self._output_dir.replace('\\', '/'),
             'log_filename': name + '.log', 'log_statement': 'all', 'log_min_messages': 'debug1',
-            'unix_socket_directories': tempfile.gettempdir()})
+            'shared_buffers': '1MB', 'unix_socket_directories': tempfile.gettempdir().replace('\\', '/')})
+        config['postgresql']['pg_hba'] = [
+            'local all all trust',
+            'local replication all trust',
+            'host replication replicator all md5',
+            'host all all all md5'
+        ]
+
+        if self._context.postgres_supports_ssl and self._context.certfile:
+            config['postgresql']['parameters'].update({
+                'ssl': 'on',
+                'ssl_ca_file': self._context.certfile.replace('\\', '/'),
+                'ssl_cert_file': self._context.certfile.replace('\\', '/'),
+                'ssl_key_file': self._context.keyfile.replace('\\', '/')
+            })
+            for user in config['postgresql'].get('authentication').keys():
+                config['postgresql'].get('authentication', {}).get(user, {}).update({
+                    'sslmode': 'verify-ca',
+                    'sslrootcert': self._context.certfile,
+                    'sslcert': self._context.certfile,
+                    'sslkey': self._context.keyfile
+                })
+            for i, line in enumerate(list(config['postgresql']['pg_hba'])):
+                if line.endswith('md5'):
+                    # we want to verify client cert first and than password
+                    config['postgresql']['pg_hba'][i] = 'hostssl' + line[4:] + ' clientcert=verify-ca'
 
         if 'bootstrap' in config:
             config['bootstrap']['post_bootstrap'] = 'psql -w -c "SELECT 1"'
@@ -197,26 +244,43 @@ class PatroniController(AbstractController):
             self.recursive_update(config, custom_config)
 
         self.recursive_update(config, {
-            'bootstrap': {'dcs': {'postgresql': {'parameters': {'wal_keep_segments': 100}}}}})
+            'bootstrap': {
+                'dcs': {
+                    'loop_wait': 2,
+                    'postgresql': {
+                        'parameters': {
+                            'wal_keep_segments': 100,
+                            'archive_mode': 'on',
+                            'archive_command': (PatroniPoolController.ARCHIVE_RESTORE_SCRIPT +
+                                                ' --mode archive ' +
+                                                '--dirname {} --filename %f --pathname %p').format(
+                                                    os.path.join(self._work_directory, 'data', 'wal_archive'))
+                        }
+                    }
+                }
+            }
+        })
         if config['postgresql'].get('callbacks', {}).get('on_role_change'):
             config['postgresql']['callbacks']['on_role_change'] += ' ' + str(self.__PORT)
 
         with open(patroni_config_path, 'w') as f:
             yaml.safe_dump(config, f, default_flow_style=False)
 
-        user = config['postgresql'].get('authentication', config['postgresql']).get('superuser', {})
-        self._connkwargs = {k: user[n] for n, k in [('username', 'user'), ('password', 'password')] if n in user}
-        self._connkwargs.update({'host': host, 'port': self.__PORT, 'dbname': 'postgres'})
+        self._connkwargs = config['postgresql'].get('authentication', config['postgresql']).get('superuser', {})
+        self._connkwargs.update({'host': host, 'port': self.__PORT, 'dbname': 'postgres',
+                                 'user': self._connkwargs.pop('username', None)})
 
         self._replication = config['postgresql'].get('authentication', config['postgresql']).get('replication', {})
-        self._replication.update({'host': host, 'port': self.__PORT, 'dbname': 'postgres'})
+        self._replication.update({'host': host, 'port': self.__PORT, 'user': self._replication.pop('username', None)})
+        self._restapi_url = 'http://{0}'.format(config['restapi']['connect_address'])
+        if self._context.certfile:
+            self._restapi_url = self._restapi_url.replace('http://', 'https://')
 
         return patroni_config_path
 
     def _connection(self):
         if not self._conn or self._conn.closed != 0:
             self._conn = psycopg.connect(**self._connkwargs)
-            self._conn.autocommit = True
         return self._conn
 
     def _cursor(self):
@@ -269,7 +333,10 @@ class PatroniController(AbstractController):
 
     @property
     def backup_source(self):
-        return 'postgres://{username}:{password}@{host}:{port}/{dbname}'.format(**self._replication)
+        def escape(value):
+            return re.sub(r'([\'\\ ])', r'\\\1', str(value))
+
+        return ' '.join('{0}={1}'.format(k, escape(v)) for k, v in self._replication.items())
 
     def backup(self, dest=os.path.join('data', 'basebackup')):
         subprocess.call(PatroniPoolController.BACKUP_SCRIPT + ['--walmethod=none',
@@ -308,6 +375,7 @@ class AbstractDcsController(AbstractController):
 
     def __init__(self, context, mktemp=True):
         work_directory = mktemp and tempfile.mkdtemp() or None
+        self._paused = False
         super(AbstractDcsController, self).__init__(context, self.name(), work_directory, context.pctl.output_dir)
 
     def _is_accessible(self):
@@ -319,11 +387,22 @@ class AbstractDcsController(AbstractController):
         if self._work_directory:
             shutil.rmtree(self._work_directory)
 
-    def path(self, key=None, scope='batman'):
-        return self._CLUSTER_NODE.format(scope) + (key and '/' + key or '')
+    def path(self, key=None, scope='batman', group=None):
+        citus_group = '/{0}'.format(group) if group is not None else ''
+        return self._CLUSTER_NODE.format(scope) + citus_group + (key and '/' + key or '')
+
+    def start_outage(self):
+        if not self._paused and self._handle:
+            self._handle.suspend()
+            self._paused = True
+
+    def stop_outage(self):
+        if self._paused and self._handle:
+            self._handle.resume()
+            self._paused = False
 
     @abc.abstractmethod
-    def query(self, key, scope='batman'):
+    def query(self, key, scope='batman', group=None):
         """ query for a value of a given key """
 
     @abc.abstractmethod
@@ -357,8 +436,8 @@ class ConsulController(AbstractDcsController):
         self._config_file = self._work_directory + '.json'
         with open(self._config_file, 'wb') as f:
             f.write(b'{"session_ttl_min":"5s","server":true,"bootstrap":true,"advertise_addr":"127.0.0.1"}')
-        return subprocess.Popen(['consul', 'agent', '-config-file', self._config_file, '-data-dir',
-                                 self._work_directory], stdout=self._log, stderr=subprocess.STDOUT)
+        return psutil.Popen(['consul', 'agent', '-config-file', self._config_file, '-data-dir',
+                             self._work_directory], stdout=self._log, stderr=subprocess.STDOUT)
 
     def stop(self, kill=False, timeout=15):
         super(ConsulController, self).stop(kill=kill, timeout=timeout)
@@ -371,11 +450,11 @@ class ConsulController(AbstractDcsController):
         except Exception:
             return False
 
-    def path(self, key=None, scope='batman'):
-        return super(ConsulController, self).path(key, scope)[1:]
+    def path(self, key=None, scope='batman', group=None):
+        return super(ConsulController, self).path(key, scope, group)[1:]
 
-    def query(self, key, scope='batman'):
-        _, value = self._client.kv.get(self.path(key, scope))
+    def query(self, key, scope='batman', group=None):
+        _, value = self._client.kv.get(self.path(key, scope, group))
         return value and value['Value'].decode('utf-8')
 
     def cleanup_service_tree(self):
@@ -394,8 +473,8 @@ class AbstractEtcdController(AbstractDcsController):
         self._client_cls = client_cls
 
     def _start(self):
-        return subprocess.Popen(["etcd", "--debug", "--data-dir", self._work_directory],
-                                stdout=self._log, stderr=subprocess.STDOUT)
+        return psutil.Popen(["etcd", "--enable-v2=true", "--data-dir", self._work_directory],
+                            stdout=self._log, stderr=subprocess.STDOUT)
 
     def _is_running(self):
         from patroni.dcs.etcd import DnsCachingResolver
@@ -415,10 +494,10 @@ class EtcdController(AbstractEtcdController):
         super(EtcdController, self).__init__(context, EtcdClient)
         os.environ['PATRONI_ETCD_HOST'] = 'localhost:2379'
 
-    def query(self, key, scope='batman'):
+    def query(self, key, scope='batman', group=None):
         import etcd
         try:
-            return self._client.get(self.path(key, scope)).value
+            return self._client.get(self.path(key, scope, group)).value
         except etcd.EtcdKeyNotFound:
             return None
 
@@ -439,9 +518,9 @@ class Etcd3Controller(AbstractEtcdController):
         super(Etcd3Controller, self).__init__(context, Etcd3Client)
         os.environ['PATRONI_ETCD3_HOST'] = 'localhost:2379'
 
-    def query(self, key, scope='batman'):
+    def query(self, key, scope='batman', group=None):
         import base64
-        response = self._client.range(self.path(key, scope))
+        response = self._client.range(self.path(key, scope, group))
         for k in response.get('kvs', []):
             return base64.b64decode(k['value']).decode('utf-8') if 'value' in k else None
 
@@ -452,7 +531,43 @@ class Etcd3Controller(AbstractEtcdController):
             assert False, "exception when cleaning up etcd contents: {0}".format(e)
 
 
-class KubernetesController(AbstractDcsController):
+class AbstractExternalDcsController(AbstractDcsController):
+
+    def __init__(self, context, mktemp=True):
+        super(AbstractExternalDcsController, self).__init__(context, mktemp)
+        self._wrapper = ['sudo']
+
+    def _start(self):
+        return self._external_pid
+
+    def start_outage(self):
+        if not self._paused:
+            subprocess.call(self._wrapper + ['kill', '-SIGSTOP', self._external_pid])
+            self._paused = True
+
+    def stop_outage(self):
+        if self._paused:
+            subprocess.call(self._wrapper + ['kill', '-SIGCONT', self._external_pid])
+            self._paused = False
+
+    def _has_started(self):
+        return True
+
+    @abc.abstractmethod
+    def process_name():
+        """process name to search with pgrep"""
+
+    def _is_running(self):
+        if not self._handle:
+            self._external_pid = subprocess.check_output(['pgrep', '-nf', self.process_name()]).decode('utf-8').strip()
+            return False
+        return True
+
+    def stop(self):
+        pass
+
+
+class KubernetesController(AbstractExternalDcsController):
 
     def __init__(self, context):
         super(KubernetesController, self).__init__(context)
@@ -461,19 +576,48 @@ class KubernetesController(AbstractDcsController):
         self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
         os.environ['PATRONI_KUBERNETES_LABELS'] = json.dumps(self._labels)
         os.environ['PATRONI_KUBERNETES_USE_ENDPOINTS'] = 'true'
-        os.environ['PATRONI_KUBERNETES_BYPASS_API_SERVICE'] = 'true'
+        os.environ.setdefault('PATRONI_KUBERNETES_BYPASS_API_SERVICE', 'true')
 
         from patroni.dcs.kubernetes import k8s_client, k8s_config
-        k8s_config.load_kube_config(context='local')
+        k8s_config.load_kube_config(context=os.environ.setdefault('PATRONI_KUBERNETES_CONTEXT', 'kind-kind'))
         self._client = k8s_client
         self._api = self._client.CoreV1Api()
 
-    def _start(self):
-        pass
+    def process_name(self):
+        return "localkube"
 
-    def create_pod(self, name, scope):
+    def _is_running(self):
+        if not self._handle:
+            context = os.environ.get('PATRONI_KUBERNETES_CONTEXT')
+            if context.startswith('kind-'):
+                container = '{0}-control-plane'.format(context[5:])
+                api_process = 'kube-apiserver'
+            elif context.startswith('k3d-'):
+                container = '{0}-server-0'.format(context)
+                api_process = 'k3s'
+            else:
+                return super(KubernetesController, self)._is_running()
+            try:
+                docker = 'docker'
+                with open(os.devnull, 'w') as null:
+                    if subprocess.call([docker, 'info'], stdout=null, stderr=null) != 0:
+                        raise Exception
+            except Exception:
+                docker = 'podman'
+                with open(os.devnull, 'w') as null:
+                    if subprocess.call([docker, 'info'], stdout=null, stderr=null) != 0:
+                        raise Exception
+            self._wrapper = [docker, 'exec', container]
+            self._external_pid = subprocess.check_output(self._wrapper + ['pidof', api_process]).decode('utf-8').strip()
+            return False
+        return True
+
+    def create_pod(self, name, scope, group=None):
+        self.delete_pod(name)
         labels = self._labels.copy()
         labels['cluster-name'] = scope
+        if group is not None:
+            labels['citus-group'] = str(group)
         metadata = self._client.V1ObjectMeta(namespace=self._namespace, name=name, labels=labels)
         spec = self._client.V1PodSpec(containers=[self._client.V1Container(name=name, image='empty')])
         body = self._client.V1Pod(metadata=metadata, spec=spec)
@@ -490,12 +634,14 @@ class KubernetesController(AbstractDcsController):
             except Exception:
                 break
 
-    def query(self, key, scope='batman'):
+    def query(self, key, scope='batman', group=None):
         if key.startswith('members/'):
             pod = self._api.read_namespaced_pod(key[8:], self._namespace)
             return (pod.metadata.annotations or {}).get('status', '')
         else:
             try:
+                if group is not None:
+                    scope = '{0}-{1}'.format(scope, group)
                 ep = scope + {'leader': '', 'history': '-config', 'initialize': '-config'}.get(key, '-' + key)
                 e = self._api.read_namespaced_endpoints(ep, self._namespace)
                 if key != 'sync':
@@ -520,11 +666,8 @@ class KubernetesController(AbstractDcsController):
             if len(result.items) < 1:
                 break
 
-    def _is_running(self):
-        return True
 
-
-class ZooKeeperController(AbstractDcsController):
+class ZooKeeperController(AbstractExternalDcsController):
 
     """ handles all zookeeper related tasks, used for the tests setup and cleanup """
 
@@ -536,13 +679,13 @@ class ZooKeeperController(AbstractDcsController):
         import kazoo.client
         self._client = kazoo.client.KazooClient()
 
-    def _start(self):
-        pass  # TODO: implement later
+    def process_name(self):
+        return "zookeeper"
 
-    def query(self, key, scope='batman'):
+    def query(self, key, scope='batman', group=None):
         import kazoo.exceptions
         try:
-            return self._client.get(self.path(key, scope))[0].decode('utf-8')
+            return self._client.get(self.path(key, scope, group))[0].decode('utf-8')
         except kazoo.exceptions.NoNodeError:
             return None
 
@@ -556,6 +699,9 @@ class ZooKeeperController(AbstractDcsController):
             assert False, "exception when cleaning up zookeeper contents: {0}".format(e)
 
     def _is_running(self):
+        if not super(ZooKeeperController, self)._is_running():
+            return False
+
         # if zookeeper is running, but we didn't start it
         if self._client.connected:
             return True
@@ -605,12 +751,12 @@ class RaftController(AbstractDcsController):
         del env['PATRONI_RAFT_PARTNER_ADDRS']
         env['PATRONI_RAFT_SELF_ADDR'] = self.CONTROLLER_ADDR
         env['PATRONI_RAFT_DATA_DIR'] = self._work_directory
-        return subprocess.Popen([sys.executable, '-m', 'coverage', 'run',
-                                '--source=patroni', '-p', 'patroni_raft_controller.py'],
-                                stdout=self._log, stderr=subprocess.STDOUT, env=env)
+        return psutil.Popen([sys.executable, '-m', 'coverage', 'run',
+                             '--source=patroni', '-p', 'patroni_raft_controller.py'],
+                            stdout=self._log, stderr=subprocess.STDOUT, env=env)
 
-    def query(self, key, scope='batman'):
-        ret = self._raft.get(self.path(key, scope))
+    def query(self, key, scope='batman', group=None):
+        ret = self._raft.get(self.path(key, scope, group))
         return ret and ret['value']
 
     def set(self, key, value):
@@ -626,15 +772,18 @@ class RaftController(AbstractDcsController):
             self.start()
 
         ready_event = threading.Event()
-        self._raft = KVStoreTTL(ready_event.set, None, None, partner_addrs=[self.CONTROLLER_ADDR], password=self.PASSWORD)
+        self._raft = KVStoreTTL(ready_event.set, None, None,
+                                partner_addrs=[self.CONTROLLER_ADDR], password=self.PASSWORD)
         self._raft.startAutoTick()
         ready_event.wait()
 
 
 class PatroniPoolController(object):
 
-    BACKUP_SCRIPT = [sys.executable, 'features/backup_create.py']
-    ARCHIVE_RESTORE_SCRIPT = ' '.join((sys.executable, os.path.abspath('features/archive-restore.py')))
+    PYTHON = sys.executable.replace('\\', '/')
+    BACKUP_SCRIPT = [PYTHON, 'features/backup_create.py']
+    BACKUP_RESTORE_SCRIPT = ' '.join((PYTHON, os.path.abspath('features/backup_restore.py'))).replace('\\', '/')
+    ARCHIVE_RESTORE_SCRIPT = ' '.join((PYTHON, os.path.abspath('features/archive-restore.py')))
 
     def __init__(self, context):
         self._context = context
@@ -643,8 +792,17 @@ class PatroniPoolController(object):
         self._patroni_path = None
         self._processes = {}
         self.create_and_set_output_directory('')
+        self._check_postgres_ssl()
         self.known_dcs = {subclass.name(): subclass for subclass in AbstractDcsController.get_subclasses()}
 
+    def _check_postgres_ssl(self):
+        try:
+            subprocess.check_output(['postgres', '-D', os.devnull, '-c', 'ssl=on'], stderr=subprocess.STDOUT)
+            raise Exception  # this one should never happen because the previous line will always raise and exception
+        except Exception as e:
+            self._context.postgres_supports_ssl = isinstance(e, subprocess.CalledProcessError)\
+                    and 'SSL is not supported by this build' not in e.output.decode()
+
     @property
     def patroni_path(self):
         if self._patroni_path is None:
@@ -695,7 +853,8 @@ class PatroniPoolController(object):
             'bootstrap': {
                 'method': 'pg_basebackup',
                 'pg_basebackup': {
-                    'command': " ".join(self.BACKUP_SCRIPT) + ' --walmethod=stream --dbname=' + f.backup_source
+                    'command': " ".join(self.BACKUP_SCRIPT +
+                                        ['--walmethod=stream', '--dbname="{0}"'.format(f.backup_source)])
                 },
                 'dcs': {
                     'postgresql': {
@@ -710,7 +869,7 @@ class PatroniPoolController(object):
                     'archive_mode': 'on',
                     'archive_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode archive ' +
                                         '--dirname {} --filename %f --pathname %p').format(
-                                        os.path.join(self.patroni_path, 'data', 'wal_archive'))
+                                        os.path.join(self.patroni_path, 'data', 'wal_archive_clone').replace('\\', '/'))
                 },
                 'authentication': {
                     'superuser': {'password': 'zalando1'},
@@ -726,14 +885,14 @@ class PatroniPoolController(object):
             'bootstrap': {
                 'method': 'backup_restore',
                 'backup_restore': {
-                    'command': (sys.executable + ' features/backup_restore.py --sourcedir=' +
-                                os.path.join(self.patroni_path, 'data', 'basebackup')),
+                    'command': (self.BACKUP_RESTORE_SCRIPT + ' --sourcedir=' +
+                                os.path.join(self.patroni_path, 'data', 'basebackup').replace('\\', '/')),
                     'recovery_conf': {
                         'recovery_target_action': 'promote',
                         'recovery_target_timeline': 'latest',
                         'restore_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode restore ' +
                                             '--dirname {} --filename %f --pathname %p').format(
-                                            os.path.join(self.patroni_path, 'data', 'wal_archive'))
+                            os.path.join(self.patroni_path, 'data', 'wal_archive_clone').replace('\\', '/'))
                     }
                 }
             },
@@ -746,6 +905,25 @@ class PatroniPoolController(object):
         }
         self.start(name, custom_config=custom_config)
 
+    def bootstrap_from_backup_no_leader(self, name, cluster_name):
+        custom_config = {
+            'scope': cluster_name,
+            'postgresql': {
+                'recovery_conf': {
+                    'restore_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode restore ' +
+                                        '--dirname {} --filename %f --pathname %p').format(
+                                        os.path.join(self.patroni_path, 'data', 'wal_archive').replace('\\', '/'))
+                },
+                'create_replica_methods': ['no_leader_bootstrap'],
+                'no_leader_bootstrap': {
+                    'command': (self.BACKUP_RESTORE_SCRIPT + ' --sourcedir=' +
+                                os.path.join(self.patroni_path, 'data', 'basebackup').replace('\\', '/')),
+                    'no_leader': '1'
+                }
+            }
+        }
+        self.start(name, custom_config=custom_config)
+
     @property
     def dcs(self):
         if self._dcs is None:
@@ -872,10 +1050,32 @@ class WatchdogMonitor(object):
 
 # actions to execute on start/stop of the tests and before running individual features
 def before_all(context):
-    os.environ.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
-    context.ci = any(a in os.environ for a in ('TRAVIS_BUILD_NUMBER', 'BUILD_NUMBER', 'GITHUB_ACTIONS'))
+    context.ci = os.name == 'nt' or\
+        any(a in os.environ for a in ('TRAVIS_BUILD_NUMBER', 'BUILD_NUMBER', 'GITHUB_ACTIONS'))
     context.timeout_multiplier = 5 if context.ci else 1  # MacOS sometimes is VERY slow
     context.pctl = PatroniPoolController(context)
+
+    context.keyfile = os.path.join(context.pctl.output_dir, 'patroni.key')
+    context.certfile = os.path.join(context.pctl.output_dir, 'patroni.crt')
+    try:
+        with open(os.devnull, 'w') as null:
+            ret = subprocess.call(['openssl', 'req', '-nodes', '-new', '-x509', '-subj', '/CN=batman.patroni',
+                                   '-keyout', context.keyfile, '-out', context.certfile], stdout=null, stderr=null)
+            if ret != 0:
+                raise Exception
+    except Exception:
+        context.keyfile = context.certfile = None
+
+    os.environ.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
+    ctl = {'auth': os.environ['PATRONI_RESTAPI_USERNAME'] + ':' + os.environ['PATRONI_RESTAPI_PASSWORD']}
+    if context.certfile:
+        os.environ.update({'PATRONI_RESTAPI_CAFILE': context.certfile,
+                           'PATRONI_RESTAPI_CERTFILE': context.certfile,
+                           'PATRONI_RESTAPI_KEYFILE': context.keyfile,
+                           'PATRONI_RESTAPI_VERIFY_CLIENT': 'required',
+                           'PATRONI_CTL_INSECURE': 'on'})
+        ctl.update({'cacert': context.certfile, 'certfile': context.certfile, 'keyfile': context.keyfile})
+    context.request_executor = PatroniRequest({'ctl': ctl}, True)
     context.dcs_ctl = context.pctl.known_dcs[context.pctl.dcs](context)
     context.dcs_ctl.start()
     try:
@@ -893,13 +1093,45 @@ def after_all(context):
 
 def before_feature(context, feature):
     """ create per-feature output directory to collect Patroni and PostgreSQL logs """
+    if feature.name == 'watchdog' and os.name == 'nt':
+        return feature.skip("Watchdog isn't supported on Windows")
+    elif feature.name == 'citus':
+        lib = subprocess.check_output(['pg_config', '--pkglibdir']).decode('utf-8').strip()
+        if not os.path.exists(os.path.join(lib, 'citus.so')):
+            return feature.skip("Citus extenstion isn't available")
     context.pctl.create_and_set_output_directory(feature.name)
 
 
 def after_feature(context, feature):
-    """ stop all Patronis, remove their data directory and cleanup the keys in etcd """
+    """ send SIGCONT to a dcs if neccessary,
+    stop all Patronis remove their data directory and cleanup the keys in etcd """
+    context.dcs_ctl.stop_outage()
     context.pctl.stop_all()
-    shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data'))
+    data = os.path.join(context.pctl.patroni_path, 'data')
+    if os.path.exists(data):
+        shutil.rmtree(data)
     context.dcs_ctl.cleanup_service_tree()
-    if feature.status == 'failed':
+
+    found = False
+    logs = glob.glob(context.pctl.output_dir + '/patroni_*.log')
+    for log in logs:
+        with open(log) as f:
+            for line in f:
+                if 'please report it as a BUG' in line:
+                    print(':'.join([log, line.rstrip()]))
+                    found = True
+
+    if feature.status == 'failed' or found:
         shutil.copytree(context.pctl.output_dir, context.pctl.output_dir + '_failed')
+    if found:
+        raise Exception('Unexpected errors in Patroni log files')
+
+
+def before_scenario(context, scenario):
+    if 'slot-advance' in scenario.effective_tags:
+        for p in context.pctl._processes.values():
+            if p._conn and p._conn.server_version < 110000:
+                scenario.skip('pg_replication_slot_advance() is not supported on {0}'.format(p._conn.server_version))
+                break
+    if 'dcs-failsafe' in scenario.effective_tags and not context.dcs_ctl._handle:
+        scenario.skip('it is not possible to control state of {0} from tests'.format(context.dcs_ctl.name()))

features/ignored_slots.feature

@@ -3,7 +3,7 @@ Feature: ignored slots
     Given I start postgres1
     Then postgres1 is a leader after 10 seconds
     And there is a non empty initialize key in DCS after 15 seconds
-    When I issue a PATCH request to http://127.0.0.1:8009/config with {"loop_wait": 2, "ignore_slots": [{"name": "unmanaged_slot_0", "database": "postgres", "plugin": "test_decoding", "type": "logical"}, {"name": "unmanaged_slot_1", "database": "postgres", "plugin": "test_decoding"}, {"name": "unmanaged_slot_2", "database": "postgres"}, {"name": "unmanaged_slot_3"}], "postgresql": {"parameters": {"wal_level": "logical"}}}
+    When I issue a PATCH request to http://127.0.0.1:8009/config with {"ignore_slots": [{"name": "unmanaged_slot_0", "database": "postgres", "plugin": "test_decoding", "type": "logical"}, {"name": "unmanaged_slot_1", "database": "postgres", "plugin": "test_decoding"}, {"name": "unmanaged_slot_2", "database": "postgres"}, {"name": "unmanaged_slot_3"}], "postgresql": {"parameters": {"wal_level": "logical"}}}
     Then I receive a response code 200
     And Response on GET http://127.0.0.1:8009/config contains ignore_slots after 10 seconds
     # Make sure the wal_level has been changed.
@@ -52,7 +52,7 @@ Feature: ignored slots
     And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin
     And postgres1 does not have a logical replication slot named dummy_slot
 
-    # 3. After a failover the server (now a master) still has the slot.
+    # 3. After a failover the server (now a primary) still has the slot.
     When I shut down postgres0
     Then "members/postgres1" key in DCS has role=master after 3 seconds
     And postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin

features/patroni_api.feature

@@ -14,9 +14,9 @@ Scenario: check API requests on a stand-alone server
 	Then I receive a response code 200
 	When I issue a GET request to http://127.0.0.1:8008/replica
 	Then I receive a response code 503
-	When I run patronictl.py reinit batman postgres0 --force
-	Then I receive a response returncode 0
-	And I receive a response output "Failed: reinitialize for member postgres0, status code=503, (I am the leader, can not reinitialize)"
+	When I issue a POST request to http://127.0.0.1:8008/reinitialize with {"force": true}
+	Then I receive a response code 503
+	And I receive a response text I am the leader, can not reinitialize
 	When I run patronictl.py switchover batman --master postgres0 --force
 	Then I receive a response returncode 1
 	And I receive a response output "Error: No candidates found to switchover to"
@@ -35,13 +35,13 @@ Scenario: check local configuration reload
 	Then I receive a response code 202
 
 Scenario: check dynamic configuration change via DCS
-	Given I run patronictl.py edit-config -s 'ttl=10' -s 'loop_wait=2' -p 'max_connections=101' --force batman
+	Given I run patronictl.py edit-config -s 'ttl=10' -p 'max_connections=101' --force batman
 	Then I receive a response returncode 0
-	And I receive a response output "+loop_wait: 2"
+	And I receive a response output "+ttl: 10"
 	And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 11 seconds
 	When I issue a GET request to http://127.0.0.1:8008/config
 	Then I receive a response code 200
-	And I receive a response loop_wait 2
+	And I receive a response ttl 10
 	When I issue a GET request to http://127.0.0.1:8008/patroni
 	Then I receive a response code 200
 	And I receive a response tags {'new_tag': 'new_value'}
@@ -94,11 +94,11 @@ Scenario: check the switchover via the API in the pause mode
 	And postgres0 role is the secondary after 10 seconds
 	And replication works from postgres1 to postgres0 after 20 seconds
 	And "members/postgres0" key in DCS has state=running after 10 seconds
-	When I issue a GET request to http://127.0.0.1:8008/master
+	When I issue a GET request to http://127.0.0.1:8008/primary
 	Then I receive a response code 503
 	When I issue a GET request to http://127.0.0.1:8008/replica
 	Then I receive a response code 200
-	When I issue a GET request to http://127.0.0.1:8009/master
+	When I issue a GET request to http://127.0.0.1:8009/primary
 	Then I receive a response code 200
 	When I issue a GET request to http://127.0.0.1:8009/replica
 	Then I receive a response code 503
@@ -109,18 +109,18 @@ Scenario: check the scheduled switchover
 	And I receive a response output "Can't schedule switchover in the paused state"
 	When I run patronictl.py resume batman
 	Then I receive a response returncode 0
-	Given I issue a scheduled switchover from postgres1 to postgres0 in 5 seconds
+	Given I issue a scheduled switchover from postgres1 to postgres0 in 10 seconds
 	Then I receive a response returncode 0
 	And postgres0 is a leader after 20 seconds
 	And postgres0 role is the primary after 10 seconds
 	And postgres1 role is the secondary after 10 seconds
 	And replication works from postgres0 to postgres1 after 25 seconds
 	And "members/postgres1" key in DCS has state=running after 10 seconds
-	When I issue a GET request to http://127.0.0.1:8008/master
+	When I issue a GET request to http://127.0.0.1:8008/primary
 	Then I receive a response code 200
 	When I issue a GET request to http://127.0.0.1:8008/replica
 	Then I receive a response code 503
-	When I issue a GET request to http://127.0.0.1:8009/master
+	When I issue a GET request to http://127.0.0.1:8009/primary
 	Then I receive a response code 503
 	When I issue a GET request to http://127.0.0.1:8009/replica
 	Then I receive a response code 200

features/standby_cluster.feature

@@ -3,7 +3,7 @@ Feature: standby cluster
     Given I start postgres1
     Then postgres1 is a leader after 10 seconds
     And there is a non empty initialize key in DCS after 15 seconds
-    When I issue a PATCH request to http://127.0.0.1:8009/config with {"loop_wait": 2, "slots": {"pm_1": {"type": "physical"}}, "postgresql": {"parameters": {"wal_level": "logical"}}}
+    When I issue a PATCH request to http://127.0.0.1:8009/config with {"slots": {"pm_1": {"type": "physical"}}, "postgresql": {"parameters": {"wal_level": "logical"}}}
     Then I receive a response code 200
     And Response on GET http://127.0.0.1:8009/config contains slots after 10 seconds
     And I sleep for 3 seconds
@@ -14,7 +14,7 @@ Feature: standby cluster
     Then "members/postgres0" key in DCS has state=running after 10 seconds
     And replication works from postgres1 to postgres0 after 15 seconds
 
-  @skip
+  @slot-advance
   Scenario: check permanent logical slots are synced to the replica
     Given I run patronictl.py restart batman postgres1 --force
     Then Logical slot test_logical is in sync between postgres0 and postgres1 after 10 seconds
@@ -35,7 +35,7 @@ Feature: standby cluster
     When I add the table foo to postgres0
     Then table foo is present on postgres1 after 20 seconds
     And I sleep for 3 seconds
-    When I issue a GET request to http://127.0.0.1:8009/master
+    When I issue a GET request to http://127.0.0.1:8009/primary
     Then I receive a response code 503
     When I issue a GET request to http://127.0.0.1:8009/standby_leader
     Then I receive a response code 200
@@ -50,7 +50,7 @@ Feature: standby cluster
     When I kill postgres1
     And I kill postmaster on postgres1
     Then postgres2 is replicating from postgres0 after 32 seconds
-    When I issue a GET request to http://127.0.0.1:8010/master
+    When I issue a GET request to http://127.0.0.1:8010/primary
     Then I receive a response code 503
     And I sleep for 3 seconds
     When I issue a GET request to http://127.0.0.1:8010/standby_leader

features/steps/basic_replication.py

@@ -28,7 +28,7 @@ def stop_postgres(context, name):
 def add_table(context, table_name, pg_name):
     # parse the configuration file and get the port
     try:
-        context.pctl.query(pg_name, "CREATE TABLE {0}()".format(table_name))
+        context.pctl.query(pg_name, "CREATE TABLE public.{0}()".format(table_name))
     except pg.Error as e:
         assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
 
@@ -37,9 +37,9 @@ def add_table(context, table_name, pg_name):
 def toggle_wal_replay(context, action, pg_name):
     # pause or resume the wal replay process
     try:
-       version = context.pctl.query(pg_name, "select pg_catalog.pg_read_file('PG_VERSION', 0, 2)").fetchone()
-       wal = version and version[0] and int(version[0].split('.')[0]) < 10 and "xlog" or "wal"
-       context.pctl.query(pg_name, "SELECT pg_{0}_replay_{1}()".format(wal, action))
+        version = context.pctl.query(pg_name, "SHOW server_version_num").fetchone()[0]
+        wal_name = 'xlog' if int(version)/10000 < 10 else 'wal'
+        context.pctl.query(pg_name, "SELECT pg_{0}_replay_{1}()".format(wal_name, action))
     except pg.Error as e:
         assert False, "Error during {0} wal recovery on {1}: {2}".format(action, pg_name, e)
 
@@ -47,10 +47,10 @@ def toggle_wal_replay(context, action, pg_name):
 @step('I {action:w} table on {pg_name:w}')
 def crdr_mytest(context, action, pg_name):
     try:
-       if (action == "create"):
-           context.pctl.query(pg_name, "create table if not exists mytest(id Numeric)")
-       else:
-           context.pctl.query(pg_name, "drop table if exists mytest")
+        if (action == "create"):
+            context.pctl.query(pg_name, "create table if not exists public.mytest(id numeric)")
+        else:
+            context.pctl.query(pg_name, "drop table if exists public.mytest")
     except pg.Error as e:
         assert False, "Error {0} table mytest on {1}: {2}".format(action, pg_name, e)
 
@@ -59,7 +59,7 @@ def crdr_mytest(context, action, pg_name):
 def initiate_load(context, pg_name):
     # perform dummy load
     try:
-       context.pctl.query(pg_name, "begin; insert into mytest select r::numeric from generate_series(1, 350000) r; commit;")
+        context.pctl.query(pg_name, "insert into public.mytest select r::numeric from generate_series(1, 350000) r")
     except pg.Error as e:
         assert False, "Error loading test data on {0}: {1}".format(pg_name, e)
 
@@ -68,7 +68,7 @@ def initiate_load(context, pg_name):
 def table_is_present_on(context, table_name, pg_name, max_replication_delay):
     max_replication_delay *= context.timeout_multiplier
     for _ in range(int(max_replication_delay)):
-        if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
+        if context.pctl.query(pg_name, "SELECT 1 FROM public.{0}".format(table_name), fail_ok=True) is not None:
             break
         sleep(1)
     else:
@@ -83,10 +83,10 @@ def check_role(context, pg_name, pg_role, max_promotion_timeout):
         "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
 
 
-@step('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
-@then('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
-def replication_works(context, master, replica, time_limit):
+@step('replication works from {primary:w} to {replica:w} after {time_limit:d} seconds')
+@then('replication works from {primary:w} to {replica:w} after {time_limit:d} seconds')
+def replication_works(context, primary, replica, time_limit):
     context.execute_steps(u"""
         When I add the table test_{0} to {1}
         Then table test_{0} is present on {2} after {3} seconds
-    """.format(int(time()), master, replica, time_limit))
+    """.format(int(time()), primary, replica, time_limit))

features/steps/cascading_replication.py

@@ -11,11 +11,8 @@ def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value):
 
 @then('There is a {label} with "{content}" in {name:w} data directory')
 def check_label(context, label, content, name):
-    label = context.pctl.read_label(name, label)
-    if label is None:
-        label = ""
-    label = label.replace('\n', '\\n')
-    assert content in label, "\"{0}\" doesn't contain {1}".format(label, content)
+    value = (context.pctl.read_label(name, label) or '').replace('\n', '\\n')
+    assert content in value, "\"{0}\" in {1} doesn't contain {2}".format(value, label, content)
 
 
 @step('I create label with "{content:w}" in {name:w} data directory')

features/steps/citus.py

@@ -0,0 +1,117 @@
+import json
+import time
+
+from behave import step, then
+from dateutil import tz
+from datetime import datetime
+from functools import partial
+from threading import Thread, Event
+
+tzutc = tz.tzutc()
+
+
+@step('{name:w} is a leader in a group {group:d} after {time_limit:d} seconds')
+@then('{name:w} is a leader in a group {group:d} after {time_limit:d} seconds')
+def is_a_group_leader(context, name, group, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    while (context.dcs_ctl.query("leader", group=group) != name):
+        time.sleep(1)
+        assert time.time() < max_time, "{0} is not a leader in dcs after {1} seconds".format(name, time_limit)
+
+
+@step('"{name}" key in a group {group:d} in DCS has {key:w}={value} after {time_limit:d} seconds')
+def check_group_member(context, name, group, key, value, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    dcs_value = None
+    response = None
+    while time.time() < max_time:
+        try:
+            response = json.loads(context.dcs_ctl.query(name, group=group))
+            dcs_value = response.get(key)
+            if dcs_value == value:
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    assert False, ("{0} in a group {1} does not have {2}={3} (found {4}) in dcs" +
+                   " after {5} seconds").format(name, group, key, value, response, time_limit)
+
+
+@step('I start {name:w} in citus group {group:d}')
+def start_citus(context, name, group):
+    return context.pctl.start(name, custom_config={"citus": {"database": "postgres", "group": int(group)}})
+
+
+@step('{name1:w} is registered in the {name2:w} as the worker in group {group:d}')
+def check_registration(context, name1, name2, group):
+    worker_port = int(context.pctl.query(name1, "SHOW port").fetchone()[0])
+    r = context.pctl.query(name2, "SELECT nodeport FROM pg_catalog.pg_dist_node WHERE groupid = {0}".format(group))
+    assert worker_port == r.fetchone()[0],\
+        "Worker {0} is not registered in pg_dist_node on the coordinator {1}".format(name1, name2)
+
+
+@step('I create a distributed table on {name:w}')
+def create_distributed_table(context, name):
+    context.pctl.query(name, 'CREATE TABLE public.d(id int not null)')
+    context.pctl.query(name, "SELECT create_distributed_table('public.d', 'id')")
+
+
+@step('I cleanup a distributed table on {name:w}')
+def cleanup_distributed_table(context, name):
+    context.pctl.query(name, 'TRUNCATE public.d')
+
+
+def insert_thread(query_func, context):
+    while True:
+        if context.thread_stop_event.is_set():
+            break
+
+        context.insert_counter += 1
+        query_func('INSERT INTO public.d VALUES({0})'.format(context.insert_counter))
+
+        context.thread_stop_event.wait(0.01)
+
+
+@step('I start a thread inserting data on {name:w}')
+def start_insert_thread(context, name):
+    context.thread_stop_event = Event()
+    context.insert_counter = 0
+    query_func = partial(context.pctl.query, name)
+    thread_func = partial(insert_thread, query_func, context)
+    context.thread = Thread(target=thread_func)
+    context.thread.daemon = True
+    context.thread.start()
+
+
+@then('a thread is still alive')
+def thread_is_alive(context):
+    assert context.thread.is_alive(), "Thread is not alive"
+
+
+@step("I stop a thread")
+def stop_insert_thread(context):
+    context.thread_stop_event.set()
+    context.thread.join(1*context.timeout_multiplier)
+    assert not context.thread.is_alive(), "Thread is still alive"
+
+
+@step("a distributed table on {name:w} has expected rows")
+def count_rows(context, name):
+    rows = context.pctl.query(name, "SELECT COUNT(*) FROM public.d").fetchone()[0]
+    assert rows == context.insert_counter, "Distributed table doesn't have expected amount of rows"
+
+
+@step("There is a transaction in progress on {name:w} changing pg_dist_node")
+def check_transaction(context, name):
+    cur = context.pctl.query(name, "SELECT xact_start FROM pg_stat_activity WHERE pid <> pg_backend_pid()"
+                                   " AND state = 'idle in transaction' AND query ~ 'citus_update_node'")
+    assert cur.rowcount == 1, "There is no idle in transaction updating pg_dist_node"
+    context.xact_start = cur.fetchone()[0]
+
+
+@step("a transaction finishes in {timeout:d} seconds")
+def check_transaction_timeout(context, timeout):
+    assert (datetime.now(tzutc) - context.xact_start).seconds > timeout,\
+            "a transaction finished earlier than in {0} seconds".format(timeout)

features/steps/dcs_failsafe_mode.py

@@ -0,0 +1,16 @@
+from behave import step
+
+
+@step('DCS is down')
+def start_dcs_outage(context):
+    context.dcs_ctl.start_outage()
+
+
+@step('DCS is up')
+def stop_dcs_outage(context):
+    context.dcs_ctl.stop_outage()
+
+
+@step('I start {name:w} in a cluster {cluster_name:w} from backup with no_leader')
+def start_cluster_from_backup_no_leader(context, name, cluster_name):
+    context.pctl.bootstrap_from_backup_no_leader(name, cluster_name)

features/steps/patroni_api.py

@@ -1,5 +1,4 @@
 import json
-import os
 import parse
 import shlex
 import subprocess
@@ -10,10 +9,8 @@ import yaml
 from behave import register_type, step, then
 from dateutil import tz
 from datetime import datetime, timedelta
-from patroni.request import PatroniRequest
 
 tzutc = tz.tzutc()
-request_executor = PatroniRequest({'ctl': {'auth': 'username:password'}})
 
 
 @parse.with_pattern(r'https?://(?:\w|\.|:|/)+')
@@ -73,11 +70,13 @@ def do_post_empty(context, url):
 
 @step('I issue a {request_method:w} request to {url:url} with {data}')
 def do_request(context, request_method, url, data):
+    if context.certfile:
+        url = url.replace('http://', 'https://')
     data = data and json.loads(data)
     try:
-        r = request_executor.request(request_method, url, data)
+        r = context.request_executor.request(request_method, url, data)
         if request_method == 'PATCH' and r.status == 409:
-            r = request_executor.request(request_method, url, data)
+            r = context.request_executor.request(request_method, url, data)
     except Exception:
         context.status_code = context.response = None
     else:
@@ -88,10 +87,7 @@ def do_request(context, request_method, url, data):
 def do_run(context, cmd):
     cmd = [sys.executable, '-m', 'coverage', 'run', '--source=patroni', '-p'] + shlex.split(cmd)
     try:
-        # XXX: Dirty hack! We need to take name/passwd from the config!
-        env = os.environ.copy()
-        env.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
-        response = subprocess.check_output(cmd, stderr=subprocess.STDOUT, env=env)
+        response = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         context.status_code = 0
     except subprocess.CalledProcessError as e:
         response = e.output
@@ -113,6 +109,8 @@ def check_response(context, component, data):
         assert data.strip('"') in context.response, "response {0} does not contain {1}".format(context.response, data)
     else:
         assert component in context.response, "{0} is not part of the response".format(component)
+        if context.certfile:
+            data = data.replace('http://', 'https://')
         assert str(context.response[component]) == str(data), "{0} does not contain {1}".format(component, data)
 
 
@@ -135,11 +133,27 @@ def add_tag_to_config(context, tag, value, pg_name):
     context.pctl.add_tag_to_config(pg_name, tag, value)
 
 
-@then('Response on GET {url} contains {value} after {timeout:d} seconds')
-def check_http_response(context, url, value, timeout, negate=False):
+@then('Status code on GET {url:url} is {code:d} after {timeout:d} seconds')
+def check_http_code(context, url, code, timeout):
+    if context.certfile:
+        url = url.replace('http://', 'https://')
     timeout *= context.timeout_multiplier
     for _ in range(int(timeout)):
-        r = request_executor.request('GET', url)
+        r = context.request_executor.request('GET', url)
+        if int(code) == int(r.status):
+            break
+        time.sleep(1)
+    else:
+        assert False, "HTTP Status Code is not {0} after {1} seconds".format(code, timeout)
+
+
+@then('Response on GET {url:url} contains {value} after {timeout:d} seconds')
+def check_http_response(context, url, value, timeout, negate=False):
+    if context.certfile:
+        url = url.replace('http://', 'https://')
+    timeout *= context.timeout_multiplier
+    for _ in range(int(timeout)):
+        r = context.request_executor.request('GET', url)
         if (value in r.data.decode('utf-8')) != negate:
             break
         time.sleep(1)

features/steps/standby_cluster.py

@@ -1,17 +1,12 @@
 import os
-import sys
 import time
 
 from behave import step
 
 
-select_replication_query = """
-SELECT * FROM pg_catalog.pg_stat_replication
-WHERE application_name = '{0}'
-"""
-
-executable = sys.executable if os.name != 'nt' else sys.executable.replace('\\', '/')
-callback = executable + " features/callback2.py "
+def callbacks(context, name):
+    return {c: '{0} features/callback2.py {1}'.format(context.pctl.PYTHON, name)
+            for c in ('on_start', 'on_stop', 'on_restart', 'on_role_change')}
 
 
 @step('I start {name:w} in a cluster {cluster_name:w}')
@@ -19,10 +14,10 @@ def start_patroni(context, name, cluster_name):
     return context.pctl.start(name, custom_config={
         "scope": cluster_name,
         "postgresql": {
-            "callbacks": {c: callback + name for c in ('on_start', 'on_stop', 'on_restart', 'on_role_change')},
+            "callbacks": callbacks(context, name),
             "backup_restore": {
-                "command": (executable + " features/backup_restore.py --sourcedir=" +
-                            os.path.join(context.pctl.patroni_path, 'data', 'basebackup'))}
+                "command": (context.pctl.PYTHON + " features/backup_restore.py --sourcedir=" +
+                            os.path.join(context.pctl.patroni_path, 'data', 'basebackup').replace('\\', '/'))}
         }
     })
 
@@ -49,7 +44,7 @@ def start_patroni_standby_cluster(context, name, cluster_name, name2):
             }
         },
         "postgresql": {
-            "callbacks": {c: callback + name for c in ('on_start', 'on_stop', 'on_restart', 'on_role_change')}
+            "callbacks": callbacks(context, name)
         }
     })
     return context.pctl.start(name)
@@ -62,7 +57,7 @@ def check_replication_status(context, pg_name1, pg_name2, timeout):
     while time.time() < bound_time:
         cur = context.pctl.query(
             pg_name2,
-            select_replication_query.format(pg_name1),
+            "SELECT * FROM pg_catalog.pg_stat_replication WHERE application_name = '{0}'".format(pg_name1),
             fail_ok=True
         )
 

features/steps/watchdog.py

@@ -15,7 +15,7 @@ def polling_loop(timeout, interval=1):
 
 @step('I start {name:w} with watchdog')
 def start_patroni_with_watchdog(context, name):
-    return context.pctl.start(name, custom_config={'watchdog': True})
+    return context.pctl.start(name, custom_config={'watchdog': True, 'bootstrap': {'dcs': {'ttl': 20}}})
 
 
 @step('{name:w} watchdog has been pinged after {timeout:d} seconds')
@@ -31,6 +31,11 @@ def watchdog_was_closed(context, name):
     assert context.pctl.get_watchdog(name).was_closed
 
 
+@step('{name:w} watchdog has a {timeout:d} second timeout')
+def watchdog_has_timeout(context, name, timeout):
+    assert context.pctl.get_watchdog(name).timeout == timeout
+
+
 @step('I reset {name:w} watchdog state')
 def watchdog_reset_pinged(context, name):
     context.pctl.get_watchdog(name).reset()

features/watchdog.feature

@@ -6,6 +6,14 @@ Feature: watchdog
     Then postgres0 is a leader after 10 seconds
     And postgres0 role is the primary after 10 seconds
     And postgres0 watchdog has been pinged after 10 seconds
+    And postgres0 watchdog has a 15 second timeout
+
+  Scenario: watchdog is reconfigured after global ttl changed
+    Given I run patronictl.py edit-config batman -s ttl=30 --force
+    Then I receive a response returncode 0
+    And I receive a response output "+ttl: 30"
+    When I sleep for 4 seconds
+    Then postgres0 watchdog has a 25 second timeout
 
   Scenario: watchdog is disabled during pause
     Given I run patronictl.py pause batman

kubernetes/Dockerfile

@@ -1,10 +1,9 @@
-FROM postgres:11
-MAINTAINER Alexander Kukushkin <alexander.kukushkin@zalando.de>
+FROM postgres:15
+LABEL maintainer="Alexander Kukushkin <akukushkin@microsoft.com>"
 
 RUN export DEBIAN_FRONTEND=noninteractive \
     && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
     && apt-get update -y \
-    && apt-get upgrade -y \
     && apt-cache depends patroni | sed -n -e 's/.* Depends: \(python3-.\+\)$/\1/p' \
             | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
             | xargs apt-get install -y vim-tiny curl jq locales git python3-pip python3-wheel \
@@ -25,7 +24,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/* /root/.cache
 
-ADD entrypoint.sh /
+COPY entrypoint.sh /
 
 EXPOSE 5432 8008
 ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 EDITOR=/usr/bin/editor

kubernetes/Dockerfile.citus

@@ -0,0 +1,42 @@
+FROM postgres:15
+LABEL maintainer="Alexander Kukushkin <[email protected]>"
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && apt-get update -y \
+    && apt-get upgrade -y \
+    && apt-cache depends patroni | sed -n -e 's/.* Depends: \(python3-.\+\)$/\1/p' \
+            | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
+            | xargs apt-get install -y busybox vim-tiny curl jq less locales git python3-pip python3-wheel \
+    ## Make sure we have a en_US.UTF-8 locale available
+    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
+    && curl https://install.citusdata.com/community/deb.sh | bash \
+    && apt-get -y install postgresql-15-citus-11.2 \
+    && pip3 install setuptools \
+    && pip3 install 'git+https://github.com/zalando/patroni.git#egg=patroni[kubernetes]' \
+    && PGHOME=/home/postgres \
+    && mkdir -p $PGHOME \
+    && chown postgres $PGHOME \
+    && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \
+    && /bin/busybox --install -s \
+    # Set permissions for OpenShift
+    && chmod 775 $PGHOME \
+    && chmod 664 /etc/passwd \
+    # Clean up
+    && apt-get remove -y git python3-pip python3-wheel \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* /root/.cache
+
+ADD entrypoint.sh /
+ENV PGSSLMODE=verify-ca PGSSLKEY=/etc/ssl/private/ssl-cert-snakeoil.key PGSSLCERT=/etc/ssl/certs/ssl-cert-snakeoil.pem PGSSLROOTCERT=/etc/ssl/certs/ssl-cert-snakeoil.pem
+
+RUN sed -i 's/^postgresql:/&\n  basebackup:\n    checkpoint: fast/' /entrypoint.sh \
+    && sed -i "s|^    postgresql:|&\n      pg_hba:\n      - local all all trust\n      - hostssl replication all all md5 clientcert=$PGSSLMODE\n      - hostssl all all all md5 clientcert=$PGSSLMODE\n      parameters:\n        max_connections: 100\n        shared_buffers: 16MB\n        ssl: 'on'\n        ssl_ca_file: $PGSSLROOTCERT\n        ssl_cert_file: $PGSSLCERT\n        ssl_key_file: $PGSSLKEY\n        citus.node_conninfo: 'sslrootcert=$PGSSLROOTCERT sslkey=$PGSSLKEY sslcert=$PGSSLCERT sslmode=$PGSSLMODE'|"  /entrypoint.sh \
+    && sed -i "s#^    \(superuser\|replication\):#&\n      sslmode: $PGSSLMODE\n      sslkey: $PGSSLKEY\n      sslcert: $PGSSLCERT\n      sslrootcert: $PGSSLROOTCERT#" /entrypoint.sh
+
+EXPOSE 5432 8008
+ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 EDITOR=/usr/bin/editor
+USER postgres
+WORKDIR /home/postgres
+CMD ["/bin/bash", "/entrypoint.sh"]

kubernetes/README.md

@@ -0,0 +1,154 @@
+# Kubernetes deployment examples
+Below you will find examples of Patroni deployments using [kind](https://kind.sigs.k8s.io/).
+
+# Patroni on K8s
+The Patroni cluster deployment with a StatefulSet consisting of three Pods.
+
+Example session:
+
+    $ kind create cluster
+    Creating cluster "kind" ...
+     ✓ Ensuring node image (kindest/node:v1.25.3) 🖼
+     ✓ Preparing nodes 📦
+     ✓ Writing configuration 📜
+     ✓ Starting control-plane 🕹️
+     ✓ Installing CNI 🔌
+     ✓ Installing StorageClass 💾
+    Set kubectl context to "kind-kind"
+    You can now use your cluster with:
+
+    kubectl cluster-info --context kind-kind
+
+    Thanks for using kind! 😊
+
+    $ docker build -t patroni .
+    Sending build context to Docker daemon  138.8kB
+    Step 1/9 : FROM postgres:15
+    ...
+    Successfully built e9bfe69c5d2b
+    Successfully tagged patroni:latest
+
+    $ kind load docker-image patroni
+    Image: "" with ID "sha256:e9bfe69c5d2b319dec0cf564fb895484537664775e18f37f9b707914cc5537e6" not yet present on node "kind-control-plane", loading...
+
+    $ kubectl apply -f patroni_k8s.yaml
+    service/patronidemo-config created
+    statefulset.apps/patronidemo created
+    endpoints/patronidemo created
+    service/patronidemo created
+    service/patronidemo-repl created
+    secret/patronidemo created
+    serviceaccount/patronidemo created
+    role.rbac.authorization.k8s.io/patronidemo created
+    rolebinding.rbac.authorization.k8s.io/patronidemo created
+    clusterrole.rbac.authorization.k8s.io/patroni-k8s-ep-access created
+    clusterrolebinding.rbac.authorization.k8s.io/patroni-k8s-ep-access created
+
+    $ kubectl get pods -L role
+    NAME            READY   STATUS    RESTARTS   AGE   ROLE
+    patronidemo-0   1/1     Running   0          34s   master
+    patronidemo-1   1/1     Running   0          30s   replica
+    patronidemo-2   1/1     Running   0          26s   replica
+
+    $ kubectl exec -ti patronidemo-0 -- bash
+    postgres@patronidemo-0:~$ patronictl list
+    + Cluster: patronidemo (7186662553319358497) ----+----+-----------+
+    | Member        | Host       | Role    | State   | TL | Lag in MB |
+    +---------------+------------+---------+---------+----+-----------+
+    | patronidemo-0 | 10.244.0.5 | Leader  | running |  1 |           |
+    | patronidemo-1 | 10.244.0.6 | Replica | running |  1 |         0 |
+    | patronidemo-2 | 10.244.0.7 | Replica | running |  1 |         0 |
+    +---------------+------------+---------+---------+----+-----------+
+
+# Citus on K8s
+The Citus cluster with the StatefulSets, one coordinator with three Pods and two workers with two pods each.
+
+Example session:
+
+    $ kind create cluster
+    Creating cluster "kind" ...
+     ✓ Ensuring node image (kindest/node:v1.25.3) 🖼
+     ✓ Preparing nodes 📦
+     ✓ Writing configuration 📜
+     ✓ Starting control-plane 🕹️
+     ✓ Installing CNI 🔌
+     ✓ Installing StorageClass 💾
+    Set kubectl context to "kind-kind"
+    You can now use your cluster with:
+
+    kubectl cluster-info --context kind-kind
+
+    Thanks for using kind! 😊
+
+    demo@localhost:~/git/patroni/kubernetes$ docker build -f Dockerfile.citus -t patroni-citus-k8s .
+    Sending build context to Docker daemon  138.8kB
+    Step 1/11 : FROM postgres:15
+    ...
+    Successfully built 8cd73e325028
+    Successfully tagged patroni-citus-k8s:latest
+
+    $ kind load docker-image patroni-citus-k8s
+    Image: "" with ID "sha256:8cd73e325028d7147672494965e53453f5540400928caac0305015eb2c7027c7" not yet present on node "kind-control-plane", loading...
+
+    $ kubectl apply -f citus_k8s.yaml
+    service/citusdemo-0-config created
+    service/citusdemo-1-config created
+    service/citusdemo-2-config created
+    statefulset.apps/citusdemo-0 created
+    statefulset.apps/citusdemo-1 created
+    statefulset.apps/citusdemo-2 created
+    endpoints/citusdemo-0 created
+    service/citusdemo-0 created
+    endpoints/citusdemo-1 created
+    service/citusdemo-1 created
+    endpoints/citusdemo-2 created
+    service/citusdemo-2 created
+    service/citusdemo-workers created
+    secret/citusdemo created
+    serviceaccount/citusdemo created
+    role.rbac.authorization.k8s.io/citusdemo created
+    rolebinding.rbac.authorization.k8s.io/citusdemo created
+    clusterrole.rbac.authorization.k8s.io/patroni-k8s-ep-access created
+    clusterrolebinding.rbac.authorization.k8s.io/patroni-k8s-ep-access created
+
+    $ kubectl get sts
+    NAME          READY   AGE
+    citusdemo-0   1/3     6s  # coodinator (group=0)
+    citusdemo-1   1/2     6s  # worker (group=1)
+    citusdemo-2   1/2     6s  # worker (group=2)
+
+    $ kubectl get pods -l cluster-name=citusdemo -L role
+    NAME            READY   STATUS    RESTARTS   AGE    ROLE
+    citusdemo-0-0   1/1     Running   0          105s   master
+    citusdemo-0-1   1/1     Running   0          101s   replica
+    citusdemo-0-2   1/1     Running   0          96s    replica
+    citusdemo-1-0   1/1     Running   0          105s   master
+    citusdemo-1-1   1/1     Running   0          101s   replica
+    citusdemo-2-0   1/1     Running   0          105s   master
+    citusdemo-2-1   1/1     Running   0          101s   replica
+
+    $ kubectl exec -ti citusdemo-0-0 -- bash
+    postgres@citusdemo-0-0:~$ patronictl list
+    + Citus cluster: citusdemo -----------+--------------+---------+----+-----------+
+    | Group | Member        | Host        | Role         | State   | TL | Lag in MB |
+    +-------+---------------+-------------+--------------+---------+----+-----------+
+    |     0 | citusdemo-0-0 | 10.244.0.10 | Leader       | running |  1 |           |
+    |     0 | citusdemo-0-1 | 10.244.0.12 | Replica      | running |  1 |         0 |
+    |     0 | citusdemo-0-2 | 10.244.0.14 | Sync Standby | running |  1 |         0 |
+    |     1 | citusdemo-1-0 | 10.244.0.8  | Leader       | running |  1 |           |
+    |     1 | citusdemo-1-1 | 10.244.0.11 | Sync Standby | running |  1 |         0 |
+    |     2 | citusdemo-2-0 | 10.244.0.9  | Leader       | running |  1 |           |
+    |     2 | citusdemo-2-1 | 10.244.0.13 | Sync Standby | running |  1 |         0 |
+    +-------+---------------+-------------+--------------+---------+----+-----------+
+
+    postgres@citusdemo-0-0:~$ psql citus
+    psql (15.1 (Debian 15.1-1.pgdg110+1))
+    Type "help" for help.
+
+    citus=# table pg_dist_node;
+     nodeid | groupid |  nodename   | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards
+    --------+---------+-------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------
+          1 |       0 | 10.244.0.10 |     5432 | default  | t           | t        | primary  | default     | t              | f
+          2 |       1 | 10.244.0.8  |     5432 | default  | t           | t        | primary  | default     | t              | t
+          3 |       2 | 10.244.0.9  |     5432 | default  | t           | t        | primary  | default     | t              | t
+    (3 rows)

kubernetes/citus_k8s.yaml

@@ -0,0 +1,590 @@
+# headless services to avoid deletion of citusdemo-*-config endpoints
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-0-config
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '0'
+spec:
+  clusterIP: None
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-1-config
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '1'
+spec:
+  clusterIP: None
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-2-config
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '2'
+spec:
+  clusterIP: None
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: &cluster_name citusdemo-0
+  labels: &labels
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '0'
+    citus-type: coordinator
+spec:
+  replicas: 3
+  serviceName: *cluster_name
+  selector:
+    matchLabels:
+      <<: *labels
+  template:
+    metadata:
+      labels:
+        <<: *labels
+    spec:
+      serviceAccountName: citusdemo
+      containers:
+      - name: *cluster_name
+        image: patroni-citus-k8s  # docker build -f Dockerfile.citus -t patroni-citus-k8s .
+        imagePullPolicy: IfNotPresent
+        readinessProbe:
+          httpGet:
+            scheme: HTTP
+            path: /readiness
+            port: 8008
+          initialDelaySeconds: 3
+          periodSeconds: 10
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 3
+        ports:
+        - containerPort: 8008
+          protocol: TCP
+        - containerPort: 5432
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /home/postgres/pgdata
+          name: pgdata
+        env:
+        - name: PATRONI_KUBERNETES_POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: PATRONI_KUBERNETES_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: PATRONI_KUBERNETES_BYPASS_API_SERVICE
+          value: 'true'
+        - name: PATRONI_KUBERNETES_USE_ENDPOINTS
+          value: 'true'
+        - name: PATRONI_KUBERNETES_LABELS
+          value: '{application: patroni, cluster-name: citusdemo}'
+        - name: PATRONI_CITUS_DATABASE
+          value: citus
+        - name: PATRONI_CITUS_GROUP
+          value: '0'
+        - name: PATRONI_SUPERUSER_USERNAME
+          value: postgres
+        - name: PATRONI_SUPERUSER_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: citusdemo
+              key: superuser-password
+        - name: PATRONI_REPLICATION_USERNAME
+          value: standby
+        - name: PATRONI_REPLICATION_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: citusdemo
+              key: replication-password
+        - name: PATRONI_SCOPE
+          value: citusdemo
+        - name: PATRONI_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: PATRONI_POSTGRESQL_DATA_DIR
+          value: /home/postgres/pgdata/pgroot/data
+        - name: PATRONI_POSTGRESQL_PGPASS
+          value: /tmp/pgpass
+        - name: PATRONI_POSTGRESQL_LISTEN
+          value: '0.0.0.0:5432'
+        - name: PATRONI_RESTAPI_LISTEN
+          value: '0.0.0.0:8008'
+      terminationGracePeriodSeconds: 0
+      volumes:
+      - name: pgdata
+        emptyDir: {}
+#  volumeClaimTemplates:
+#  - metadata:
+#      labels:
+#        application: spilo
+#        spilo-cluster: *cluster_name
+#      annotations:
+#        volume.alpha.kubernetes.io/storage-class: anything
+#      name: pgdata
+#    spec:
+#      accessModes:
+#      - ReadWriteOnce
+#      resources:
+#        requests:
+#          storage: 5Gi
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: &cluster_name citusdemo-1
+  labels: &labels
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '1'
+    citus-type: worker
+spec:
+  replicas: 2
+  serviceName: *cluster_name
+  selector:
+    matchLabels:
+      <<: *labels
+  template:
+    metadata:
+      labels:
+        <<: *labels
+    spec:
+      serviceAccountName: citusdemo
+      containers:
+      - name: *cluster_name
+        image: patroni-citus-k8s  # docker build -f Dockerfile.citus -t patroni-citus-k8s .
+        imagePullPolicy: IfNotPresent
+        readinessProbe:
+          httpGet:
+            scheme: HTTP
+            path: /readiness
+            port: 8008
+          initialDelaySeconds: 3
+          periodSeconds: 10
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 3
+        ports:
+        - containerPort: 8008
+          protocol: TCP
+        - containerPort: 5432
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /home/postgres/pgdata
+          name: pgdata
+        env:
+        - name: PATRONI_KUBERNETES_POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: PATRONI_KUBERNETES_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: PATRONI_KUBERNETES_BYPASS_API_SERVICE
+          value: 'true'
+        - name: PATRONI_KUBERNETES_USE_ENDPOINTS
+          value: 'true'
+        - name: PATRONI_KUBERNETES_LABELS
+          value: '{application: patroni, cluster-name: citusdemo}'
+        - name: PATRONI_CITUS_DATABASE
+          value: citus
+        - name: PATRONI_CITUS_GROUP
+          value: '1'
+        - name: PATRONI_SUPERUSER_USERNAME
+          value: postgres
+        - name: PATRONI_SUPERUSER_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: citusdemo
+              key: superuser-password
+        - name: PATRONI_REPLICATION_USERNAME
+          value: standby
+        - name: PATRONI_REPLICATION_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: citusdemo
+              key: replication-password
+        - name: PATRONI_SCOPE
+          value: citusdemo
+        - name: PATRONI_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: PATRONI_POSTGRESQL_DATA_DIR
+          value: /home/postgres/pgdata/pgroot/data
+        - name: PATRONI_POSTGRESQL_PGPASS
+          value: /tmp/pgpass
+        - name: PATRONI_POSTGRESQL_LISTEN
+          value: '0.0.0.0:5432'
+        - name: PATRONI_RESTAPI_LISTEN
+          value: '0.0.0.0:8008'
+      terminationGracePeriodSeconds: 0
+      volumes:
+      - name: pgdata
+        emptyDir: {}
+#  volumeClaimTemplates:
+#  - metadata:
+#      labels:
+#        application: spilo
+#        spilo-cluster: *cluster_name
+#      annotations:
+#        volume.alpha.kubernetes.io/storage-class: anything
+#      name: pgdata
+#    spec:
+#      accessModes:
+#      - ReadWriteOnce
+#      resources:
+#        requests:
+#          storage: 5Gi
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: &cluster_name citusdemo-2
+  labels: &labels
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '2'
+    citus-type: worker
+spec:
+  replicas: 2
+  serviceName: *cluster_name
+  selector:
+    matchLabels:
+      <<: *labels
+  template:
+    metadata:
+      labels:
+        <<: *labels
+    spec:
+      serviceAccountName: citusdemo
+      containers:
+      - name: *cluster_name
+        image: patroni-citus-k8s  # docker build -f Dockerfile.citus -t patroni-citus-k8s .
+        imagePullPolicy: IfNotPresent
+        readinessProbe:
+          httpGet:
+            scheme: HTTP
+            path: /readiness
+            port: 8008
+          initialDelaySeconds: 3
+          periodSeconds: 10
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 3
+        ports:
+        - containerPort: 8008
+          protocol: TCP
+        - containerPort: 5432
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /home/postgres/pgdata
+          name: pgdata
+        env:
+        - name: PATRONI_KUBERNETES_POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: PATRONI_KUBERNETES_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: PATRONI_KUBERNETES_BYPASS_API_SERVICE
+          value: 'true'
+        - name: PATRONI_KUBERNETES_USE_ENDPOINTS
+          value: 'true'
+        - name: PATRONI_KUBERNETES_LABELS
+          value: '{application: patroni, cluster-name: citusdemo}'
+        - name: PATRONI_CITUS_DATABASE
+          value: citus
+        - name: PATRONI_CITUS_GROUP
+          value: '2'
+        - name: PATRONI_SUPERUSER_USERNAME
+          value: postgres
+        - name: PATRONI_SUPERUSER_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: citusdemo
+              key: superuser-password
+        - name: PATRONI_REPLICATION_USERNAME
+          value: standby
+        - name: PATRONI_REPLICATION_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: citusdemo
+              key: replication-password
+        - name: PATRONI_SCOPE
+          value: citusdemo
+        - name: PATRONI_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: PATRONI_POSTGRESQL_DATA_DIR
+          value: /home/postgres/pgdata/pgroot/data
+        - name: PATRONI_POSTGRESQL_PGPASS
+          value: /tmp/pgpass
+        - name: PATRONI_POSTGRESQL_LISTEN
+          value: '0.0.0.0:5432'
+        - name: PATRONI_RESTAPI_LISTEN
+          value: '0.0.0.0:8008'
+      terminationGracePeriodSeconds: 0
+      volumes:
+      - name: pgdata
+        emptyDir: {}
+#  volumeClaimTemplates:
+#  - metadata:
+#      labels:
+#        application: spilo
+#        spilo-cluster: *cluster_name
+#      annotations:
+#        volume.alpha.kubernetes.io/storage-class: anything
+#      name: pgdata
+#    spec:
+#      accessModes:
+#      - ReadWriteOnce
+#      resources:
+#        requests:
+#          storage: 5Gi
+          
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: citusdemo-0
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '0'
+    citus-type: coordinator
+subsets: []
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-0
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '0'
+    citus-type: coordinator
+spec:
+  type: ClusterIP
+  ports:
+  - port: 5432
+    targetPort: 5432
+
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: citusdemo-1
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '1'
+    citus-type: worker
+subsets: []
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-1
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '1'
+    citus-type: worker
+spec:
+  type: ClusterIP
+  ports:
+  - port: 5432
+    targetPort: 5432
+
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: citusdemo-2
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '2'
+    citus-type: worker
+subsets: []
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-2
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '2'
+    citus-type: worker
+spec:
+  type: ClusterIP
+  ports:
+  - port: 5432
+    targetPort: 5432
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-workers
+  labels: &labels
+    application: patroni
+    cluster-name: citusdemo
+    citus-type: worker
+    role: master
+spec:
+  type: ClusterIP
+  selector:
+    <<: *labels
+  ports:
+  - port: 5432
+    targetPort: 5432
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: &cluster_name citusdemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+type: Opaque
+data:
+  superuser-password: emFsYW5kbw==
+  replication-password: cmVwLXBhc3M=
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: citusdemo
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: citusdemo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
+  # delete and deletecollection are required only for 'patronictl remove'
+  - delete
+  - deletecollection
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - patch
+  - update
+  # the following three privileges are necessary only when using endpoints
+  - create
+  - list
+  - watch
+  # delete and deletecollection are required only for for 'patronictl remove'
+  - delete
+  - deletecollection
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+# The following privilege is only necessary for creation of headless service
+# for citusdemo-config endpoint, in order to prevent cleaning it up by the
+# k8s master. You can avoid giving this privilege by explicitly creating the
+# service like it is done in this manifest (lines 2..10)
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: citusdemo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: citusdemo
+subjects:
+- kind: ServiceAccount
+  name: citusdemo
+
+# Following privileges are only required if deployed not in the "default"
+# namespace and you want Patroni to bypass kubernetes service
+# (PATRONI_KUBERNETES_BYPASS_API_SERVICE=true)
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: patroni-k8s-ep-access
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  resourceNames:
+  - kubernetes
+  verbs:
+  - get
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: patroni-k8s-ep-access
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: patroni-k8s-ep-access
+subjects:
+- kind: ServiceAccount
+  name: citusdemo
+# The namespace must be specified explicitly.
+# If deploying to the different namespace you have to change it.
+  namespace: default

kubernetes/openshift-example/README.md

@@ -1,5 +1,5 @@
 # Patroni OpenShift Configuration
-Patroni can be run in OpenShift. Based on the kubernetes configuration, the Dockerfile and Entrypoint has been modified to support the dynamic UID/GID configuration that is applied in OpenShift. This can be run under the standard `restricted` SCC. 
+Patroni can be run in OpenShift. Based on the kubernetes configuration, the Dockerfile and Entrypoint has been modified to support the dynamic UID/GID configuration that is applied in OpenShift. This can be run under the standard `restricted` SCC.
 
 # Examples
 
@@ -11,39 +11,39 @@ oc new-project patroni-test
 
 ## Build the image
 
-Note: Update the references when merged upstream. 
-Note: If deploying as a template for multiple users, the following commands should be performed in a shared namespace like `openshift`. 
+Note: Update the references when merged upstream.
+Note: If deploying as a template for multiple users, the following commands should be performed in a shared namespace like `openshift`.
 
 ```
 oc import-image postgres:10 --confirm -n openshift
 oc new-build https://github.com/zalando/patroni --context-dir=kubernetes -n openshift
 ```
 
-## Deploy the Image 
-Two configuration templates exist in [templates](templates) directory: 
-- Patroni Ephemeral
-- Patroni Persistent
+## Deploy the Image
+Two configuration templates exist in [templates](templates) directory:
+-   Patroni Ephemeral
+-   Patroni Persistent
 
-The only difference is whether or not the statefulset requests persistent storage. 
+The only difference is whether or not the statefulset requests persistent storage.
 
 ## Create the Template
-Install the template into the `openshift` namespace if this should be shared across projects: 
+Install the template into the `openshift` namespace if this should be shared across projects:
 
 ```
 oc create -f templates/template_patroni_ephemeral.yml -n openshift
 ```
 
-Then, from your own project: 
+Then, from your own project:
 
 ```
 oc new-app patroni-pgsql-ephemeral
 ```
 
-Once the pods are running, two configmaps should be available: 
+Once the pods are running, two configmaps should be available:
 
 ```
 $ oc get configmap
 NAME                DATA      AGE
 patroniocp-config   0         1m
 patroniocp-leader   0         1m
-```
+```

kubernetes/openshift-example/test/README.md

@@ -1,2 +1,2 @@
 # Jenkins Test
-This pipeline test will create a separate deployment config for a pgbench pod and execute a test against the patroni cluster. This is a sample and should be customized. 
+This pipeline test will create a separate deployment config for a pgbench pod and execute a test against the patroni cluster. This is a sample and should be customized.

mkbinary.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 set -e
 
-pip install --ignore-installed setuptools==19.2 pyinstaller
-pyinstaller --clean --onefile patroni.spec
+pip install --ignore-installed pyinstaller
+pyinstaller --clean patroni.spec

patroni.spec

@@ -8,7 +8,7 @@ def hiddenimports():
     sys.path.insert(0, '.')
     try:
         import patroni.dcs
-        return patroni.dcs.dcs_modules()
+        return patroni.dcs.dcs_modules() + ['http.server']
     finally:
         sys.path.pop(0)
 

patroni/__main__.py

@@ -3,7 +3,7 @@ import os
 import signal
 import time
 
-from .daemon import AbstractPatroniDaemon, abstract_main
+from patroni.daemon import AbstractPatroniDaemon, abstract_main
 
 logger = logging.getLogger(__name__)
 
@@ -11,13 +11,13 @@ logger = logging.getLogger(__name__)
 class Patroni(AbstractPatroniDaemon):
 
     def __init__(self, config):
-        from .api import RestApiServer
-        from .dcs import get_dcs
-        from .ha import Ha
-        from .postgresql import Postgresql
-        from .request import PatroniRequest
-        from .version import __version__
-        from .watchdog import Watchdog
+        from patroni.api import RestApiServer
+        from patroni.dcs import get_dcs
+        from patroni.ha import Ha
+        from patroni.postgresql import Postgresql
+        from patroni.request import PatroniRequest
+        from patroni.version import __version__
+        from patroni.watchdog import Watchdog
 
         super(Patroni, self).__init__(config)
 
@@ -47,6 +47,7 @@ class Patroni(AbstractPatroniDaemon):
                 elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
                     if self.config.set_dynamic_configuration(self.config['bootstrap']['dcs']):
                         self.dcs.reload_config(self.config)
+                        self.watchdog.reload_config(self.config)
                 break
             except DCSError:
                 logger.warning('Can not get cluster from dcs')
@@ -137,7 +138,7 @@ def patroni_main():
 
 def main():
     if os.getpid() != 1:
-        from . import check_psycopg
+        from patroni import check_psycopg
 
         check_psycopg()
         return patroni_main()

patroni/api.py

@@ -7,15 +7,14 @@ import traceback
 import dateutil.parser
 import datetime
 import os
-import six
 import socket
 import sys
 
-from ipaddress import ip_address, ip_network as _ip_network
-from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
-from six.moves.socketserver import ThreadingMixIn
-from six.moves.urllib_parse import urlparse, parse_qs
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from ipaddress import ip_address, ip_network
+from socketserver import ThreadingMixIn
 from threading import Thread
+from urllib.parse import urlparse, parse_qs
 
 from . import psycopg
 from .exceptions import PostgresConnectionException, PostgresException
@@ -26,10 +25,6 @@ from .utils import deep_compare, enable_keepalive, parse_bool, patch_config, Ret
 logger = logging.getLogger(__name__)
 
 
-def ip_network(value):
-    return _ip_network(value.decode('utf-8') if six.PY2 else value, False)
-
-
 class RestApiHandler(BaseHTTPRequestHandler):
 
     def _write_status_code_only(self, status_code):
@@ -38,6 +33,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         self.log_request(status_code)
 
     def _write_response(self, status_code, body, content_type='text/html', headers=None):
+        # TODO: try-catch ConnectionResetError: [Errno 104] Connection reset by peer and log it in DEBUG level
         self.send_response(status_code)
         headers = headers or {}
         if content_type:
@@ -96,7 +92,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_GET(self, write_status_code_only=False):
         """Default method for processing all GET requests which can not be routed to other methods"""
 
-        path = '/master' if self.path == '/' else self.path
+        path = '/primary' if self.path == '/' else self.path
         response = self.get_postgresql_status()
 
         patroni = self.server.patroni
@@ -113,8 +109,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response.get('role') == 'replica' and response.get('state') == 'running' else 503
 
         if not cluster and patroni.ha.is_paused():
-            leader_status_code = 200 if response.get('role') in ('master', 'standby_leader') else 503
-            primary_status_code = 200 if response.get('role') == 'master' else 503
+            leader_status_code = 200 if response.get('role') in ('master', 'primary', 'standby_leader') else 503
+            primary_status_code = 200 if response.get('role') in ('master', 'primary') else 503
             standby_leader_status_code = 200 if response.get('role') == 'standby_leader' else 503
         elif patroni.ha.is_leader():
             leader_status_code = 200
@@ -141,13 +137,12 @@ class RestApiHandler(BaseHTTPRequestHandler):
             ignore_tags = True
         elif 'replica' in path:
             status_code = replica_status_code
-        elif 'read-only' in path:
+        elif 'read-only' in path and 'sync' not in path:
             status_code = 200 if 200 in (primary_status_code, standby_leader_status_code) else replica_status_code
         elif 'health' in path:
             status_code = 200 if response.get('state') == 'running' else 503
         elif cluster:  # dcs is available
-            is_synchronous = cluster.is_synchronous_mode() and cluster.sync \
-                    and patroni.postgresql.name in cluster.sync.members
+            is_synchronous = response.get('sync_standby')
             if path in ('/sync', '/synchronous') and is_synchronous:
                 status_code = replica_status_code
             elif path in ('/async', '/asynchronous') and not is_synchronous:
@@ -171,7 +166,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 if instance_tag_value is None:
                     status_code = 503
                     break
-                if not isinstance(instance_tag_value, six.string_types):
+                if not isinstance(instance_tag_value, str):
                     instance_tag_value = str(instance_tag_value).lower()
                 if instance_tag_value != qs_value:
                     status_code = 503
@@ -185,8 +180,20 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_OPTIONS(self):
         self.do_GET(write_status_code_only=True)
 
+    def do_HEAD(self):
+        self.do_GET(write_status_code_only=True)
+
     def do_GET_liveness(self):
-        self._write_status_code_only(200)
+        patroni = self.server.patroni
+        is_primary = patroni.postgresql.role in ('master', 'primary') and patroni.postgresql.is_running()
+        # We can tolerate Patroni problems longer on the replica.
+        # On the primary the liveness probe most likely will start failing only after the leader key expired.
+        # It should not be a big problem because replicas will see that the primary is still alive via REST API call.
+        liveness_threshold = patroni.dcs.ttl * (1 if is_primary else 2)
+
+        # In maintenance mode (pause) we are fine if heartbeat loop stuck.
+        status_code = 200 if patroni.ha.is_paused() or patroni.next_run + liveness_threshold > time.time() else 503
+        self._write_status_code_only(status_code)
 
     def do_GET_readiness(self):
         patroni = self.server.patroni
@@ -242,7 +249,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         metrics.append("# HELP patroni_master Value is 1 if this node is the leader, 0 otherwise.")
         metrics.append("# TYPE patroni_master gauge")
-        metrics.append("patroni_master{0} {1}".format(scope_label, int(postgres['role'] == 'master')))
+        metrics.append("patroni_master{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary'))))
+
+        metrics.append("# HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise.")
+        metrics.append("# TYPE patroni_primary gauge")
+        metrics.append("patroni_primary{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary'))))
 
         metrics.append("# HELP patroni_xlog_location Current location of the Postgres"
                        " transaction log, 0 if this node is not the leader.")
@@ -257,6 +268,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
         metrics.append("# TYPE patroni_replica gauge")
         metrics.append("patroni_replica{0} {1}".format(scope_label, int(postgres['role'] == 'replica')))
 
+        metrics.append("# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.")
+        metrics.append("# TYPE patroni_sync_standby gauge")
+        metrics.append("patroni_sync_standby{0} {1}".format(scope_label, int(postgres.get('sync_standby', False))))
+
         metrics.append("# HELP patroni_xlog_received_location Current location of the received"
                        " Postgres transaction log, 0 if this node is not a replica.")
         metrics.append("# TYPE patroni_xlog_received_location counter")
@@ -289,6 +304,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
         metrics.append("# TYPE patroni_cluster_unlocked gauge")
         metrics.append("patroni_cluster_unlocked{0} {1}".format(scope_label, int(postgres.get('cluster_unlocked', 0))))
 
+        metrics.append("# HELP patroni_failsafe_mode_is_active Value is 1 if the cluster is unlocked, 0 if locked.")
+        metrics.append("# TYPE patroni_failsafe_mode_is_active gauge")
+        metrics.append("patroni_failsafe_mode_is_active{0} {1}"
+                       .format(scope_label, int(postgres.get('failsafe_mode_is_active', 0))))
+
         metrics.append("# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.")
         metrics.append("# TYPE patroni_postgres_timeline counter")
         metrics.append("patroni_postgres_timeline{0} {1}".format(scope_label, postgres.get('timeline', 0)))
@@ -355,6 +375,32 @@ class RestApiHandler(BaseHTTPRequestHandler):
         self.server.patroni.sighup_handler()
         self._write_response(202, 'reload scheduled')
 
+    def do_GET_failsafe(self):
+        failsafe = self.server.patroni.dcs.failsafe
+        if isinstance(failsafe, dict):
+            self._write_json_response(200, failsafe)
+        else:
+            self.send_error(502)
+
+    @check_access
+    def do_POST_failsafe(self):
+        if self.server.patroni.ha.is_failsafe_mode():
+            request = self._read_json_content()
+            if request:
+                message = self.server.patroni.ha.update_failsafe(request) or 'Accepted'
+                code = 200 if message == 'Accepted' else 500
+                self._write_response(code, message)
+        else:
+            self.send_error(502)
+
+    @check_access
+    def do_POST_sigterm(self):
+        """Only for behave testing on windows"""
+
+        if os.name == 'nt' and os.getenv('BEHAVE_DEBUG'):
+            self.server.patroni.api_sigterm()
+        self._write_response(202, 'shutdown scheduled')
+
     @staticmethod
     def parse_schedule(schedule, action):
         """ parses the given schedule and validates at """
@@ -399,9 +445,9 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     status_code = _
                     break
             elif k == 'role':
-                if request[k] not in ('master', 'replica'):
+                if request[k] not in ('master', 'primary', 'replica'):
                     status_code = 400
-                    data = "PostgreSQL role should be either master or replica"
+                    data = "PostgreSQL role should be either primary or replica"
                     break
             elif k == 'postgres_version':
                 try:
@@ -571,6 +617,18 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_POST_switchover(self):
         self.do_POST_failover(action='switchover')
 
+    @check_access
+    def do_POST_citus(self):
+        request = self._read_json_content()
+        if not request:
+            return
+
+        patroni = self.server.patroni
+        if patroni.postgresql.citus_handler.is_coordinator() and patroni.ha.is_leader():
+            cluster = patroni.dcs.get_cluster(True)
+            patroni.postgresql.citus_handler.handle_event(cluster, request)
+        self._write_response(200, 'OK')
+
     def parse_request(self):
         """Override parse_request method to enrich basic functionality of `BaseHTTPRequestHandler` class
 
@@ -608,7 +666,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
             stmt = ("SELECT " + postgresql.POSTMASTER_START_TIME + ", " + postgresql.TL_LSN + ","
                     " pg_catalog.pg_last_xact_replay_timestamp(),"
                     " pg_catalog.array_to_json(pg_catalog.array_agg(pg_catalog.row_to_json(ri))) "
-                    "FROM (SELECT (SELECT rolname FROM pg_authid WHERE oid = usesysid) AS usename,"
+                    "FROM (SELECT (SELECT rolname FROM pg_catalog.pg_authid WHERE oid = usesysid) AS usename,"
                     " application_name, client_addr, w.state, sync_state, sync_priority"
                     " FROM pg_catalog.pg_stat_get_wal_senders() w, pg_catalog.pg_stat_get_activity(pid)) AS ri")
 
@@ -631,6 +689,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
             if result['role'] == 'replica' and self.server.patroni.ha.is_standby_cluster():
                 result['role'] = postgresql.role
 
+            if result['role'] == 'replica' and cluster and cluster.is_synchronous_mode()\
+                    and cluster.sync and postgresql.name in cluster.sync.members:
+                result['sync_standby'] = True
+
             if row[1] > 0:
                 result['timeline'] = row[1]
             else:
@@ -649,6 +711,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         if not cluster or cluster.is_unlocked():
             result['cluster_unlocked'] = True
+        if self.server.patroni.ha.failsafe_is_active():
+            result['failsafe_mode_is_active'] = True
         result['dcs_last_seen'] = self.server.patroni.dcs.last_seen
         return result
 
@@ -706,28 +770,28 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
     def __resolve_ips(host, port):
         try:
             for _, _, _, _, sa in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP):
-                yield ip_network(sa[0])
+                yield ip_network(sa[0], False)
         except Exception as e:
             logger.error('Failed to resolve %s: %r', host, e)
 
     def __members_ips(self):
         cluster = self.patroni.dcs.cluster
         if self.__allowlist_include_members and cluster:
-            for member in cluster.members:
-                if member.api_url:
-                    try:
-                        r = urlparse(member.api_url)
-                        host = r.hostname
-                        port = r.port or (443 if r.scheme == 'https' else 80)
-                        for ip in self.__resolve_ips(host, port):
-                            yield ip
-                    except Exception as e:
-                        logger.debug('Failed to parse url %s: %r', member.api_url, e)
+            for cluster in [cluster] + list(cluster.workers.values()):
+                for member in cluster.members:
+                    if member.api_url:
+                        try:
+                            r = urlparse(member.api_url)
+                            host = r.hostname
+                            port = r.port or (443 if r.scheme == 'https' else 80)
+                            for ip in self.__resolve_ips(host, port):
+                                yield ip
+                        except Exception as e:
+                            logger.debug('Failed to parse url %s: %r', member.api_url, e)
 
     def check_access(self, rh):
         if self.__allowlist or self.__allowlist_include_members:
-            incoming_ip = rh.client_address[0]
-            incoming_ip = ip_address(incoming_ip.decode('utf-8') if six.PY2 else incoming_ip)
+            incoming_ip = ip_address(rh.client_address[0])
             if not any(incoming_ip in net for net in self.__allowlist + tuple(self.__members_ips())):
                 return rh._write_response(403, 'Access is denied')
 
@@ -812,32 +876,23 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
                 else:
                     logger.error('Bad value in the "restapi.verify_client": %s', verify_client)
             self.__ssl_serial_number = self.get_certificate_serial_number()
-            self.socket = ctx.wrap_socket(self.socket, server_side=True)
+            self.socket = ctx.wrap_socket(self.socket, server_side=True, do_handshake_on_connect=False)
         if reloading_config:
             self.start()
 
     def process_request_thread(self, request, client_address):
-        if isinstance(request, tuple):
-            sock, newsock = request
-            try:
-                request = sock.context.wrap_socket(newsock, do_handshake_on_connect=sock.do_handshake_on_connect,
-                                                   suppress_ragged_eofs=sock.suppress_ragged_eofs, server_side=True)
-            except socket.error:
-                return
+        enable_keepalive(request, 10, 3)
+        if hasattr(request, 'context'):  # SSLSocket
+            request.do_handshake()
         super(RestApiServer, self).process_request_thread(request, client_address)
 
-    def get_request(self):
-        sock = self.socket
-        newsock, addr = socket.socket.accept(sock)
-        enable_keepalive(newsock, 10, 3)
-        if hasattr(sock, 'context'):  # SSLSocket, we want to do the deferred handshake from a thread
-            newsock = (sock, newsock)
-        return newsock, addr
-
     def shutdown_request(self, request):
-        if isinstance(request, tuple):
-            _, request = request  # SSLSocket
-        return super(RestApiServer, self).shutdown_request(request)
+        if hasattr(request, 'context'):  # SSLSocket
+            try:
+                request.unwrap()
+            except Exception as e:
+                logger.debug('Failed to shutdown SSL connection: %r', e)
+        super(RestApiServer, self).shutdown_request(request)
 
     def get_certificate_serial_number(self):
         if self.__ssl_options.get('certfile'):
@@ -861,7 +916,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
             for v in value:
                 if '/' in v:  # netmask
                     try:
-                        yield ip_network(v)
+                        yield ip_network(v, False)
                     except Exception as e:
                         logger.error('Invalid value "%s" in the allowlist: %r', v, e)
                 else:  # ip or hostname, try to resolve it
@@ -881,7 +936,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         self.http_extra_headers = config.get('http_extra_headers') or {}
         self.http_extra_headers.update((config.get('https_extra_headers') or {}) if ssl_options.get('certfile') else {})
 
-        if isinstance(config.get('verify_client'), six.string_types):
+        if isinstance(config.get('verify_client'), str):
             ssl_options['verify_client'] = config['verify_client'].lower()
 
         if self.__listen != config['listen'] or self.__ssl_options != ssl_options or self._received_new_cert:

patroni/config.py

@@ -32,7 +32,7 @@ _AUTH_ALLOWED_PARAMETERS = (
 
 def default_validator(conf):
     if not conf:
-        return "Config is empty."
+        raise ConfigParseError("Config is empty.")
 
 
 class Config(object):
@@ -58,16 +58,21 @@ class Config(object):
     PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION'
 
     __CACHE_FILENAME = 'patroni.dynamic.json'
+    __REMAP_KEYS = {
+        'master_start_timeout': 'primary_start_timeout',
+        'master_stop_timeout': 'primary_stop_timeout'
+    }
     __DEFAULT_CONFIG = {
         'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10,
         'maximum_lag_on_failover': 1048576,
         'maximum_lag_on_syncnode': -1,
         'check_timeline': False,
-        'master_start_timeout': 300,
-        'master_stop_timeout': 0,
+        'primary_start_timeout': 300,
+        'primary_stop_timeout': 0,
         'synchronous_mode': False,
         'synchronous_mode_strict': False,
         'synchronous_node_count': 1,
+        'failsafe_mode': False,
         'standby_cluster': {
             'create_replica_methods': '',
             'host': '',
@@ -102,9 +107,9 @@ class Config(object):
             config_env = os.environ.pop(self.PATRONI_CONFIG_VARIABLE, None)
             self._local_configuration = config_env and yaml.safe_load(config_env) or self.__environment_configuration
         if validator:
-            error = validator(self._local_configuration)
-            if error:
-                raise ConfigParseError(error)
+            errors = validator(self._local_configuration)
+            if errors:
+                raise ConfigParseError("\n".join(errors))
 
         self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration)
         self._data_dir = self.__effective_configuration.get('postgresql', {}).get('data_dir', "")
@@ -223,18 +228,22 @@ class Config(object):
         config = deepcopy(self.__DEFAULT_CONFIG)
 
         for name, value in dynamic_configuration.items():
+            # allow copying master_start_timeout->primary_start_timeout when the latter isn't in dynamic_configuration
+            if name in self.__REMAP_KEYS and self.__REMAP_KEYS[name] not in dynamic_configuration:
+                name = self.__REMAP_KEYS[name]
             if name == 'postgresql':
                 for name, value in (value or {}).items():
                     if name == 'parameters':
                         config['postgresql'][name].update(self._process_postgresql_parameters(value))
-                    elif name not in ('connect_address', 'listen', 'data_dir', 'pgpass', 'authentication'):
+                    elif name not in ('connect_address', 'proxy_address', 'listen',
+                                      'config_dir', 'data_dir', 'pgpass', 'authentication'):
                         config['postgresql'][name] = deepcopy(value)
             elif name == 'standby_cluster':
                 for name, value in (value or {}).items():
                     if name in self.__DEFAULT_CONFIG['standby_cluster']:
                         config['standby_cluster'][name] = deepcopy(value)
             elif name in config:  # only variables present in __DEFAULT_CONFIG allowed to be overridden from DCS
-                if name in ('synchronous_mode', 'synchronous_mode_strict'):
+                if name in ('synchronous_mode', 'synchronous_mode_strict', 'failsafe_mode'):
                     config[name] = value
                 else:
                     config[name] = int(value)
@@ -271,7 +280,8 @@ class Config(object):
                                         'cafile', 'ciphers', 'verify_client', 'http_extra_headers',
                                         'https_extra_headers', 'allowlist', 'allowlist_include_members'])
         _set_section_values('ctl', ['insecure', 'cacert', 'certfile', 'keyfile', 'keyfile_password'])
-        _set_section_values('postgresql', ['listen', 'connect_address', 'config_dir', 'data_dir', 'pgpass', 'bin_dir'])
+        _set_section_values('postgresql', ['listen', 'connect_address', 'proxy_address',
+                                           'config_dir', 'data_dir', 'pgpass', 'bin_dir'])
         _set_section_values('log', ['level', 'traceback_level', 'format', 'dateformat', 'max_queue_size',
                                     'dir', 'file_size', 'file_num', 'loggers'])
         _set_section_values('raft', ['data_dir', 'self_addr', 'partner_addrs', 'password', 'bind_addr'])
@@ -351,18 +361,24 @@ class Config(object):
                 if suffix in ('HOST', 'HOSTS', 'PORT', 'USE_PROXIES', 'PROTOCOL', 'SRV', 'SRV_SUFFIX', 'URL', 'PROXY',
                               'CACERT', 'CERT', 'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'CONSISTENCY',
                               'REGISTER_SERVICE', 'SERVICE_CHECK_INTERVAL', 'SERVICE_CHECK_TLS_SERVER_NAME',
-                              'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP',
-                              'PORTS', 'LABELS', 'BYPASS_API_SERVICE', 'KEY_PASSWORD', 'USE_SSL', 'SET_ACLS') and name:
+                              'SERVICE_TAGS', 'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL',
+                              'POD_IP', 'PORTS', 'LABELS', 'BYPASS_API_SERVICE', 'RETRIABLE_HTTP_CODES', 'KEY_PASSWORD',
+                              'USE_SSL', 'SET_ACLS', 'GROUP', 'DATABASE') and name:
                     value = os.environ.pop(param)
-                    if suffix == 'PORT':
+                    if name == 'CITUS':
+                        if suffix == 'GROUP':
+                            value = parse_int(value)
+                        elif suffix != 'DATABASE':
+                            continue
+                    elif suffix == 'PORT':
                         value = value and parse_int(value)
-                    elif suffix in ('HOSTS', 'PORTS', 'CHECKS'):
+                    elif suffix in ('HOSTS', 'PORTS', 'CHECKS', 'SERVICE_TAGS', 'RETRIABLE_HTTP_CODES'):
                         value = value and _parse_list(value)
                     elif suffix in ('LABELS', 'SET_ACLS'):
                         value = _parse_dict(value)
                     elif suffix in ('USE_PROXIES', 'REGISTER_SERVICE', 'USE_ENDPOINTS', 'BYPASS_API_SERVICE', 'VERIFY'):
                         value = parse_bool(value)
-                    if value:
+                    if value is not None:
                         ret[name.lower()][suffix.lower()] = value
         for dcs in ('etcd', 'etcd3'):
             if dcs in ret:
@@ -390,7 +406,11 @@ class Config(object):
     def _build_effective_configuration(self, dynamic_configuration, local_configuration):
         config = self._safe_copy_dynamic_configuration(dynamic_configuration)
         for name, value in local_configuration.items():
-            if name == 'postgresql':
+            if name == 'citus':  # remove invalid citus configuration
+                if isinstance(value, dict) and isinstance(value.get('group'), int)\
+                        and isinstance(value.get('database'), str):
+                    config[name] = value
+            elif name == 'postgresql':
                 for name, value in (value or {}).items():
                     if name == 'parameters':
                         config['postgresql'][name].update(self._process_postgresql_parameters(value, True))
@@ -428,6 +448,12 @@ class Config(object):
         if 'name' not in config and 'name' in pg_config:
             config['name'] = pg_config['name']
 
+        # when bootstrapping the new Citus cluster (coordinator/worker) enable sync replication in global configuration
+        if 'citus' in config:
+            bootstrap = config.setdefault('bootstrap', {})
+            dcs = bootstrap.setdefault('dcs', {})
+            dcs.setdefault('synchronous_mode', True)
+
         updated_fields = (
             'name',
             'scope',
@@ -435,7 +461,8 @@ class Config(object):
             'synchronous_mode',
             'synchronous_mode_strict',
             'synchronous_node_count',
-            'maximum_lag_on_syncnode'
+            'maximum_lag_on_syncnode',
+            'citus'
         )
 
         pg_config.update({p: config[p] for p in updated_fields if p in config})

patroni/ctl.py

@@ -4,17 +4,17 @@ Patroni Control
 
 import click
 import codecs
+import copy
 import datetime
 import dateutil.parser
 import dateutil.tz
-import copy
 import difflib
 import io
 import json
 import logging
 import os
 import random
-import six
+import shutil
 import subprocess
 import sys
 import tempfile
@@ -25,7 +25,7 @@ from click import ClickException
 from collections import defaultdict
 from contextlib import contextmanager
 from prettytable import ALL, FRAME, PrettyTable
-from six.moves.urllib_parse import urlparse
+from urllib.parse import urlparse
 
 try:
     from ydiff import markup_to_pager, PatchStream
@@ -34,9 +34,8 @@ except ImportError:  # pragma: no cover
 
 from .dcs import get_dcs as _get_dcs
 from .exceptions import PatroniException
-from .postgresql import Postgresql
 from .postgresql.misc import postgres_version_to_int
-from .utils import cluster_as_json, find_executable, patch_config, polling_loop
+from .utils import cluster_as_json, patch_config, polling_loop, is_standby_cluster
 from .request import PatroniRequest
 from .version import __version__
 
@@ -60,6 +59,18 @@ class PatronictlPrettyTable(PrettyTable):
         self.__hline_num = 0
         self.__hline = None
 
+    def __build_header(self, line):
+        header = self.__table_header[:len(line) - 2]
+        return "".join([line[0], header, line[1 + len(header):]])
+
+    def _stringify_hrule(self, *args, **kwargs):
+        ret = super(PatronictlPrettyTable, self)._stringify_hrule(*args, **kwargs)
+        where = args[1] if len(args) > 1 else kwargs.get('where')
+        if where == 'top_' and self.__table_header:
+            ret = self.__build_header(ret)
+            self.__hline_num += 1
+        return ret
+
     def _is_first_hline(self):
         return self.__hline_num == 0
 
@@ -71,8 +82,7 @@ class PatronictlPrettyTable(PrettyTable):
 
         # Inject nice table header
         if self._is_first_hline() and self.__table_header:
-            header = self.__table_header[:len(ret) - 2]
-            ret = "".join([ret[0], header, ret[1 + len(header):]])
+            ret = self.__build_header(ret)
 
         self.__hline_num += 1
         return ret
@@ -99,7 +109,7 @@ def parse_dcs(dcs):
     return yaml.safe_load(default['template'].format(host=parsed.hostname or 'localhost', port=port or default['port']))
 
 
-def load_config(path, dcs):
+def load_config(path, dcs_url):
     from patroni.config import Config
 
     if not (os.path.exists(path) and os.access(path, os.R_OK)):
@@ -112,53 +122,54 @@ def load_config(path, dcs):
         logging.debug('Loading configuration from file %s', path)
     config = Config(path, validator=None).copy()
 
-    dcs = parse_dcs(dcs) or parse_dcs(config.get('dcs_api')) or {}
-    if dcs:
+    dcs_url = parse_dcs(dcs_url) or {}
+    if dcs_url:
         for d in DCS_DEFAULTS:
             config.pop(d, None)
-        config.update(dcs)
+        config.update(dcs_url)
     return config
 
 
-def store_config(config, path):
-    dir_path = os.path.dirname(path)
-    if dir_path and not os.path.isdir(dir_path):
-        os.makedirs(dir_path)
-    with open(path, 'w') as fd:
-        yaml.dump(config, fd)
-
-
 option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, tsv, json, yaml)', default='pretty')
 option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
 option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
 option_force = click.option('--force', is_flag=True, help='Do not ask for confirmation at any point')
 arg_cluster_name = click.argument('cluster_name', required=False,
                                   default=lambda: click.get_current_context().obj.get('scope'))
+option_default_citus_group = click.option('--group', required=False, type=int, help='Citus group',
+                                          default=lambda: click.get_current_context().obj.get('citus', {}).get('group'))
+option_citus_group = click.option('--group', required=False, type=int, help='Citus group')
 option_insecure = click.option('-k', '--insecure', is_flag=True, help='Allow connections to SSL sites without certs')
+role_choice = click.Choice(['leader', 'primary', 'standby-leader', 'replica', 'standby', 'any', 'master'])
 
 
 @click.group()
 @click.option('--config-file', '-c', help='Configuration file',
               envvar='PATRONICTL_CONFIG_FILE', default=CONFIG_FILE_PATH)
-@click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
+@click.option('--dcs-url', '--dcs', '-d', 'dcs_url', help='The DCS connect url', envvar='DCS_URL')
 @option_insecure
 @click.pass_context
-def ctl(ctx, config_file, dcs, insecure):
+def ctl(ctx, config_file, dcs_url, insecure):
     level = 'WARNING'
     for name in ('LOGLEVEL', 'PATRONI_LOGLEVEL', 'PATRONI_LOG_LEVEL'):
         level = os.environ.get(name, level)
     logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=level)
     logging.captureWarnings(True)  # Capture eventual SSL warning
-    ctx.obj = load_config(config_file, dcs)
+    ctx.obj = load_config(config_file, dcs_url)
     # backward compatibility for configuration file where ctl section is not define
     ctx.obj.setdefault('ctl', {})['insecure'] = ctx.obj.get('ctl', {}).get('insecure') or insecure
 
 
-def get_dcs(config, scope):
+def get_dcs(config, scope, group):
     config.update({'scope': scope, 'patronictl': True})
+    if group is not None:
+        config['citus'] = {'group': group}
     config.setdefault('name', scope)
     try:
-        return _get_dcs(config)
+        dcs = _get_dcs(config)
+        if config.get('citus') and group is None:
+            dcs.get_cluster = dcs._get_citus_cluster
+        return dcs
     except PatroniException as e:
         raise PatroniCtlException(str(e))
 
@@ -183,15 +194,16 @@ def print_output(columns, rows, alignment=None, fmt='pretty', header=None, delim
             for row in rows:
                 if row[i]:
                     row[i] = format_config_for_editing(row[i], fmt != 'pretty').strip()
-        if list_cluster and fmt != 'tsv':  # skip cluster name if pretty-printing
-            columns = columns[1:] if columns else []
-            rows = [row[1:] for row in rows]
+        if list_cluster and fmt != 'tsv':  # skip cluster name and maybe Citus group if pretty-printing
+            skip_cols = 2 if ' (group: ' in header else 1
+            columns = columns[skip_cols:] if columns else []
+            rows = [row[skip_cols:] for row in rows]
 
         if fmt == 'tsv':
             for r in ([columns] if columns else []) + rows:
                 click.echo(delimiter.join(map(str, r)))
         else:
-            hrules = ALL if any(any(isinstance(c, six.string_types) and '\n' in c for c in r) for r in rows) else FRAME
+            hrules = ALL if any(any(isinstance(c, str) and '\n' in c for c in r) for r in rows) else FRAME
             table = PatronictlPrettyTable(header, columns, hrules=hrules)
             table.align = 'l'
             for k, v in (alignment or {}).items():
@@ -229,21 +241,29 @@ def watching(w, watch, max_count=None, clear=True):
         yield 0
 
 
-def get_all_members(cluster, role='master'):
-    if role == 'master':
-        if cluster.leader is not None and cluster.leader.name:
-            yield cluster.leader
+def get_all_members(obj, cluster, group, role='leader'):
+    clusters = {0: cluster}
+    if obj.get('citus') and group is None:
+        clusters.update(cluster.workers)
+    if role in ('leader', 'master', 'primary', 'standby-leader'):
+        role = {'primary': 'master', 'standby-leader': 'standby_leader'}.get(role, role)
+        for cluster in clusters.values():
+            if cluster.leader is not None and cluster.leader.name and\
+                    (role == 'leader' or
+                     cluster.leader.data.get('role') != 'master' and role == 'standby_leader' or
+                     cluster.leader.data.get('role') != 'standby_leader' and role == 'master'):
+                yield cluster.leader.member
         return
 
-    leader_name = (cluster.leader.member.name if cluster.leader else None)
-    for m in cluster.members:
-        if role == 'any' or role == 'replica' and m.name != leader_name:
-            yield m
+    for cluster in clusters.values():
+        leader_name = (cluster.leader.member.name if cluster.leader else None)
+        for m in cluster.members:
+            if role == 'any' or role in ('replica', 'standby') and m.name != leader_name:
+                yield m
 
 
-def get_any_member(cluster, role='master', member=None):
-    members = get_all_members(cluster, role)
-    for m in members:
+def get_any_member(obj, cluster, group, role='leader', member=None):
+    for m in get_all_members(obj, cluster, group, role):
         if member is None or m.name == member:
             return m
 
@@ -257,8 +277,8 @@ def get_all_members_leader_first(cluster):
             yield member
 
 
-def get_cursor(cluster, connect_parameters, role='master', member=None):
-    member = get_any_member(cluster, role=role, member=member)
+def get_cursor(obj, cluster, group, connect_parameters, role='leader', member=None):
+    member = get_any_member(obj, cluster, group, role=role, member=member)
     if member is None:
         return None
 
@@ -271,15 +291,15 @@ def get_cursor(cluster, connect_parameters, role='master', member=None):
 
     from . import psycopg
     conn = psycopg.connect(**params)
-    conn.autocommit = True
     cursor = conn.cursor()
-    if role == 'any':
+    if role in ('any', 'leader'):
         return cursor
 
     cursor.execute('SELECT pg_catalog.pg_is_in_recovery()')
     in_recovery = cursor.fetchone()[0]
 
-    if in_recovery and role == 'replica' or not in_recovery and role == 'master':
+    if in_recovery and role in ('replica', 'standby', 'standby-leader')\
+            or not in_recovery and role in ('master', 'primary'):
         return cursor
 
     conn.close()
@@ -287,32 +307,31 @@ def get_cursor(cluster, connect_parameters, role='master', member=None):
     return None
 
 
-def get_members(cluster, cluster_name, member_names, role, force, action, ask_confirmation=True):
-    candidates = {m.name: m for m in cluster.members}
+def get_members(obj, cluster, cluster_name, member_names, role, force, action, ask_confirmation=True, group=None):
+    members = list(get_all_members(obj, cluster, group, role))
 
+    candidates = {m.name for m in members}
     if not force or role:
         if not member_names and not candidates:
             raise PatroniCtlException('{0} cluster doesn\'t have any members'.format(cluster_name))
-        output_members(cluster, cluster_name)
+        output_members(obj, cluster, cluster_name, group=group)
 
-    if role:
-        role_names = [m.name for m in get_all_members(cluster, role)]
-        if member_names:
-            member_names = list(set(member_names) & set(role_names))
-            if not member_names:
-                raise PatroniCtlException('No {0} among provided members'.format(role))
-        else:
-            member_names = role_names
+    if member_names:
+        member_names = list(set(member_names) & candidates)
+        if not member_names:
+            raise PatroniCtlException('No {0} among provided members'.format(role))
+    elif action != 'reinitialize':
+        member_names = list(candidates)
 
     if not member_names and not force:
         member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(action,
-                        ', '.join(candidates.keys())), type=str, default='')]
+                        ', '.join(candidates)), type=str, default='')]
 
     for member_name in member_names:
         if member_name not in candidates:
             raise PatroniCtlException('{0} is not a member of cluster'.format(member_name))
 
-    members = [candidates[n] for n in member_names]
+    members = [m for m in members if m.name in member_names]
     if ask_confirmation:
         confirm_members_action(members, force, action)
     return members
@@ -333,20 +352,22 @@ def confirm_members_action(members, force, action, scheduled_at=None):
                 raise PatroniCtlException('Aborted {0}'.format(action))
 
 
-@ctl.command('dsn', help='Generate a dsn for the provided member, defaults to a dsn of the master')
-@click.option('--role', '-r', help='Give a dsn of any member with this role', type=click.Choice(['master', 'replica',
-              'any']), default=None)
+@ctl.command('dsn', help='Generate a dsn for the provided member, defaults to a dsn of the leader')
+@click.option('--role', '-r', help='Give a dsn of any member with this role', type=role_choice, default=None)
 @click.option('--member', '-m', help='Generate a dsn for this member', type=str)
 @arg_cluster_name
+@option_citus_group
 @click.pass_obj
-def dsn(obj, cluster_name, role, member):
-    if role is not None and member is not None:
-        raise PatroniCtlException('--role and --member are mutually exclusive options')
+def dsn(obj, cluster_name, group, role, member):
+    if member is not None:
+        if role is not None:
+            raise PatroniCtlException('--role and --member are mutually exclusive options')
+        role = 'any'
     if member is None and role is None:
-        role = 'master'
+        role = 'leader'
 
-    cluster = get_dcs(obj, cluster_name).get_cluster()
-    m = get_any_member(cluster, role=role, member=member)
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
+    m = get_any_member(obj, cluster, group, role=role, member=member)
     if m is None:
         raise PatroniCtlException('Can not find a suitable member')
 
@@ -356,14 +377,14 @@ def dsn(obj, cluster_name, role, member):
 
 @ctl.command('query', help='Query a Patroni PostgreSQL member')
 @arg_cluster_name
+@option_citus_group
 @click.option('--format', 'fmt', help='Output format (pretty, tsv, json, yaml)', default='tsv')
 @click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb'))
 @click.option('--password', help='force password prompt', is_flag=True)
 @click.option('-U', '--username', help='database user name', type=str)
 @option_watch
 @option_watchrefresh
-@click.option('--role', '-r', help='The role of the query', type=click.Choice(['master', 'replica', 'any']),
-              default=None)
+@click.option('--role', '-r', help='The role of the query', type=role_choice, default=None)
 @click.option('--member', '-m', help='Query a specific member', type=str)
 @click.option('--delimiter', help='The column delimiter', default='\t')
 @click.option('--command', '-c', help='The SQL commands to execute')
@@ -372,6 +393,7 @@ def dsn(obj, cluster_name, role, member):
 def query(
     obj,
     cluster_name,
+    group,
     role,
     member,
     w,
@@ -384,10 +406,12 @@ def query(
     dbname,
     fmt='tsv',
 ):
-    if role is not None and member is not None:
-        raise PatroniCtlException('--role and --member are mutually exclusive options')
+    if member is not None:
+        if role is not None:
+            raise PatroniCtlException('--role and --member are mutually exclusive options')
+        role = 'any'
     if member is None and role is None:
-        role = 'master'
+        role = 'leader'
 
     if p_file is not None and command is not None:
         raise PatroniCtlException('--file and --command are mutually exclusive options')
@@ -406,25 +430,25 @@ def query(
     if p_file is not None:
         command = p_file.read()
 
-    dcs = get_dcs(obj, cluster_name)
+    dcs = get_dcs(obj, cluster_name, group)
 
     cursor = None
     for _ in watching(w, watch, clear=False):
         if cursor is None:
             cluster = dcs.get_cluster()
 
-        output, header = query_member(cluster, cursor, member, role, command, connect_parameters)
+        output, header = query_member(obj, cluster, group, cursor, member, role, command, connect_parameters)
         print_output(header, output, fmt=fmt, delimiter=delimiter)
 
 
-def query_member(cluster, cursor, member, role, command, connect_parameters):
+def query_member(obj, cluster, group, cursor, member, role, command, connect_parameters):
     from . import psycopg
     try:
         if cursor is None:
-            cursor = get_cursor(cluster, connect_parameters, role=role, member=member)
+            cursor = get_cursor(obj, cluster, group, connect_parameters, role=role, member=member)
 
         if cursor is None:
-            if role is None:
+            if member is not None:
                 message = 'No connection to member {0} is available'.format(member)
             else:
                 message = 'No connection to role={0} is available'.format(role)
@@ -444,13 +468,16 @@ def query_member(cluster, cursor, member, role, command, connect_parameters):
 
 @ctl.command('remove', help='Remove cluster from DCS')
 @click.argument('cluster_name')
+@option_citus_group
 @option_format
 @click.pass_obj
-def remove(obj, cluster_name, fmt):
-    dcs = get_dcs(obj, cluster_name)
+def remove(obj, cluster_name, group, fmt):
+    dcs = get_dcs(obj, cluster_name, group)
     cluster = dcs.get_cluster()
 
-    output_members(cluster, cluster_name, fmt=fmt)
+    if obj.get('citus') and group is None:
+        raise PatroniCtlException('For Citus clusters the --group must me specified')
+    output_members(obj, cluster, cluster_name, fmt=fmt)
 
     confirm = click.prompt('Please confirm the cluster name to remove', type=str)
     if confirm != cluster_name:
@@ -464,9 +491,9 @@ def remove(obj, cluster_name, fmt):
         raise PatroniCtlException('You did not exactly type "{0}"'.format(message))
 
     if cluster.leader and cluster.leader.name:
-        confirm = click.prompt('This cluster currently is healthy. Please specify the master name to continue')
+        confirm = click.prompt('This cluster currently is healthy. Please specify the leader name to continue')
         if confirm != cluster.leader.name:
-            raise PatroniCtlException('You did not specify the current master of the cluster')
+            raise PatroniCtlException('You did not specify the current leader of the cluster')
 
     dcs.delete_cluster()
 
@@ -499,14 +526,14 @@ def parse_scheduled(scheduled):
 @ctl.command('reload', help='Reload cluster member configuration')
 @click.argument('cluster_name')
 @click.argument('member_names', nargs=-1)
-@click.option('--role', '-r', help='Reload only members with this role', default='any',
-              type=click.Choice(['master', 'replica', 'any']))
+@option_citus_group
+@click.option('--role', '-r', help='Reload only members with this role', type=role_choice, default='any')
 @option_force
 @click.pass_obj
-def reload(obj, cluster_name, member_names, force, role):
-    cluster = get_dcs(obj, cluster_name).get_cluster()
+def reload(obj, cluster_name, member_names, group, force, role):
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
 
-    members = get_members(cluster, cluster_name, member_names, role, force, 'reload')
+    members = get_members(obj, cluster, cluster_name, member_names, role, force, 'reload', group=group)
 
     for member in members:
         r = request_patroni(member, 'post', 'reload')
@@ -525,8 +552,8 @@ def reload(obj, cluster_name, member_names, force, role):
 @ctl.command('restart', help='Restart cluster member')
 @click.argument('cluster_name')
 @click.argument('member_names', nargs=-1)
-@click.option('--role', '-r', help='Restart only members with this role', default='any',
-              type=click.Choice(['master', 'replica', 'any']))
+@option_citus_group
+@click.option('--role', '-r', help='Restart only members with this role', type=role_choice, default='any')
 @click.option('--any', 'p_any', help='Restart a single member only', is_flag=True)
 @click.option('--scheduled', help='Timestamp of a scheduled restart in unambiguous format (e.g. ISO 8601)',
               default=None)
@@ -537,10 +564,10 @@ def reload(obj, cluster_name, member_names, force, role):
               help='Return error and fail over if necessary when restarting takes longer than this.')
 @option_force
 @click.pass_obj
-def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, version, pending, timeout):
-    cluster = get_dcs(obj, cluster_name).get_cluster()
+def restart(obj, cluster_name, group, member_names, force, role, p_any, scheduled, version, pending, timeout):
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
 
-    members = get_members(cluster, cluster_name, member_names, role, force, 'restart', False)
+    members = get_members(obj, cluster, cluster_name, member_names, role, force, 'restart', False, group=group)
     if scheduled is None and not force:
         next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
         scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ',
@@ -598,13 +625,14 @@ def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, vers
 
 @ctl.command('reinit', help='Reinitialize cluster member')
 @click.argument('cluster_name')
+@option_citus_group
 @click.argument('member_names', nargs=-1)
 @option_force
 @click.option('--wait', help='Wait until reinitialization completes', is_flag=True)
 @click.pass_obj
-def reinit(obj, cluster_name, member_names, force, wait):
-    cluster = get_dcs(obj, cluster_name).get_cluster()
-    members = get_members(cluster, cluster_name, member_names, None, force, 'reinitialize')
+def reinit(obj, cluster_name, group, member_names, force, wait):
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
+    members = get_members(obj, cluster, cluster_name, member_names, 'replica', force, 'reinitialize', group=group)
 
     wait_on_members = []
     for member in members:
@@ -635,31 +663,42 @@ def reinit(obj, cluster_name, member_names, force, wait):
                 wait_on_members.remove(member)
 
 
-def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, force, scheduled=None):
+def _do_failover_or_switchover(obj, action, cluster_name, group, leader, candidate, force, scheduled=None):
     """
         We want to trigger a failover or switchover for the specified cluster name.
 
-        We verify that the cluster name, master name and candidate name are correct.
+        We verify that the cluster name, leader name and candidate name are correct.
         If so, we trigger an action and keep the client up to date.
     """
 
-    dcs = get_dcs(obj, cluster_name)
+    dcs = get_dcs(obj, cluster_name, group)
     cluster = dcs.get_cluster()
+    click.echo('Current cluster topology')
+    output_members(obj, cluster, cluster_name, group=group)
+
+    if obj.get('citus') and group is None:
+        if force:
+            raise PatroniCtlException('For Citus clusters the --group must me specified')
+        else:
+            group = click.prompt('Citus group', type=int)
+            dcs = get_dcs(obj, cluster_name, group)
+            cluster = dcs.get_cluster()
 
     if action == 'switchover' and (cluster.leader is None or not cluster.leader.name):
-        raise PatroniCtlException('This cluster has no master')
+        raise PatroniCtlException('This cluster has no leader')
 
-    if master is None:
+    if leader is None:
         if force or action == 'failover':
-            master = cluster.leader and cluster.leader.name
+            leader = cluster.leader and cluster.leader.name
         else:
-            master = click.prompt('Master', type=str, default=cluster.leader.member.name)
+            prompt = 'Standby Leader' if is_standby_cluster(cluster.config) else 'Primary'
+            leader = click.prompt(prompt, type=str, default=cluster.leader.member.name)
 
-    if master is not None and cluster.leader and cluster.leader.member.name != master:
-        raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(master, cluster_name))
+    if leader is not None and cluster.leader and cluster.leader.member.name != leader:
+        raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(leader, cluster_name))
 
     # excluding members with nofailover tag
-    candidate_names = [str(m.name) for m in cluster.members if m.name != master and not m.nofailover]
+    candidate_names = [str(m.name) for m in cluster.members if m.name != leader and not m.nofailover]
     # We sort the names for consistent output to the client
     candidate_names.sort()
 
@@ -672,7 +711,7 @@ def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, for
     if action == 'failover' and not candidate:
         raise PatroniCtlException('Failover could be performed only to a specific candidate')
 
-    if candidate == master:
+    if candidate == leader:
         raise PatroniCtlException(action.title() + ' target and source are the same.')
 
     if candidate and candidate not in candidate_names:
@@ -693,16 +732,13 @@ def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, for
                 raise PatroniCtlException("Can't schedule switchover in the paused state")
             scheduled_at_str = scheduled_at.isoformat()
 
-    failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at_str}
+    failover_value = {'leader': leader, 'candidate': candidate, 'scheduled_at': scheduled_at_str}
 
     logging.debug(failover_value)
 
     # By now we have established that the leader exists and the candidate exists
-    click.echo('Current cluster topology')
-    output_members(dcs.get_cluster(), cluster_name)
-
     if not force:
-        demote_msg = ', demoting current master ' + master if master else ''
+        demote_msg = ', demoting current leader ' + leader if leader else ''
         if scheduled_at_str:
             if not click.confirm('Are you sure you want to schedule {0} of cluster {1} at {2}{3}?'
                                  .format(action, cluster_name, scheduled_at_str, demote_msg)):
@@ -734,32 +770,34 @@ def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, for
         logging.exception(r)
         logging.warning('Failing over to DCS')
         click.echo('{0} Could not {1} using Patroni api, falling back to DCS'.format(timestamp(), action))
-        dcs.manual_failover(master, candidate, scheduled_at=scheduled_at)
+        dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at)
 
-    output_members(cluster, cluster_name)
+    output_members(obj, cluster, cluster_name, group=group)
 
 
 @ctl.command('failover', help='Failover to a replica')
 @arg_cluster_name
-@click.option('--master', help='The name of the current master', default=None)
+@option_citus_group
+@click.option('--leader', '--primary', '--master', 'leader', help='The name of the current leader', default=None)
 @click.option('--candidate', help='The name of the candidate', default=None)
 @option_force
 @click.pass_obj
-def failover(obj, cluster_name, master, candidate, force):
-    action = 'switchover' if master else 'failover'
-    _do_failover_or_switchover(obj, action, cluster_name, master, candidate, force)
+def failover(obj, cluster_name, group, leader, candidate, force):
+    action = 'switchover' if leader else 'failover'
+    _do_failover_or_switchover(obj, action, cluster_name, group, leader, candidate, force)
 
 
 @ctl.command('switchover', help='Switchover to a replica')
 @arg_cluster_name
-@click.option('--master', help='The name of the current master', default=None)
+@option_citus_group
+@click.option('--leader', '--primary', '--master', 'leader', help='The name of the current leader', default=None)
 @click.option('--candidate', help='The name of the candidate', default=None)
 @click.option('--scheduled', help='Timestamp of a scheduled switchover in unambiguous format (e.g. ISO 8601)',
               default=None)
 @option_force
 @click.pass_obj
-def switchover(obj, cluster_name, master, candidate, force, scheduled):
-    _do_failover_or_switchover(obj, 'switchover', cluster_name, master, candidate, force, scheduled)
+def switchover(obj, cluster_name, group, leader, candidate, force, scheduled):
+    _do_failover_or_switchover(obj, 'switchover', cluster_name, group, leader, candidate, force, scheduled)
 
 
 def generate_topology(level, member, topology):
@@ -789,48 +827,7 @@ def topology_sort(members):
         yield member
 
 
-def output_members(cluster, name, extended=False, fmt='pretty'):
-    rows = []
-    logging.debug(cluster)
-    initialize = {None: 'uninitialized', '': 'initializing'}.get(cluster.initialize, cluster.initialize)
-    cluster = cluster_as_json(cluster)
-
-    columns = ['Cluster', 'Member', 'Host', 'Role', 'State', 'TL', 'Lag in MB']
-    for c in ('Pending restart', 'Scheduled restart', 'Tags'):
-        if extended or any(m.get(c.lower().replace(' ', '_')) for m in cluster['members']):
-            columns.append(c)
-
-    # Show Host as 'host:port' if somebody is running on non-standard port or two nodes are running on the same host
-    members = [m for m in cluster['members'] if 'host' in m]
-    append_port = any('port' in m and m['port'] != 5432 for m in members) or\
-        len(set(m['host'] for m in members)) < len(members)
-
-    sort = topology_sort if fmt == 'topology' else iter
-    for m in sort(cluster['members']):
-        logging.debug(m)
-
-        lag = m.get('lag', '')
-        m.update(cluster=name, member=m['name'], host=m.get('host', ''), tl=m.get('timeline', ''),
-                 role=m['role'].replace('_', ' ').title(),
-                 lag_in_mb=round(lag/1024/1024) if isinstance(lag, six.integer_types) else lag,
-                 pending_restart='*' if m.get('pending_restart') else '')
-
-        if append_port and m['host'] and m.get('port'):
-            m['host'] = ':'.join([m['host'], str(m['port'])])
-
-        if 'scheduled_restart' in m:
-            value = m['scheduled_restart']['schedule']
-            if 'postgres_version' in m['scheduled_restart']:
-                value += ' if version < {0}'.format(m['scheduled_restart']['postgres_version'])
-            m['scheduled_restart'] = value
-
-        rows.append([m.get(n.lower().replace(' ', '_'), '') for n in columns])
-
-    print_output(columns, rows, {'Lag in MB': 'r', 'TL': 'r'}, fmt, ' Cluster: {0} ({1}) '.format(name, initialize))
-
-    if fmt not in ('pretty', 'topology'):  # Omit service info when using machine-readable formats
-        return
-
+def get_cluster_service_info(cluster):
     service_info = []
     if cluster.get('pause'):
         service_info.append('Maintenance mode: on')
@@ -841,44 +838,109 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
             if name in cluster['scheduled_switchover']:
                 info += '\n{0:>24}: {1}'.format(name, cluster['scheduled_switchover'][name])
         service_info.append(info)
+    return service_info
 
-    if service_info:
-        click.echo(' ' + '\n '.join(service_info))
+
+def output_members(obj, cluster, name, extended=False, fmt='pretty', group=None):
+    rows = []
+    logging.debug(cluster)
+
+    initialize = {None: 'uninitialized', '': 'initializing'}.get(cluster.initialize, cluster.initialize)
+    columns = ['Cluster', 'Member', 'Host', 'Role', 'State', 'TL', 'Lag in MB']
+
+    clusters = {group or 0: cluster_as_json(cluster)}
+
+    is_citus_cluster = obj.get('citus')
+    if is_citus_cluster:
+        columns.insert(1, 'Group')
+        if group is None:
+            clusters.update({g: cluster_as_json(c) for g, c in cluster.workers.items()})
+
+    all_members = [m for c in clusters.values() for m in c['members'] if 'host' in m]
+
+    for c in ('Pending restart', 'Scheduled restart', 'Tags'):
+        if extended or any(m.get(c.lower().replace(' ', '_')) for m in all_members):
+            columns.append(c)
+
+    # Show Host as 'host:port' if somebody is running on non-standard port or two nodes are running on the same host
+    append_port = any('port' in m and m['port'] != 5432 for m in all_members) or\
+        len(set(m['host'] for m in all_members)) < len(all_members)
+
+    sort = topology_sort if fmt == 'topology' else iter
+    for g, cluster in sorted(clusters.items()):
+        for member in sort(cluster['members']):
+            logging.debug(member)
+
+            lag = member.get('lag', '')
+            member.update(cluster=name, member=member['name'], group=g,
+                          host=member.get('host', ''), tl=member.get('timeline', ''),
+                          role=member['role'].replace('_', ' ').title(),
+                          lag_in_mb=round(lag/1024/1024) if isinstance(lag, int) else lag,
+                          pending_restart='*' if member.get('pending_restart') else '')
+
+            if append_port and member['host'] and member.get('port'):
+                member['host'] = ':'.join([member['host'], str(member['port'])])
+
+            if 'scheduled_restart' in member:
+                value = member['scheduled_restart']['schedule']
+                if 'postgres_version' in member['scheduled_restart']:
+                    value += ' if version < {0}'.format(member['scheduled_restart']['postgres_version'])
+                member['scheduled_restart'] = value
+
+            rows.append([member.get(n.lower().replace(' ', '_'), '') for n in columns])
+
+    title = 'Citus cluster' if is_citus_cluster else 'Cluster'
+    group_title = '' if group is None else 'group: {0}, '.format(group)
+    title_details = group_title and ' ({0}{1})'.format(group_title, initialize)
+    title = ' {0}: {1}{2} '.format(title, name, title_details)
+    print_output(columns, rows, {'Group': 'r', 'Lag in MB': 'r', 'TL': 'r'}, fmt, title)
+
+    if fmt not in ('pretty', 'topology'):  # Omit service info when using machine-readable formats
+        return
+
+    for g, cluster in sorted(clusters.items()):
+        service_info = get_cluster_service_info(cluster)
+        if service_info:
+            if is_citus_cluster and group is None:
+                click.echo('Citus group: {0}'.format(g))
+            click.echo(' ' + '\n '.join(service_info))
 
 
 @ctl.command('list', help='List the Patroni members for a given Patroni')
 @click.argument('cluster_names', nargs=-1)
+@option_citus_group
 @click.option('--extended', '-e', help='Show some extra information', is_flag=True)
 @click.option('--timestamp', '-t', 'ts', help='Print timestamp', is_flag=True)
 @option_format
 @option_watch
 @option_watchrefresh
 @click.pass_obj
-def members(obj, cluster_names, fmt, watch, w, extended, ts):
+def members(obj, cluster_names, group, fmt, watch, w, extended, ts):
     if not cluster_names:
         if 'scope' in obj:
             cluster_names = [obj['scope']]
         if not cluster_names:
             return logging.warning('Listing members: No cluster names were provided')
 
-    for cluster_name in cluster_names:
-        dcs = get_dcs(obj, cluster_name)
+    for _ in watching(w, watch):
+        if ts:
+            click.echo(timestamp(0))
 
-        for _ in watching(w, watch):
-            if ts:
-                click.echo(timestamp(0))
+        for cluster_name in cluster_names:
+            dcs = get_dcs(obj, cluster_name, group)
 
             cluster = dcs.get_cluster()
-            output_members(cluster, cluster_name, extended, fmt)
+            output_members(obj, cluster, cluster_name, extended, fmt, group)
 
 
 @ctl.command('topology', help='Prints ASCII topology for given cluster')
 @click.argument('cluster_names', nargs=-1)
+@option_citus_group
 @option_watch
 @option_watchrefresh
 @click.pass_obj
 @click.pass_context
-def topology(ctx, obj, cluster_names, watch, w):
+def topology(ctx, obj, cluster_names, group, watch, w):
     ctx.forward(members, fmt='topology')
 
 
@@ -886,83 +948,20 @@ def timestamp(precision=6):
     return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:precision - 7]
 
 
-@ctl.command('configure', help='Create configuration file')
-@click.option('--config-file', '-c', help='Configuration file', prompt='Configuration file', default=CONFIG_FILE_PATH)
-@click.option('--dcs', '-d', help='The DCS connect url', prompt='DCS connect url', default='etcd://localhost:2379')
-@click.option('--namespace', '-n', help='The namespace', prompt='Namespace', default='/service/')
-def configure(config_file, dcs, namespace):
-    store_config({'dcs_api': str(dcs), 'namespace': str(namespace)}, config_file)
-
-
-def touch_member(config, dcs):
-    ''' Rip-off of the ha.touch_member without inter-class dependencies '''
-    p = Postgresql(config['postgresql'])
-    p.set_state('running')
-    p.set_role('master')
-
-    def restapi_connection_string(config):
-        protocol = 'https' if config.get('certfile') else 'http'
-        connect_address = config.get('connect_address')
-        listen = config['listen']
-        return '{0}://{1}/patroni'.format(protocol, connect_address or listen)
-
-    data = {
-        'conn_url': p.connection_string,
-        'api_url': restapi_connection_string(config['restapi']),
-        'state': p.state,
-        'role': p.role
-    }
-
-    return dcs.touch_member(data, permanent=True)
-
-
-def set_defaults(config, cluster_name):
-    """fill-in some basic configuration parameters if config file is not set """
-    config['postgresql'].setdefault('name', cluster_name)
-    config['postgresql'].setdefault('scope', cluster_name)
-    config['postgresql'].setdefault('listen', '127.0.0.1')
-    config['postgresql']['authentication'] = {'replication': None}
-    config['restapi']['listen'] = ':' in config['restapi']['listen'] and config['restapi']['listen'] or '127.0.0.1:8008'
-
-
-@ctl.command('scaffold', help='Create a structure for the cluster in DCS')
-@click.argument('cluster_name')
-@click.option('--sysid', '-s', help='System ID of the cluster to put into the initialize key', default="")
-@click.pass_obj
-def scaffold(obj, cluster_name, sysid):
-    dcs = get_dcs(obj, cluster_name)
-    cluster = dcs.get_cluster()
-    if cluster and cluster.initialize is not None:
-        raise PatroniCtlException("This cluster is already initialized")
-
-    if not dcs.initialize(create_new=True, sysid=sysid):
-        # initialize key already exists, don't touch this cluster
-        raise PatroniCtlException("Initialize key for cluster {0} already exists".format(cluster_name))
-
-    set_defaults(obj, cluster_name)
-
-    # make sure the leader keys will never expire
-    if not (touch_member(obj, dcs) and dcs.attempt_to_acquire_leader(permanent=True)):
-        # we did initialize this cluster, but failed to write the leader or member keys, wipe it down completely.
-        dcs.delete_cluster()
-        raise PatroniCtlException("Unable to install permanent leader for cluster {0}".format(cluster_name))
-    click.echo("Cluster {0} has been created successfully".format(cluster_name))
-
-
 @ctl.command('flush', help='Discard scheduled events')
 @click.argument('cluster_name')
+@option_citus_group
 @click.argument('member_names', nargs=-1)
 @click.argument('target', type=click.Choice(['restart', 'switchover']))
-@click.option('--role', '-r', help='Flush only members with this role', default='any',
-              type=click.Choice(['master', 'replica', 'any']))
+@click.option('--role', '-r', help='Flush only members with this role', type=role_choice, default='any')
 @option_force
 @click.pass_obj
-def flush(obj, cluster_name, member_names, force, role, target):
-    dcs = get_dcs(obj, cluster_name)
+def flush(obj, cluster_name, group, member_names, force, role, target):
+    dcs = get_dcs(obj, cluster_name, group)
     cluster = dcs.get_cluster()
 
     if target == 'restart':
-        for member in get_members(cluster, cluster_name, member_names, role, force, 'flush'):
+        for member in get_members(obj, cluster, cluster_name, member_names, role, force, 'flush', group=group):
             if member.data.get('scheduled_restart'):
                 r = request_patroni(member, 'delete', 'restart')
                 check_response(r, member.name, 'flush scheduled restart')
@@ -1008,8 +1007,8 @@ def wait_until_pause_is_applied(dcs, paused, old_cluster):
     return click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
 
 
-def toggle_pause(config, cluster_name, paused, wait):
-    dcs = get_dcs(config, cluster_name)
+def toggle_pause(config, cluster_name, group, paused, wait):
+    dcs = get_dcs(config, cluster_name, group)
     cluster = dcs.get_cluster()
     if cluster.is_paused() == paused:
         raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not'))
@@ -1037,18 +1036,20 @@ def toggle_pause(config, cluster_name, paused, wait):
 
 @ctl.command('pause', help='Disable auto failover')
 @arg_cluster_name
+@option_default_citus_group
 @click.pass_obj
 @click.option('--wait', help='Wait until pause is applied on all nodes', is_flag=True)
-def pause(obj, cluster_name, wait):
-    return toggle_pause(obj, cluster_name, True, wait)
+def pause(obj, cluster_name, group, wait):
+    return toggle_pause(obj, cluster_name, group, True, wait)
 
 
 @ctl.command('resume', help='Resume auto failover')
 @arg_cluster_name
+@option_default_citus_group
 @click.option('--wait', help='Wait until pause is cleared on all nodes', is_flag=True)
 @click.pass_obj
-def resume(obj, cluster_name, wait):
-    return toggle_pause(obj, cluster_name, False, wait)
+def resume(obj, cluster_name, group, wait):
+    return toggle_pause(obj, cluster_name, group, False, wait)
 
 
 @contextmanager
@@ -1083,8 +1084,7 @@ def show_diff(before_editing, after_editing):
     if sys.stdout.isatty():
         buf = io.StringIO()
         for line in unified_diff:
-            # Force cast to unicode as difflib on Python 2.7 returns a mix of unicode and str.
-            buf.write(six.text_type(line))
+            buf.write(str(line))
         buf.seek(0)
 
         class opts:
@@ -1092,12 +1092,28 @@ def show_diff(before_editing, after_editing):
             width = 80
             tab_width = 8
             wrap = True
-            if find_executable('less'):
-                pager = None
-            else:
-                pager = 'more.com' if sys.platform == 'win32' else 'more'
+            pager = next(
+                (
+                    os.path.basename(p)
+                    for p in (os.environ.get('PAGER'), "less", "more")
+                    if p is not None and shutil.which(p)
+                ),
+                None,
+            )
             pager_options = None
 
+        if opts.pager is None:
+            raise PatroniCtlException(
+                'No pager could be found. Either set PAGER environment variable with '
+                'your pager or install either "less" or "more" in the host.'
+            )
+
+        # if we end up selecting "less" as "pager" then we set "pager" attribute
+        # to "None". "less" is the default pager for "ydiff" module, and that
+        # module adds some command-line options to "less" when "pager" is "None"
+        if opts.pager == 'less':
+            opts.pager = None
+
         markup_to_pager(PatchStream(buf), opts)
     else:
         for line in unified_diff:
@@ -1183,7 +1199,7 @@ def invoke_editor(before_editing, cluster_name):
     editor_cmd = os.environ.get('EDITOR')
     if not editor_cmd:
         for editor in ('editor', 'vi'):
-            editor_cmd = find_executable(editor)
+            editor_cmd = shutil.which(editor)
             if editor_cmd:
                 logging.debug('Setting fallback editor_cmd=%s', editor)
                 break
@@ -1205,6 +1221,7 @@ def invoke_editor(before_editing, cluster_name):
 
 @ctl.command('edit-config', help="Edit cluster configuration")
 @arg_cluster_name
+@option_default_citus_group
 @click.option('--quiet', '-q', is_flag=True, help='Do not show changes')
 @click.option('--set', '-s', 'kvpairs', multiple=True,
               help='Set specific configuration value. Can be specified multiple times')
@@ -1216,8 +1233,8 @@ def invoke_editor(before_editing, cluster_name):
               ' Use - for stdin.')
 @option_force
 @click.pass_obj
-def edit_config(obj, cluster_name, force, quiet, kvpairs, pgkvpairs, apply_filename, replace_filename):
-    dcs = get_dcs(obj, cluster_name)
+def edit_config(obj, cluster_name, group, force, quiet, kvpairs, pgkvpairs, apply_filename, replace_filename):
+    dcs = get_dcs(obj, cluster_name, group)
     cluster = dcs.get_cluster()
 
     before_editing = format_config_for_editing(cluster.config.data)
@@ -1259,9 +1276,10 @@ def edit_config(obj, cluster_name, force, quiet, kvpairs, pgkvpairs, apply_filen
 
 @ctl.command('show-config', help="Show cluster configuration")
 @arg_cluster_name
+@option_default_citus_group
 @click.pass_obj
-def show_config(obj, cluster_name):
-    cluster = get_dcs(obj, cluster_name).get_cluster()
+def show_config(obj, cluster_name, group):
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
 
     click.echo(format_config_for_editing(cluster.config.data))
 
@@ -1269,16 +1287,17 @@ def show_config(obj, cluster_name):
 @ctl.command('version', help='Output version of patronictl command or a running Patroni instance')
 @click.argument('cluster_name', required=False)
 @click.argument('member_names', nargs=-1)
+@option_citus_group
 @click.pass_obj
-def version(obj, cluster_name, member_names):
+def version(obj, cluster_name, group, member_names):
     click.echo("patronictl version {0}".format(__version__))
 
     if not cluster_name:
         return
 
     click.echo("")
-    cluster = get_dcs(obj, cluster_name).get_cluster()
-    for m in cluster.members:
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
+    for m in get_all_members(obj, cluster, group, 'any'):
         if m.api_url:
             if not member_names or m.name in member_names:
                 try:
@@ -1294,10 +1313,11 @@ def version(obj, cluster_name, member_names):
 
 @ctl.command('history', help="Show the history of failovers/switchovers")
 @arg_cluster_name
+@option_default_citus_group
 @option_format
 @click.pass_obj
-def history(obj, cluster_name, fmt):
-    cluster = get_dcs(obj, cluster_name).get_cluster()
+def history(obj, cluster_name, group, fmt):
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
     history = cluster.history and cluster.history.lines or []
     table_header_row = ['TL', 'LSN', 'Reason', 'Timestamp', 'New Leader']
     for line in history:

patroni/daemon.py

@@ -1,16 +1,40 @@
+"""Daemon processes abstraction module.
+
+This module implements abstraction classes and functions for creating and managing daemon processes in Patroni.
+Currently it is only used for the main "Thread" of ``patroni`` and ``patroni_raft_controller`` commands.
+"""
+from __future__ import print_function
+
 import abc
 import os
 import signal
-import six
 import sys
 
 from threading import Lock
+from typing import Any, Optional, Type
+
+from .config import Config
+from .validator import Schema
 
 
-@six.add_metaclass(abc.ABCMeta)
-class AbstractPatroniDaemon(object):
+class AbstractPatroniDaemon(abc.ABC):
+    """A Patroni daemon process.
 
-    def __init__(self, config):
+    .. note::
+
+        When inheriting from :class:`AbstractPatroniDaemon` you are expected to define the methods :func:`_run_cycle`
+        to determine what it should do in each execution cycle, and :func:`_shutdown` to determine what it should do
+        when shutting down.
+
+    :ivar logger: log handler used by this daemon.
+    :ivar config: configuration options for this daemon.
+    """
+
+    def __init__(self, config: Config) -> None:
+        """Set up signal handlers, logging handler and configuration.
+
+        :param config: configuration options for this daemon.
+        """
         from patroni.log import PatroniLogger
 
         self.setup_signal_handlers()
@@ -19,16 +43,44 @@ class AbstractPatroniDaemon(object):
         self.config = config
         AbstractPatroniDaemon.reload_config(self, local=True)
 
-    def sighup_handler(self, *args):
+    def sighup_handler(self, *_: Any) -> None:
+        """Handle SIGHUP signals.
+
+        Flag the daemon as "SIGHUP received".
+        """
         self._received_sighup = True
 
-    def sigterm_handler(self, *args):
+    def api_sigterm(self) -> bool:
+        """Guarantee only a single SIGTERM is being processed.
+
+        Flag the daemon as "SIGTERM received" with a lock-based approach.
+
+        :returns: ``True`` if the daemon was flagged as "SIGTERM received".
+        """
+        ret = False
         with self._sigterm_lock:
             if not self._received_sigterm:
                 self._received_sigterm = True
-                sys.exit()
+                ret = True
+        return ret
 
-    def setup_signal_handlers(self):
+    def sigterm_handler(self, *_: Any) -> None:
+        """Handle SIGTERM signals.
+
+        Terminate the daemon process through :func:`api_sigterm`.
+        """
+        if self.api_sigterm():
+            sys.exit()
+
+    def setup_signal_handlers(self) -> None:
+        """Set up daemon signal handlers.
+
+        Set up SIGHUP and SIGTERM signal handlers.
+
+        .. note::
+
+            SIGHUP is only handled in non-Windows environments.
+        """
         self._received_sighup = False
         self._sigterm_lock = Lock()
         self._received_sigterm = False
@@ -37,19 +89,34 @@ class AbstractPatroniDaemon(object):
         signal.signal(signal.SIGTERM, self.sigterm_handler)
 
     @property
-    def received_sigterm(self):
+    def received_sigterm(self) -> bool:
+        """If daemon was signaled with SIGTERM."""
         with self._sigterm_lock:
             return self._received_sigterm
 
-    def reload_config(self, sighup=False, local=False):
+    def reload_config(self, sighup: Optional[bool] = False, local: Optional[bool] = False) -> None:
+        """Reload configuration.
+
+        :param sighup: if it is related to a SIGHUP signal.
+                       The sighup parameter could be used in the method overridden in a child class.
+        :param local: will be ``True`` if there are changes in the local configuration file.
+        """
         if local:
             self.logger.reload_config(self.config.get('log', {}))
 
     @abc.abstractmethod
-    def _run_cycle(self):
-        """_run_cycle"""
+    def _run_cycle(self) -> None:
+        """Define what the daemon should do in each execution cycle.
 
-    def run(self):
+        Keep being called in the daemon's main loop until the daemon is eventually terminated.
+        """
+
+    def run(self) -> None:
+        """Run the daemon process.
+
+        Start the logger thread and keep running execution cycles until a SIGTERM is eventually received. Also reload
+        configuration uppon receiving SIGHUP.
+        """
         self.logger.start()
         while not self.received_sigterm:
             if self._received_sighup:
@@ -59,17 +126,29 @@ class AbstractPatroniDaemon(object):
             self._run_cycle()
 
     @abc.abstractmethod
-    def _shutdown(self):
-        """_shutdown"""
+    def _shutdown(self) -> None:
+        """Define what the daemon should do when shutting down."""
 
-    def shutdown(self):
+    def shutdown(self) -> None:
+        """Shut the daemon down when a SIGTERM is received.
+
+        Shut down the daemon process and the logger thread.
+        """
         with self._sigterm_lock:
             self._received_sigterm = True
         self._shutdown()
         self.logger.shutdown()
 
 
-def abstract_main(cls, validator=None):
+def abstract_main(cls: Type[AbstractPatroniDaemon], validator: Optional[Schema] = None) -> None:
+    """Create the main entry point of a given daemon process.
+
+    Expose a basic argument parser, parse the command-line arguments, and run the given daemon process.
+
+    :param cls: a class that should inherit from :class:`AbstractPatroniDaemon`.
+    :param validator: used to validate the daemon configuration schema, if requested by the user through
+        ``--validate-config`` CLI option.
+    """
     import argparse
 
     from .config import Config, ConfigParseError
@@ -83,16 +162,18 @@ def abstract_main(cls, validator=None):
                         help='Patroni may also read the configuration from the {0} environment variable'
                         .format(Config.PATRONI_CONFIG_VARIABLE))
     args = parser.parse_args()
+    validate_config = validator and args.validate_config
     try:
-        if validator and args.validate_config:
+        if validate_config:
             Config(args.configfile, validator=validator)
             sys.exit()
 
         config = Config(args.configfile)
     except ConfigParseError as e:
         if e.value:
-            print(e.value)
-        parser.print_help()
+            print(e.value, file=sys.stderr)
+        if not validate_config:
+            parser.print_help()
         sys.exit(1)
 
     controller = cls(config)

patroni/dcs/__init__.py

@@ -7,19 +7,21 @@ import logging
 import os
 import pkgutil
 import re
-import six
 import sys
 import time
 
 from collections import defaultdict, namedtuple
 from copy import deepcopy
 from random import randint
-from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
 from threading import Event, Lock
+from typing import Any, Dict, List, Optional, Union
+from urllib.parse import urlparse, urlunparse, parse_qsl
 
 from ..exceptions import PatroniFatalException
 from ..utils import deep_compare, parse_bool, uri
 
+CITUS_COORDINATOR_GROUP_ID = 0
+citus_group_re = re.compile('^(0|[1-9][0-9]*)$')
 slot_name_re = re.compile('^[a-z0-9_]{1,63}$')
 logger = logging.getLogger(__name__)
 
@@ -94,6 +96,9 @@ def get_dcs(config):
                         # propagate some parameters
                         config[name].update({p: config[p] for p in ('namespace', 'name', 'scope', 'loop_wait',
                                              'patronictl', 'ttl', 'retry_timeout') if p in config})
+                        # From citus section we only need "group" parameter, but will propagate everything just in case.
+                        if isinstance(config.get('citus'), dict):
+                            config[name].update(config['citus'])
                         return item(config[name])
             except ImportError:
                 logger.debug('Failed to import %s', module_name)
@@ -224,8 +229,7 @@ class Member(namedtuple('Member', 'index,name,session,data')):
 
 
 class RemoteMember(Member):
-    """ Represents a remote master for a standby cluster
-    """
+    """Represents a remote member (typically a primary) for a standby cluster"""
     def __new__(cls, name, data):
         return super(RemoteMember, cls).__new__(cls, None, name, None, data)
 
@@ -278,7 +282,7 @@ class Leader(namedtuple('Leader', 'index,session,member')):
         version = self.member.version
         # 1.5.6 is the last version which doesn't expose checkpoint_after_promote: false
         if version and version > (1, 5, 6):
-            return self.data.get('role') == 'master' and 'checkpoint_after_promote' not in self.data
+            return self.data.get('role') in ('master', 'primary') and 'checkpoint_after_promote' not in self.data
 
 
 class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
@@ -371,7 +375,7 @@ class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
     """
 
     @staticmethod
-    def from_node(index, value):
+    def from_node(index: Union[str, int], value: Union[str, Dict[str, Any]]) -> 'SyncState':
         """
         >>> SyncState.from_node(1, None).leader is None
         True
@@ -386,27 +390,31 @@ class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
         >>> SyncState.from_node(1, {"leader": "leader"}).leader == "leader"
         True
         """
-        if isinstance(value, dict):
-            data = value
-        elif value:
-            try:
-                data = json.loads(value)
-                if not isinstance(data, dict):
-                    data = {}
-            except (TypeError, ValueError):
-                data = {}
-        else:
-            data = {}
-        return SyncState(index, data.get('leader'), data.get('sync_standby'))
+        try:
+            if value and isinstance(value, str):
+                value = json.loads(value)
+            if not isinstance(value, dict):
+                return SyncState.empty(index)
+            return SyncState(index, value.get('leader'), value.get('sync_standby'))
+        except (TypeError, ValueError):
+            return SyncState.empty(index)
+
+    @staticmethod
+    def empty(index: Optional[Union[str, int]] = '') -> 'SyncState':
+        return SyncState(index, None, '')
 
     @property
-    def members(self):
-        """ Returns sync_standby in list """
-        return self.sync_standby and self.sync_standby.split(',') or []
+    def is_empty(self) -> bool:
+        """:returns: True if /sync key doesn't have a leader"""
+        return self.leader is None
 
-    def matches(self, name):
-        """
-        Returns if a node name matches one of the nodes in the sync state
+    @property
+    def members(self) -> List[str]:
+        """:returns: sync_standby as list"""
+        return list(filter(lambda a: a, [s.strip() for s in self.sync_standby.split(',')])) if self.sync_standby else []
+
+    def matches(self, name: str) -> bool:
+        """:returns: True if a node name matches one of the nodes in the sync state (including leader)
 
         >>> s = SyncState(1, 'foo', 'bar,zoo')
         >>> s.matches('foo')
@@ -444,7 +452,8 @@ class TimelineHistory(namedtuple('TimelineHistory', 'index,value,lines')):
         return TimelineHistory(index, value, lines)
 
 
-class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,failover,sync,history,slots')):
+class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,'
+                                    'failover,sync,history,slots,failsafe,workers')):
 
     """Immutable object (namedtuple) which represents PostgreSQL cluster.
     Consists of the following fields:
@@ -458,7 +467,18 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,f
     :param sync: reference to `SyncState` object, last observed synchronous replication state.
     :param history: reference to `TimelineHistory` object
     :param slots: state of permanent logical replication slots on the primary in the format: {"slot_name": int}
-    """
+    :param failsafe: failsafe topology. Node is allowed to become the leader only if its name is found in this list.
+    :param workers: workers of the Citus cluster, optional. Format: {int(group): Cluster()}"""
+
+    def __new__(cls, *args):
+        # Make workers argument optional
+        if len(cls._fields) == len(args) + 1:
+            args = args + ({},)
+        return super(Cluster, cls).__new__(cls, *args)
+
+    @staticmethod
+    def empty():
+        return Cluster(None, None, None, 0, [], None, SyncState.empty(), None, None, None)
 
     @property
     def leader_name(self):
@@ -507,16 +527,16 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,f
 
     def get_replication_slots(self, my_name, role, nofailover, major_version, show_error=False):
         # if the replicatefrom tag is set on the member - we should not create the replication slot for it on
-        # the current master, because that member would replicate from elsewhere. We still create the slot if
+        # the current primary, because that member would replicate from elsewhere. We still create the slot if
         # the replicatefrom destination member is currently not a member of the cluster (fallback to the
-        # master), or if replicatefrom destination member happens to be the current master
+        # primary), or if replicatefrom destination member happens to be the current primary
         use_slots = self.use_slots
-        if role in ('master', 'standby_leader'):
+        if role in ('master', 'primary', 'standby_leader'):
             slot_members = [m.name for m in self.members if use_slots and m.name != my_name and
                             (m.replicatefrom is None or m.replicatefrom == my_name or
                              not self.has_member(m.replicatefrom))]
             permanent_slots = self.__permanent_slots if use_slots and \
-                role == 'master' else self.__permanent_physical_slots
+                role in ('master', 'primary') else self.__permanent_physical_slots
         else:
             # only manage slots for replicas that replicate from this one, except for the leader among them
             slot_members = [m.name for m in self.members if use_slots and
@@ -606,11 +626,11 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,f
     @property
     def timeline(self):
         """
-        >>> Cluster(0, 0, 0, 0, 0, 0, 0, 0, 0).timeline
+        >>> Cluster(0, 0, 0, 0, 0, 0, 0, 0, 0, None).timeline
         0
-        >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[]'), 0).timeline
+        >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[]'), 0, None).timeline
         1
-        >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[["a"]]'), 0).timeline
+        >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[["a"]]'), 0, None).timeline
         0
         """
         if self.history:
@@ -628,8 +648,21 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,f
         return next(iter(sorted(filter(lambda v: v, [m.version for m in self.members])) + [None]))
 
 
-@six.add_metaclass(abc.ABCMeta)
-class AbstractDCS(object):
+class ReturnFalseException(Exception):
+    pass
+
+
+def catch_return_false_exception(func):
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except ReturnFalseException:
+            return False
+
+    return wrapper
+
+
+class AbstractDCS(abc.ABC):
 
     _INITIALIZE = 'initialize'
     _CONFIG = 'config'
@@ -641,6 +674,7 @@ class AbstractDCS(object):
     _STATUS = 'status'  # JSON, contains "leader_lsn" and confirmed_flush_lsn of logical "slots" on the leader
     _LEADER_OPTIME = _OPTIME + '/' + _LEADER  # legacy
     _SYNC = 'sync'
+    _FAILSAFE = 'failsafe'
 
     def __init__(self, config):
         """
@@ -649,6 +683,7 @@ class AbstractDCS(object):
         """
         self._name = config['name']
         self._base_path = re.sub('/+', '/', '/'.join(['', config.get('namespace', 'service'), config['scope']]))
+        self._citus_group = str(config['group']) if isinstance(config.get('group'), int) else None
         self._set_loop_wait(config.get('loop_wait', 10))
 
         self._ctl = bool(config.get('patronictl', False))
@@ -658,10 +693,15 @@ class AbstractDCS(object):
         self._last_lsn = ''
         self._last_seen = 0
         self._last_status = {}
+        self._last_failsafe = {}
         self.event = Event()
 
     def client_path(self, path):
-        return '/'.join([self._base_path, path.lstrip('/')])
+        components = [self._base_path]
+        if self._citus_group:
+            components.append(self._citus_group)
+        components.append(path.lstrip('/'))
+        return '/'.join(components)
 
     @property
     def initialize_path(self):
@@ -703,6 +743,10 @@ class AbstractDCS(object):
     def sync_path(self):
         return self.client_path(self._SYNC)
 
+    @property
+    def failsafe_path(self):
+        return self.client_path(self._FAILSAFE)
+
     @abc.abstractmethod
     def set_ttl(self, ttl):
         """Set the new ttl value for leader key"""
@@ -732,28 +776,69 @@ class AbstractDCS(object):
         return self._last_seen
 
     @abc.abstractmethod
-    def _load_cluster(self):
-        """Internally this method should build  `Cluster` object which
-           represents current state and topology of the cluster in DCS.
-           this method supposed to be called only by `get_cluster` method.
+    def _cluster_loader(self, path):
+        """Load and build the `Cluster` object from DCS, which
+        represents a single Patroni cluster.
 
-           raise `~DCSError` in case of communication or other problems with DCS.
-           If the current node was running as a master and exception raised,
-           instance would be demoted."""
+        :param path: the path in DCS where to load Cluster(s) from.
+        :returns: `Cluster`"""
+
+    def _citus_cluster_loader(self, path):
+        """Load and build `Cluster` onjects from DCS that represent all
+        Patroni clusters from a single Citus cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+        :returns: all Citus groups as `dict`, with group ids as keys"""
+
+    @abc.abstractmethod
+    def _load_cluster(self, path, loader):
+        """Internally this method should call the `loader` method that
+        will build `Cluster` object which represents current state and
+        topology of the cluster in DCS. This method supposed to be
+        called only by `get_cluster` method.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+        :param loader: one of `_cluster_loader` or `_citus_cluster_loader`
+        :raise: `~DCSError` in case of communication problems with DCS.
+        If the current node was running as a primary and exception
+        raised, instance would be demoted."""
 
     def _bypass_caches(self):
         """Used only in zookeeper"""
 
+    def is_citus_coordinator(self):
+        return self._citus_group == str(CITUS_COORDINATOR_GROUP_ID)
+
+    def get_citus_coordinator(self):
+        try:
+            path = '{0}/{1}/'.format(self._base_path, CITUS_COORDINATOR_GROUP_ID)
+            return self._load_cluster(path, self._cluster_loader)
+        except Exception as e:
+            logger.error('Failed to load Citus coordinator cluster from %s: %r', self.__class__.__name__, e)
+
+    def _get_citus_cluster(self):
+        groups = self._load_cluster(self._base_path + '/', self._citus_cluster_loader)
+        if isinstance(groups, Cluster):  # Zookeeper could return a cached version
+            cluster = groups
+        else:
+            assert isinstance(groups, dict)
+            cluster = groups.pop(CITUS_COORDINATOR_GROUP_ID, Cluster.empty())
+            cluster.workers.update(groups)
+        return cluster
+
     def get_cluster(self, force=False):
         if force:
             self._bypass_caches()
         try:
-            cluster = self._load_cluster()
+            cluster = self._get_citus_cluster() if self.is_citus_coordinator()\
+                else self._load_cluster(self.client_path(''), self._cluster_loader)
         except Exception:
             self.reset_cluster()
             raise
 
         self._last_seen = int(time.time())
+        self._last_status = {self._OPTIME: cluster.last_lsn, 'slots': cluster.slots}
+        self._last_failsafe = cluster.failsafe
 
         with self._cluster_thread_lock:
             self._cluster = cluster
@@ -796,23 +881,39 @@ class AbstractDCS(object):
             self._last_lsn = value[self._OPTIME]
             self._write_leader_optime(str(value[self._OPTIME]))
 
+    @abc.abstractmethod
+    def _write_failsafe(self, value):
+        """Write current cluster topology to DCS that will be used by failsafe mechanism (if enabled).
+
+        :param value: failsafe topology serialized in JSON format
+        :returns: `!True` on success."""
+
+    def write_failsafe(self, value):
+        if not (isinstance(self._last_failsafe, dict) and deep_compare(self._last_failsafe, value))\
+                and self._write_failsafe(json.dumps(value, separators=(',', ':'))):
+            self._last_failsafe = value
+
+    @property
+    def failsafe(self):
+        return self._last_failsafe
+
     @abc.abstractmethod
     def _update_leader(self):
         """Update leader key (or session) ttl
 
         :returns: `!True` if leader key (or session) has been updated successfully.
-            If not, `!False` must be returned and current instance would be demoted.
 
         You have to use CAS (Compare And Swap) operation in order to update leader key,
-        for example for etcd `prevValue` parameter must be used."""
+        for example for etcd `prevValue` parameter must be used.
+        If update fails due to DCS not being accessible or because it is not able to
+        process requests (hopefuly temporary), the ~DCSError exception should be raised."""
 
-    def update_leader(self, last_lsn, slots=None):
+    def update_leader(self, last_lsn, slots=None, failsafe=None):
         """Update leader key (or session) ttl and optime/leader
 
         :param last_lsn: absolute WAL LSN in bytes
         :param slots: dict with permanent slots confirmed_flush_lsn
-        :returns: `!True` if leader key (or session) has been updated successfully.
-            If not, `!False` must be returned and current instance would be demoted."""
+        :returns: `!True` if leader key (or session) has been updated successfully."""
 
         ret = self._update_leader()
         if ret and last_lsn:
@@ -820,18 +921,23 @@ class AbstractDCS(object):
             if slots:
                 status['slots'] = slots
             self.write_status(status)
+
+        if ret and failsafe is not None:
+            self.write_failsafe(failsafe)
+
         return ret
 
     @abc.abstractmethod
-    def attempt_to_acquire_leader(self, permanent=False):
+    def attempt_to_acquire_leader(self):
         """Attempt to acquire leader lock
         This method should create `/leader` key with value=`~self._name`
-        :param permanent: if set to `!True`, the leader key will never expire.
-         Used in patronictl for the external master
         :returns: `!True` if key has been created successfully.
 
         Key must be created atomically. In case if key already exists it should not be
-        overwritten and `!False` must be returned"""
+        overwritten and `!False` must be returned.
+
+        If key creation fails due to DCS not being accessible or because it is not able to
+        process requests (hopefuly temporary), the ~DCSError exception should be raised"""
 
     @abc.abstractmethod
     def set_failover_value(self, value, index=None):
@@ -854,15 +960,13 @@ class AbstractDCS(object):
         """Create or update `/config` key"""
 
     @abc.abstractmethod
-    def touch_member(self, data, permanent=False):
+    def touch_member(self, data):
         """Update member key in DCS.
         This method should create or update key with the name = '/members/' + `~self._name`
         and value = data in a given DCS.
 
         :param data: information about instance (including connection strings)
         :param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used`
-        :param permanent: if set to `!True`, the member key will never expire.
-         Used in patronictl for the external master.
         :returns: `!True` on success otherwise `!False`
         """
 
@@ -929,7 +1033,7 @@ class AbstractDCS(object):
         """"""
 
     def watch(self, leader_index, timeout):
-        """If the current node is a master it should just sleep.
+        """If the current node is a leader it should just sleep.
         Any other node should watch for changes of leader key with a given timeout
 
         :param leader_index: index of a leader key

patroni/dcs/consul.py

@@ -8,13 +8,14 @@ import ssl
 import time
 import urllib3
 
-from collections import namedtuple
+from collections import defaultdict, namedtuple
 from consul import ConsulException, NotFound, base
+from http.client import HTTPException
 from urllib3.exceptions import HTTPError
-from six.moves.urllib.parse import urlencode, urlparse, quote
-from six.moves.http_client import HTTPException
+from urllib.parse import urlencode, urlparse, quote
 
-from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState, TimelineHistory
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
+        TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
 from ..exceptions import DCSError
 from ..utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port, uri, USER_AGENT
 
@@ -189,6 +190,7 @@ class Consul(AbstractDCS):
 
     def __init__(self, config):
         super(Consul, self).__init__(config)
+        self._base_path = self._base_path[1:]
         self._scope = config['scope']
         self._session = None
         self.__do_not_watch = False
@@ -236,6 +238,7 @@ class Consul(AbstractDCS):
         self._service_check_tls_server_name = config.get('service_check_tls_server_name', None)
         if not self._ctl:
             self.create_session()
+        self._previous_loop_token = self._client.token
 
     def retry(self, *args, **kwargs):
         return self._retry.copy()(*args, **kwargs)
@@ -270,7 +273,7 @@ class Consul(AbstractDCS):
 
     @property
     def ttl(self):
-        return self._client.http.ttl
+        return self._client.http.ttl * 2  # we multiply the value by 2 because it was divided in the `set_ttl()` method
 
     def set_retry_timeout(self, retry_timeout):
         self._retry.deadline = retry_timeout
@@ -285,9 +288,9 @@ class Consul(AbstractDCS):
         except Exception:
             logger.exception('adjust_ttl')
 
-    def _do_refresh_session(self):
+    def _do_refresh_session(self, force=False):
         """:returns: `!True` if it had to create new session"""
-        if self._session and self._last_session_refresh + self._loop_wait > time.time():
+        if not force and self._session and self._last_session_refresh + self._loop_wait > time.time():
             return False
 
         if self._session:
@@ -316,96 +319,108 @@ class Consul(AbstractDCS):
             logger.exception('refresh_session')
         raise ConsulError('Failed to renew/create session')
 
-    def client_path(self, path):
-        return super(Consul, self).client_path(path)[1:]
-
     @staticmethod
     def member(node):
         return Member.from_node(node['ModifyIndex'], os.path.basename(node['Key']), node.get('Session'), node['Value'])
 
-    def _load_cluster(self):
-        try:
-            path = self.client_path('/')
-            _, results = self.retry(self._client.kv.get, path, recurse=True)
+    def _cluster_from_nodes(self, nodes):
+        # get initialize flag
+        initialize = nodes.get(self._INITIALIZE)
+        initialize = initialize and initialize['Value']
 
-            if results is None:
-                raise NotFound
+        # get global dynamic configuration
+        config = nodes.get(self._CONFIG)
+        config = config and ClusterConfig.from_node(config['ModifyIndex'], config['Value'])
 
-            nodes = {}
-            for node in results:
-                node['Value'] = (node['Value'] or b'').decode('utf-8')
-                nodes[node['Key'][len(path):].lstrip('/')] = node
-
-            # get initialize flag
-            initialize = nodes.get(self._INITIALIZE)
-            initialize = initialize and initialize['Value']
-
-            # get global dynamic configuration
-            config = nodes.get(self._CONFIG)
-            config = config and ClusterConfig.from_node(config['ModifyIndex'], config['Value'])
-
-            # get timeline history
-            history = nodes.get(self._HISTORY)
-            history = history and TimelineHistory.from_node(history['ModifyIndex'], history['Value'])
-
-            # get last known leader lsn and slots
-            status = nodes.get(self._STATUS)
-            if status:
-                try:
-                    status = json.loads(status['Value'])
-                    last_lsn = status.get(self._OPTIME)
-                    slots = status.get('slots')
-                except Exception:
-                    slots = last_lsn = None
-            else:
-                last_lsn = nodes.get(self._LEADER_OPTIME)
-                last_lsn = last_lsn and last_lsn['Value']
-                slots = None
+        # get timeline history
+        history = nodes.get(self._HISTORY)
+        history = history and TimelineHistory.from_node(history['ModifyIndex'], history['Value'])
 
+        # get last known leader lsn and slots
+        status = nodes.get(self._STATUS)
+        if status:
             try:
-                last_lsn = int(last_lsn)
+                status = json.loads(status['Value'])
+                last_lsn = status.get(self._OPTIME)
+                slots = status.get('slots')
             except Exception:
-                last_lsn = 0
+                slots = last_lsn = None
+        else:
+            last_lsn = nodes.get(self._LEADER_OPTIME)
+            last_lsn = last_lsn and last_lsn['Value']
+            slots = None
 
-            # get list of members
-            members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
+        try:
+            last_lsn = int(last_lsn)
+        except Exception:
+            last_lsn = 0
 
-            # get leader
-            leader = nodes.get(self._LEADER)
-            if not self._ctl and leader and leader['Value'] == self._name \
-                    and self._session != leader.get('Session', 'x'):
-                logger.info('I am leader but not owner of the session. Removing leader node')
-                self._client.kv.delete(self.leader_path, cas=leader['ModifyIndex'])
-                leader = None
+        # get list of members
+        members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
 
-            if leader:
-                member = Member(-1, leader['Value'], None, {})
-                member = ([m for m in members if m.name == leader['Value']] or [member])[0]
-                leader = Leader(leader['ModifyIndex'], leader.get('Session'), member)
+        # get leader
+        leader = nodes.get(self._LEADER)
 
-            # failover key
-            failover = nodes.get(self._FAILOVER)
-            if failover:
-                failover = Failover.from_node(failover['ModifyIndex'], failover['Value'])
+        if leader:
+            member = Member(-1, leader['Value'], None, {})
+            member = ([m for m in members if m.name == leader['Value']] or [member])[0]
+            leader = Leader(leader['ModifyIndex'], leader.get('Session'), member)
 
-            # get synchronization state
-            sync = nodes.get(self._SYNC)
-            sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
+        # failover key
+        failover = nodes.get(self._FAILOVER)
+        if failover:
+            failover = Failover.from_node(failover['ModifyIndex'], failover['Value'])
 
-            return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+        # get synchronization state
+        sync = nodes.get(self._SYNC)
+        sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
+
+        # get failsafe topology
+        failsafe = nodes.get(self._FAILSAFE)
+        try:
+            failsafe = json.loads(failsafe['Value']) if failsafe else None
+        except Exception:
+            failsafe = None
+
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+
+    def _cluster_loader(self, path):
+        _, results = self.retry(self._client.kv.get, path, recurse=True)
+        if results is None:
+            raise NotFound
+        nodes = {}
+        for node in results:
+            node['Value'] = (node['Value'] or b'').decode('utf-8')
+            nodes[node['Key'][len(path):]] = node
+
+        return self._cluster_from_nodes(nodes)
+
+    def _citus_cluster_loader(self, path):
+        _, results = self.retry(self._client.kv.get, path, recurse=True)
+        clusters = defaultdict(dict)
+        for node in results or []:
+            key = node['Key'][len(path):].split('/', 1)
+            if len(key) == 2 and citus_group_re.match(key[0]):
+                node['Value'] = (node['Value'] or b'').decode('utf-8')
+                clusters[int(key[0])][key[1]] = node
+        return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()}
+
+    def _load_cluster(self, path, loader):
+        try:
+            return loader(path)
         except NotFound:
-            return Cluster(None, None, None, None, [], None, None, None, None)
+            return Cluster.empty()
         except Exception:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')
 
     @catch_consul_errors
-    def touch_member(self, data, permanent=False):
+    def touch_member(self, data):
         cluster = self.cluster
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
 
         try:
-            create_member = not permanent and self.refresh_session()
+            create_member = self.refresh_session()
         except DCSError:
             return False
 
@@ -423,8 +438,7 @@ class Consul(AbstractDCS):
             return True
 
         try:
-            args = {} if permanent else {'acquire': self._session}
-            self._client.kv.put(self.member_path, json.dumps(data, separators=(',', ':')), **args)
+            self._client.kv.put(self.member_path, json.dumps(data, separators=(',', ':')), acquire=self._session)
             return True
         except InvalidSession:
             self._session = None
@@ -463,7 +477,12 @@ class Consul(AbstractDCS):
             check['TLSServerName'] = self._service_check_tls_server_name
         tags = self._service_tags[:]
         tags.append(role)
+        if role == 'master':
+            tags.append('primary')
+        elif role == 'primary':
+            tags.append('master')
         self._previous_loop_service_tags = self._service_tags
+        self._previous_loop_token = self._client.token
 
         params = {
             'service_id': '{0}/{1}'.format(self._scope, self._name),
@@ -479,7 +498,7 @@ class Consul(AbstractDCS):
             return self.deregister_service(params['service_id'])
 
         self._previous_loop_register_service = self._register_service
-        if role in ['master', 'replica', 'standby-leader']:
+        if role in ['master', 'primary', 'replica', 'standby-leader']:
             if state != 'running':
                 return
             return self.register_service(service_name, **params)
@@ -500,25 +519,36 @@ class Consul(AbstractDCS):
         if (
             force or update or self._register_service != self._previous_loop_register_service
             or self._service_tags != self._previous_loop_service_tags
+            or self._client.token != self._previous_loop_token
         ):
             return self._update_service(new_data)
 
-    @catch_consul_errors
-    def _do_attempt_to_acquire_leader(self, permanent):
+    def _do_attempt_to_acquire_leader(self, retry):
         try:
-            kwargs = {} if permanent else {'acquire': self._session}
-            return self.retry(self._client.kv.put, self.leader_path, self._name, **kwargs)
+            return retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session)
         except InvalidSession:
-            self._session = None
             logger.error('Our session disappeared from Consul. Will try to get a new one and retry attempt')
-            self.refresh_session()
-            return self.retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session)
+            self._session = None
+            retry.deadline = retry.stoptime - time.time()
 
-    def attempt_to_acquire_leader(self, permanent=False):
-        if not self._session and not permanent:
-            self.refresh_session()
+            retry(self._do_refresh_session)
 
-        ret = self._do_attempt_to_acquire_leader(permanent)
+            retry.deadline = retry.stoptime - time.time()
+            if retry.deadline < 1:
+                raise ConsulError('_do_attempt_to_acquire_leader timeout')
+
+            return retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session)
+
+    @catch_return_false_exception
+    def attempt_to_acquire_leader(self):
+        retry = self._retry.copy()
+        self._run_and_handle_exceptions(self._do_refresh_session, retry=retry)
+
+        retry.deadline = retry.stoptime - time.time()
+        if retry.deadline < 1:
+            raise ConsulError('attempt_to_acquire_leader timeout')
+
+        ret = self._run_and_handle_exceptions(self._do_attempt_to_acquire_leader, retry, retry=None)
         if not ret:
             logger.info('Could not take out TTL lock')
 
@@ -544,10 +574,42 @@ class Consul(AbstractDCS):
         return self._client.kv.put(self.status_path, value)
 
     @catch_consul_errors
+    def _write_failsafe(self, value):
+        return self._client.kv.put(self.failsafe_path, value)
+
+    @staticmethod
+    def _run_and_handle_exceptions(method, *args, **kwargs):
+        retry = kwargs.pop('retry', None)
+        try:
+            return retry(method, *args, **kwargs) if retry else method(*args, **kwargs)
+        except (RetryFailedError, InvalidSession, HTTPException, HTTPError, socket.error, socket.timeout) as e:
+            raise ConsulError(e)
+        except ConsulException:
+            raise ReturnFalseException
+
+    @catch_return_false_exception
     def _update_leader(self):
+        retry = self._retry.copy()
+
+        self._run_and_handle_exceptions(self._do_refresh_session, True, retry=retry)
+
         if self._session:
-            self.retry(self._client.session.renew, self._session)
-            self._last_session_refresh = time.time()
+            cluster = self.cluster
+            leader_session = cluster and isinstance(cluster.leader, Leader) and cluster.leader.session
+            if leader_session != self._session:
+                retry.deadline = retry.stoptime - time.time()
+                if retry.deadline < 1:
+                    raise ConsulError('update_leader timeout')
+                logger.warning('Recreating the leader key due to session mismatch')
+                if cluster.leader:
+                    self._run_and_handle_exceptions(self._client.kv.delete, self.leader_path, cas=cluster.leader.index)
+
+                retry.deadline = retry.stoptime - time.time()
+                if retry.deadline < 0.5:
+                    raise ConsulError('update_leader timeout')
+                self._run_and_handle_exceptions(self._client.kv.put, self.leader_path,
+                                                self._name, acquire=self._session)
+
         return bool(self._session)
 
     @catch_consul_errors

patroni/dcs/etcd.py

@@ -6,20 +6,23 @@ import logging
 import os
 import urllib3.util.connection
 import random
-import six
 import socket
 import time
 
+from collections import defaultdict
+from copy import deepcopy
 from dns.exception import DNSException
 from dns import resolver
+from http.client import HTTPException
+from queue import Queue
+from threading import Thread
+from typing import List, Optional
+from urllib.parse import urlparse
 from urllib3 import Timeout
 from urllib3.exceptions import HTTPError, ReadTimeoutError, ProtocolError
-from six.moves.queue import Queue
-from six.moves.http_client import HTTPException
-from six.moves.urllib_parse import urlparse
-from threading import Thread
 
-from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState, TimelineHistory
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
+        TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
 from ..exceptions import DCSError
 from ..request import get as requests_get
 from ..utils import Retry, RetryFailedError, split_host_port, uri, USER_AGENT
@@ -83,8 +86,7 @@ class DnsCachingResolver(Thread):
             return []
 
 
-@six.add_metaclass(abc.ABCMeta)
-class AbstractEtcdClientWithFailover(etcd.Client):
+class AbstractEtcdClientWithFailover(abc.ABC, etcd.Client):
 
     def __init__(self, config, dns_resolver, cache_ttl=300):
         self._dns_resolver = dns_resolver
@@ -97,7 +99,6 @@ class AbstractEtcdClientWithFailover(etcd.Client):
         # Workaround for the case when https://github.com/jplana/python-etcd/pull/196 is not applied
         self.http.connection_pool_kw.pop('ssl_version', None)
         self._config = config
-        self._initial_machines_cache = []
         self._load_machines_cache()
         self._allow_reconnect = True
         # allow passing retry argument to api_execute in params
@@ -167,19 +168,12 @@ class AbstractEtcdClientWithFailover(etcd.Client):
         base_uri, cache = self._base_uri, self._machines_cache
         return ([base_uri] if base_uri in cache else []) + [machine for machine in cache if machine != base_uri]
 
-    @property
-    def machines(self):
-        """Original `machines` method(property) of `etcd.Client` class raise exception
-        when it failed to get list of etcd cluster members. This method is being called
-        only when request failed on one of the etcd members during `api_execute` call.
-        For us it's more important to execute original request rather then get new topology
-        of etcd cluster. So we will catch this exception and return empty list of machines.
-        Later, during next `api_execute` call we will forcefully update machines_cache.
+    def _get_machines_list(self, machines_cache: List[str]) -> List[str]:
+        """Gets list of members from Etcd cluster using API
 
-        Also this method implements the same timeout-retry logic as `api_execute`, because
-        the original method was retrying 2 times with the `read_timeout` on each node."""
-
-        machines_cache = self.machines_cache
+        :param machines_cache: initial list of Etcd members
+        :returns: list of clientURLs retrieved from Etcd cluster
+        :raises EtcdConnectionFailed: if failed"""
         kwargs = self._prepare_get_members(len(machines_cache))
 
         for base_uri in machines_cache:
@@ -197,6 +191,22 @@ class AbstractEtcdClientWithFailover(etcd.Client):
 
         raise etcd.EtcdConnectionFailed('No more machines in the cluster')
 
+    @property
+    def machines(self) -> List[str]:
+        """Original `machines` method(property) of `etcd.Client` class raise exception
+        when it failed to get list of etcd cluster members. This method is being called
+        only when request failed on one of the etcd members during `api_execute` call.
+        For us it's more important to execute original request rather then get new topology
+        of etcd cluster. So we will catch this exception and return empty list of machines.
+        Later, during next `api_execute` call we will forcefully update machines_cache.
+
+        Also this method implements the same timeout-retry logic as `api_execute`, because
+        the original method was retrying 2 times with the `read_timeout` on each node.
+
+        After the next refactoring the whole logic was moved to the _get_machines_list() method."""
+
+        return self._get_machines_list(self.machines_cache)
+
     def set_read_timeout(self, timeout):
         self._read_timeout = timeout
 
@@ -216,10 +226,13 @@ class AbstractEtcdClientWithFailover(etcd.Client):
                 return response
             except (HTTPError, HTTPException, socket.error, socket.timeout) as e:
                 self.http.clear()
-                # switch to the next etcd node because we don't know exactly what happened,
-                # whether the key didn't received an update or there is a network problem.
-                if not retry and i + 1 < len(machines_cache):
-                    self.set_base_uri(machines_cache[i + 1])
+                if not retry:
+                    if len(machines_cache) == 1:
+                        self.set_base_uri(self._base_uri)  # trigger Etcd3 watcher restart
+                    # switch to the next etcd node because we don't know exactly what happened,
+                    # whether the key didn't received an update or there is a network problem.
+                    elif i + 1 < len(machines_cache):
+                        self.set_base_uri(machines_cache[i + 1])
                 if (isinstance(fields, dict) and fields.get("wait") == "true" and
                         isinstance(e, (ReadTimeoutError, ProtocolError))):
                     logger.debug("Watch timed out.")
@@ -363,36 +376,46 @@ class AbstractEtcdClientWithFailover(etcd.Client):
         # enforce resolving dns name,they might get new ips
         self._update_dns_cache(self._dns_resolver.remove, machines_cache)
 
-        # The etcd cluster could change its topology over time and depending on how we resolve the initial
-        # topology (list of hosts in the Patroni config or DNS records, A or SRV) we might get into the situation
-        # the the real topology doesn't match anymore with the topology resolved from the configuration file.
-        # In case if the "initial" topology is the same as before we will not override the `_machines_cache`.
-        ret = set(machines_cache) != set(self._initial_machines_cache)
-        if ret:
-            self._initial_machines_cache = self._machines_cache = machines_cache
-
-            # After filling up the initial list of machines_cache we should ask etcd-cluster about actual list
-            self._refresh_machines_cache(True)
+        # after filling up the initial list of machines_cache we should ask etcd-cluster about actual list
+        ret = self._refresh_machines_cache(machines_cache)
 
         self._update_machines_cache = False
         return ret
 
-    def _refresh_machines_cache(self, updating_cache=False):
+    def _refresh_machines_cache(self, machines_cache: Optional[List[str]] = None) -> bool:
+        """Get etcd cluster topology using Etcd API and put it to self._machines_cache
+
+        :param machines_cache: the list of nodes we want to run through executing API request
+                               in addition to values stored in the self._machines_cache
+        :returns: `True` if self._machines_cache was updated with new values
+        :raises EtcdException: if failed to get topology and `machines_cache` was specified.
+
+        The self._machines_cache will not be updated if nodes from the list are
+        not accessible or if they are not returning correct results."""
+
         if self._use_proxies:
-            self._machines_cache = self._get_machines_cache_from_config()
+            value = self._get_machines_cache_from_config()
         else:
             try:
-                self._machines_cache = self.machines
+                # we want to go through the list obtained from the config file + last known health topology
+                value = self._get_machines_list(list(set((machines_cache or []) + self.machines_cache)))
             except etcd.EtcdConnectionFailed:
-                if updating_cache:
-                    raise etcd.EtcdException("Could not get the list of servers, "
-                                             "maybe you provided the wrong "
-                                             "host(s) to connect to?")
-                return
+                value = []
+
+        if value:
+            ret = set(self._machines_cache) != set(value)
+            self._machines_cache = value
+        elif machines_cache:  # we are just starting or all nodes were not available at some point
+            raise etcd.EtcdException("Could not get the list of servers, "
+                                     "maybe you provided the wrong "
+                                     "host(s) to connect to?")
+        else:
+            return False
 
         if self._base_uri not in self._machines_cache:
             self.set_base_uri(self._machines_cache[0])
         self._machines_cache_updated = time.time()
+        return ret
 
     def set_base_uri(self, value):
         if self._base_uri != value:
@@ -457,6 +480,18 @@ class AbstractEtcd(AbstractDCS):
         if isinstance(raise_ex, Exception):
             raise raise_ex
 
+    def _run_and_handle_exceptions(self, method, *args, **kwargs):
+        retry = kwargs.pop('retry', self.retry)
+        try:
+            return retry(method, *args, **kwargs) if retry else method(*args, **kwargs)
+        except (RetryFailedError, etcd.EtcdConnectionFailed) as e:
+            raise self._client.ERROR_CLS(e)
+        except etcd.EtcdException as e:
+            self._handle_exception(e)
+            raise ReturnFalseException
+        except Exception as e:
+            self._handle_exception(e, raise_ex=self._client.ERROR_CLS('unexpected error'))
+
     @staticmethod
     def set_socket_options(sock, socket_options):
         if socket_options:
@@ -464,6 +499,7 @@ class AbstractEtcd(AbstractDCS):
                 sock.setsockopt(*opt)
 
     def get_etcd_client(self, config, client_cls):
+        config = deepcopy(config)
         if 'proxy' in config:
             config['use_proxies'] = True
             config['url'] = config['proxy']
@@ -477,12 +513,12 @@ class AbstractEtcd(AbstractDCS):
             default_port = config.pop('port', 2379)
             protocol = config.get('protocol', 'http')
 
-            if isinstance(hosts, six.string_types):
+            if isinstance(hosts, str):
                 hosts = hosts.split(',')
 
             config['hosts'] = []
             for value in hosts:
-                if isinstance(value, six.string_types):
+                if isinstance(value, str):
                     config['hosts'].append(uri(protocol, split_host_port(value.strip(), default_port)))
         elif 'host' in config:
             host, port = split_host_port(config['host'], 2379)
@@ -588,92 +624,111 @@ class Etcd(AbstractEtcd):
     def member(node):
         return Member.from_node(node.modifiedIndex, os.path.basename(node.key), node.ttl, node.value)
 
-    def _load_cluster(self):
+    def _cluster_from_nodes(self, etcd_index, nodes):
+        # get initialize flag
+        initialize = nodes.get(self._INITIALIZE)
+        initialize = initialize and initialize.value
+
+        # get global dynamic configuration
+        config = nodes.get(self._CONFIG)
+        config = config and ClusterConfig.from_node(config.modifiedIndex, config.value)
+
+        # get timeline history
+        history = nodes.get(self._HISTORY)
+        history = history and TimelineHistory.from_node(history.modifiedIndex, history.value)
+
+        # get last know leader lsn and slots
+        status = nodes.get(self._STATUS)
+        if status:
+            try:
+                status = json.loads(status.value)
+                last_lsn = status.get(self._OPTIME)
+                slots = status.get('slots')
+            except Exception:
+                slots = last_lsn = None
+        else:
+            last_lsn = nodes.get(self._LEADER_OPTIME)
+            last_lsn = last_lsn and last_lsn.value
+            slots = None
+
+        try:
+            last_lsn = int(last_lsn)
+        except Exception:
+            last_lsn = 0
+
+        # get list of members
+        members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
+
+        # get leader
+        leader = nodes.get(self._LEADER)
+        if leader:
+            member = Member(-1, leader.value, None, {})
+            member = ([m for m in members if m.name == leader.value] or [member])[0]
+            index = etcd_index if etcd_index > leader.modifiedIndex else leader.modifiedIndex + 1
+            leader = Leader(index, leader.ttl, member)
+
+        # failover key
+        failover = nodes.get(self._FAILOVER)
+        if failover:
+            failover = Failover.from_node(failover.modifiedIndex, failover.value)
+
+        # get synchronization state
+        sync = nodes.get(self._SYNC)
+        sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
+
+        # get failsafe topology
+        failsafe = nodes.get(self._FAILSAFE)
+        try:
+            failsafe = json.loads(failsafe.value) if failsafe else None
+        except Exception:
+            failsafe = None
+
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+
+    def _cluster_loader(self, path):
+        result = self.retry(self._client.read, path, recursive=True)
+        nodes = {node.key[len(result.key):].lstrip('/'): node for node in result.leaves}
+        return self._cluster_from_nodes(result.etcd_index, nodes)
+
+    def _citus_cluster_loader(self, path):
+        clusters = defaultdict(dict)
+        result = self.retry(self._client.read, path, recursive=True)
+        for node in result.leaves:
+            key = node.key[len(result.key):].lstrip('/').split('/', 1)
+            if len(key) == 2 and citus_group_re.match(key[0]):
+                clusters[int(key[0])][key[1]] = node
+        return {group: self._cluster_from_nodes(result.etcd_index, nodes) for group, nodes in clusters.items()}
+
+    def _load_cluster(self, path, loader):
         cluster = None
         try:
-            result = self.retry(self._client.read, self.client_path(''), recursive=True)
-            nodes = {node.key[len(result.key):].lstrip('/'): node for node in result.leaves}
-
-            # get initialize flag
-            initialize = nodes.get(self._INITIALIZE)
-            initialize = initialize and initialize.value
-
-            # get global dynamic configuration
-            config = nodes.get(self._CONFIG)
-            config = config and ClusterConfig.from_node(config.modifiedIndex, config.value)
-
-            # get timeline history
-            history = nodes.get(self._HISTORY)
-            history = history and TimelineHistory.from_node(history.modifiedIndex, history.value)
-
-            # get last know leader lsn and slots
-            status = nodes.get(self._STATUS)
-            if status:
-                try:
-                    status = json.loads(status.value)
-                    last_lsn = status.get(self._OPTIME)
-                    slots = status.get('slots')
-                except Exception:
-                    slots = last_lsn = None
-            else:
-                last_lsn = nodes.get(self._LEADER_OPTIME)
-                last_lsn = last_lsn and last_lsn.value
-                slots = None
-
-            try:
-                last_lsn = int(last_lsn)
-            except Exception:
-                last_lsn = 0
-
-            # get list of members
-            members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
-
-            # get leader
-            leader = nodes.get(self._LEADER)
-            if leader:
-                member = Member(-1, leader.value, None, {})
-                member = ([m for m in members if m.name == leader.value] or [member])[0]
-                index = result.etcd_index if result.etcd_index > leader.modifiedIndex else leader.modifiedIndex + 1
-                leader = Leader(index, leader.ttl, member)
-
-            # failover key
-            failover = nodes.get(self._FAILOVER)
-            if failover:
-                failover = Failover.from_node(failover.modifiedIndex, failover.value)
-
-            # get synchronization state
-            sync = nodes.get(self._SYNC)
-            sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
-
-            cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+            cluster = loader(path)
         except etcd.EtcdKeyNotFound:
-            cluster = Cluster(None, None, None, None, [], None, None, None, None)
+            cluster = Cluster.empty()
         except Exception as e:
             self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
         self._has_failed = False
         return cluster
 
     @catch_etcd_errors
-    def touch_member(self, data, permanent=False):
+    def touch_member(self, data):
         data = json.dumps(data, separators=(',', ':'))
-        return self._client.set(self.member_path, data, None if permanent else self._ttl)
+        return self._client.set(self.member_path, data, self._ttl)
 
     @catch_etcd_errors
     def take_leader(self):
         return self.retry(self._client.write, self.leader_path, self._name, ttl=self._ttl)
 
-    def attempt_to_acquire_leader(self, permanent=False):
+    def _do_attempt_to_acquire_leader(self):
         try:
-            return bool(self.retry(self._client.write,
-                                   self.leader_path,
-                                   self._name,
-                                   ttl=None if permanent else self._ttl,
-                                   prevExist=False))
+            return bool(self.retry(self._client.write, self.leader_path, self._name, ttl=self._ttl, prevExist=False))
         except etcd.EtcdAlreadyExist:
             logger.info('Could not take out TTL lock')
-        except (RetryFailedError, etcd.EtcdException):
-            pass
-        return False
+            return False
+
+    @catch_return_false_exception
+    def attempt_to_acquire_leader(self):
+        return self._run_and_handle_exceptions(self._do_attempt_to_acquire_leader, retry=None)
 
     @catch_etcd_errors
     def set_failover_value(self, value, index=None):
@@ -691,9 +746,20 @@ class Etcd(AbstractEtcd):
     def _write_status(self, value):
         return self._client.set(self.status_path, value)
 
+    def _do_update_leader(self):
+        try:
+            return self.retry(self._client.write, self.leader_path, self._name,
+                              prevValue=self._name, ttl=self._ttl) is not None
+        except etcd.EtcdKeyNotFound:
+            return self._do_attempt_to_acquire_leader()
+
     @catch_etcd_errors
+    def _write_failsafe(self, value):
+        return self._client.set(self.failsafe_path, value)
+
+    @catch_return_false_exception
     def _update_leader(self):
-        return self.retry(self._client.write, self.leader_path, self._name, prevValue=self._name, ttl=self._ttl)
+        return self._run_and_handle_exceptions(self._do_update_leader, retry=None)
 
     @catch_etcd_errors
     def initialize(self, create_new=True, sysid=""):

patroni/dcs/etcd3.py

@@ -4,15 +4,17 @@ import etcd
 import json
 import logging
 import os
-import six
 import socket
 import sys
 import time
 import urllib3
 
+from collections import defaultdict
 from threading import Condition, Lock, Thread
+from urllib3.exceptions import ReadTimeoutError, ProtocolError
 
-from . import ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
+from . import ClusterConfig, Cluster, Failover, Leader, Member, SyncState,\
+        TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
 from .etcd import AbstractEtcdClientWithFailover, AbstractEtcd, catch_etcd_errors
 from ..exceptions import DCSError, PatroniException
 from ..utils import deep_compare, enable_keepalive, iter_response_objects, RetryFailedError, USER_AGENT
@@ -89,7 +91,7 @@ class Unavailable(Etcd3ClientError):
     code = GRPCCode.Unavailable
 
 
-# https://github.com/etcd-io/etcd/blob/master/etcdserver/api/v3rpc/rpctypes/error.go
+# https://github.com/etcd-io/etcd/commits/main/api/v3rpc/rpctypes/error.go
 class LeaseNotFound(NotFound):
     error = "etcdserver: requested lease not found"
 
@@ -174,20 +176,6 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
         self.version_prefix = '/v3beta'
         super(Etcd3Client, self).__init__(config, dns_resolver, cache_ttl)
 
-        if six.PY2:  # pragma: no cover
-            # Old grpc-gateway sometimes sends double 'transfer-encoding: chunked' headers,
-            # what breaks the old (python2.7) httplib.HTTPConnection (it closes the socket).
-            def dedup_addheader(httpm, key, value):
-                prev = httpm.dict.get(key)
-                if prev is None:
-                    httpm.dict[key] = value
-                elif key != 'transfer-encoding' or prev != value:
-                    combined = ", ".join((prev, value))
-                    httpm.dict[key] = combined
-
-            import httplib
-            httplib.HTTPMessage.addheader = dedup_addheader
-
         try:
             self.authenticate()
         except AuthFailed as e:
@@ -315,6 +303,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
     def prefix(self, key, retry=None):
         return self.range(key, prefix_range_end(key), retry)
 
+    @_handle_auth_errors
     def lease_grant(self, ttl, retry=None):
         return self.call_rpc('/lease/grant', {'TTL': ttl}, retry)['ID']
 
@@ -349,7 +338,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
     def deleteprefix(self, key, retry=None):
         return self.deleterange(key, prefix_range_end(key), retry=retry)
 
-    def watchrange(self, key, range_end=None, start_revision=None, filters=None):
+    def watchrange(self, key, range_end=None, start_revision=None, filters=None, read_timeout=None):
         """returns: response object"""
         params = build_range_request(key, range_end)
         if start_revision is not None:
@@ -357,11 +346,11 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
         params['filters'] = filters or []
         kwargs = self._prepare_common_parameters(1, self.read_timeout)
         request_executor = self._prepare_request(kwargs, {'create_request': params})
-        kwargs.update(timeout=urllib3.Timeout(connect=kwargs['timeout']), retries=0)
+        kwargs.update(timeout=urllib3.Timeout(connect=kwargs['timeout'], read=read_timeout), retries=0)
         return request_executor(self._MPOST, self._base_uri + self.version_prefix + '/watch', **kwargs)
 
-    def watchprefix(self, key, start_revision=None, filters=None):
-        return self.watchrange(key, prefix_range_end(key), start_revision, filters)
+    def watchprefix(self, key, start_revision=None, filters=None, read_timeout=None):
+        return self.watchrange(key, prefix_range_end(key), start_revision, filters, read_timeout)
 
 
 class KVCache(Thread):
@@ -450,7 +439,14 @@ class KVCache(Thread):
     def _do_watch(self, revision):
         with self._response_lock:
             self._response = None
-        response = self._client.watchprefix(self._dcs.cluster_prefix, revision)
+        # We do most of requests with timeouts. The only exception /watch requests to Etcd v3.
+        # In order to interrupt the /watch request we do socket.shutdown() from the main thread,
+        # which doesn't work on Windows. Therefore we want to use the last resort, `read_timeout`.
+        # Setting it to TTL will help to partially mitigate the problem.
+        # Setting it to lower value is not nice because for idling clusters it will increase
+        # the numbers of interrupts and reconnects.
+        read_timeout = self._dcs.ttl if os.name == 'nt' else None
+        response = self._client.watchprefix(self._dcs.cluster_prefix, revision, read_timeout=read_timeout)
         with self._response_lock:
             if self._response is None:
                 self._response = response
@@ -472,7 +468,9 @@ class KVCache(Thread):
         try:
             self._do_watch(result['header']['revision'])
         except Exception as e:
-            logger.error('watchprefix failed: %r', e)
+            # Following exceptions are expected on Windows because the /watch request  is done with `read_timeout`
+            if not (os.name == 'nt' and isinstance(e, (ReadTimeoutError, ProtocolError))):
+                logger.error('watchprefix failed: %r', e)
         finally:
             with self.condition:
                 self._is_ready = False
@@ -546,13 +544,18 @@ class PatroniEtcd3Client(Etcd3Client):
                 raise RetryFailedError('Exceeded retry deadline')
             self._kv_cache.condition.wait(timeout)
 
-    def get_cluster(self):
-        if self._kv_cache:
+    def get_cluster(self, path):
+        if self._kv_cache and path.startswith(self._etcd3.cluster_prefix):
             with self._kv_cache.condition:
                 self._wait_cache(self._etcd3._retry.deadline)
-                return self._kv_cache.copy()
+                ret = self._kv_cache.copy()
         else:
-            return self._etcd3.retry(self.prefix, self._etcd3.cluster_prefix).get('kvs', [])
+            ret = self._etcd3.retry(self.prefix, path).get('kvs', [])
+        for node in ret:
+            node.update({'key': base64_decode(node['key']),
+                         'value': base64_decode(node.get('value', '')),
+                         'lease': node.get('lease')})
+        return ret
 
     def call_rpc(self, method, fields, retry=None):
         ret = super(PatroniEtcd3Client, self).call_rpc(method, fields, retry)
@@ -598,8 +601,8 @@ class Etcd3(AbstractEtcd):
         if self.__do_not_watch:
             self._lease = None
 
-    def _do_refresh_lease(self, retry=None):
-        if self._lease and self._last_lease_refresh + self._loop_wait > time.time():
+    def _do_refresh_lease(self, force=False, retry=None):
+        if not force and self._lease and self._last_lease_refresh + self._loop_wait > time.time():
             return False
 
         if self._lease and not self._client.lease_keepalive(self._lease, retry):
@@ -629,78 +632,94 @@ class Etcd3(AbstractEtcd):
 
     @property
     def cluster_prefix(self):
-        return self.client_path('')
+        return self._base_path + '/' if self.is_citus_coordinator() else self.client_path('')
 
     @staticmethod
     def member(node):
         return Member.from_node(node['mod_revision'], os.path.basename(node['key']), node['lease'], node['value'])
 
-    def _load_cluster(self):
+    def _cluster_from_nodes(self, nodes):
+        # get initialize flag
+        initialize = nodes.get(self._INITIALIZE)
+        initialize = initialize and initialize['value']
+
+        # get global dynamic configuration
+        config = nodes.get(self._CONFIG)
+        config = config and ClusterConfig.from_node(config['mod_revision'], config['value'])
+
+        # get timeline history
+        history = nodes.get(self._HISTORY)
+        history = history and TimelineHistory.from_node(history['mod_revision'], history['value'])
+
+        # get last know leader lsn and slots
+        status = nodes.get(self._STATUS)
+        if status:
+            try:
+                status = json.loads(status['value'])
+                last_lsn = status.get(self._OPTIME)
+                slots = status.get('slots')
+            except Exception:
+                slots = last_lsn = None
+        else:
+            last_lsn = nodes.get(self._LEADER_OPTIME)
+            last_lsn = last_lsn and last_lsn['value']
+            slots = None
+
+        try:
+            last_lsn = int(last_lsn)
+        except Exception:
+            last_lsn = 0
+
+        # get list of members
+        members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
+
+        # get leader
+        leader = nodes.get(self._LEADER)
+        if not self._ctl and leader and leader['value'] == self._name and self._lease != leader.get('lease'):
+            logger.warning('I am the leader but not owner of the lease')
+
+        if leader:
+            member = Member(-1, leader['value'], None, {})
+            member = ([m for m in members if m.name == leader['value']] or [member])[0]
+            leader = Leader(leader['mod_revision'], leader['lease'], member)
+
+        # failover key
+        failover = nodes.get(self._FAILOVER)
+        if failover:
+            failover = Failover.from_node(failover['mod_revision'], failover['value'])
+
+        # get synchronization state
+        sync = nodes.get(self._SYNC)
+        sync = SyncState.from_node(sync and sync['mod_revision'], sync and sync['value'])
+
+        # get failsafe topology
+        failsafe = nodes.get(self._FAILSAFE)
+        try:
+            failsafe = json.loads(failsafe['value']) if failsafe else None
+        except Exception:
+            failsafe = None
+
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+
+    def _cluster_loader(self, path):
+        nodes = {node['key'][len(path):]: node
+                 for node in self._client.get_cluster(path)
+                 if node['key'].startswith(path)}
+        return self._cluster_from_nodes(nodes)
+
+    def _citus_cluster_loader(self, path):
+        clusters = defaultdict(dict)
+        path = self._base_path + '/'
+        for node in self._client.get_cluster(path):
+            key = node['key'][len(path):].split('/', 1)
+            if len(key) == 2 and citus_group_re.match(key[0]):
+                clusters[int(key[0])][key[1]] = node
+        return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()}
+
+    def _load_cluster(self, path, loader):
         cluster = None
         try:
-            path_len = len(self.cluster_prefix)
-
-            nodes = {}
-            for node in self._client.get_cluster():
-                node['key'] = base64_decode(node['key'])
-                node['value'] = base64_decode(node.get('value', ''))
-                node['lease'] = node.get('lease')
-                nodes[node['key'][path_len:].lstrip('/')] = node
-
-            # get initialize flag
-            initialize = nodes.get(self._INITIALIZE)
-            initialize = initialize and initialize['value']
-
-            # get global dynamic configuration
-            config = nodes.get(self._CONFIG)
-            config = config and ClusterConfig.from_node(config['mod_revision'], config['value'])
-
-            # get timeline history
-            history = nodes.get(self._HISTORY)
-            history = history and TimelineHistory.from_node(history['mod_revision'], history['value'])
-
-            # get last know leader lsn and slots
-            status = nodes.get(self._STATUS)
-            if status:
-                try:
-                    status = json.loads(status['value'])
-                    last_lsn = status.get(self._OPTIME)
-                    slots = status.get('slots')
-                except Exception:
-                    slots = last_lsn = None
-            else:
-                last_lsn = nodes.get(self._LEADER_OPTIME)
-                last_lsn = last_lsn and last_lsn['value']
-                slots = None
-
-            try:
-                last_lsn = int(last_lsn)
-            except Exception:
-                last_lsn = 0
-
-            # get list of members
-            members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
-
-            # get leader
-            leader = nodes.get(self._LEADER)
-            if not self._ctl and leader and leader['value'] == self._name and self._lease != leader.get('lease'):
-                logger.warning('I am the leader but not owner of the lease')
-
-            if leader:
-                member = Member(-1, leader['value'], None, {})
-                member = ([m for m in members if m.name == leader['value']] or [member])[0]
-                leader = Leader(leader['mod_revision'], leader['lease'], member)
-
-            # failover key
-            failover = nodes.get(self._FAILOVER)
-            if failover:
-                failover = Failover.from_node(failover['mod_revision'], failover['value'])
-
-            # get synchronization state
-            sync = nodes.get(self._SYNC)
-            sync = SyncState.from_node(sync and sync['mod_revision'], sync and sync['value'])
-
-            cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+            cluster = loader(path)
         except UnsupportedEtcdVersion:
             raise
         except Exception as e:
@@ -709,12 +728,11 @@ class Etcd3(AbstractEtcd):
         return cluster
 
     @catch_etcd_errors
-    def touch_member(self, data, permanent=False):
-        if not permanent:
-            try:
-                self.refresh_lease()
-            except Etcd3Error:
-                return False
+    def touch_member(self, data):
+        try:
+            self.refresh_lease()
+        except Etcd3Error:
+            return False
 
         cluster = self.cluster
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
@@ -724,7 +742,7 @@ class Etcd3(AbstractEtcd):
 
         data = json.dumps(data, separators=(',', ':'))
         try:
-            return self._client.put(self.member_path, data, None if permanent else self._lease)
+            return self._client.put(self.member_path, data, self._lease)
         except LeaseNotFound:
             self._lease = None
             logger.error('Our lease disappeared from Etcd, can not "touch_member"')
@@ -733,21 +751,41 @@ class Etcd3(AbstractEtcd):
     def take_leader(self):
         return self.retry(self._client.put, self.leader_path, self._name, self._lease)
 
-    @catch_etcd_errors
-    def _do_attempt_to_acquire_leader(self, permanent):
+    def _do_attempt_to_acquire_leader(self, retry):
+        def _retry(*args, **kwargs):
+            kwargs['retry'] = retry
+            return retry(*args, **kwargs)
+
         try:
-            return self.retry(self._client.put, self.leader_path, self._name, None if permanent else self._lease, 0)
+            return _retry(self._client.put, self.leader_path, self._name, self._lease, 0)
         except LeaseNotFound:
-            self._lease = None
             logger.error('Our lease disappeared from Etcd. Will try to get a new one and retry attempt')
-            self.refresh_lease()
-            return self.retry(self._client.put, self.leader_path, self._name, None if permanent else self._lease, 0)
+            self._lease = None
+            retry.deadline = retry.stoptime - time.time()
 
-    def attempt_to_acquire_leader(self, permanent=False):
-        if not self._lease and not permanent:
-            self.refresh_lease()
+            _retry(self._do_refresh_lease)
 
-        ret = self._do_attempt_to_acquire_leader(permanent)
+            retry.deadline = retry.stoptime - time.time()
+            if retry.deadline < 1:
+                raise Etcd3Error('_do_attempt_to_acquire_leader timeout')
+
+            return _retry(self._client.put, self.leader_path, self._name, self._lease, 0)
+
+    @catch_return_false_exception
+    def attempt_to_acquire_leader(self):
+        retry = self._retry.copy()
+
+        def _retry(*args, **kwargs):
+            kwargs['retry'] = retry
+            return retry(*args, **kwargs)
+
+        self._run_and_handle_exceptions(self._do_refresh_lease, retry=_retry)
+
+        retry.deadline = retry.stoptime - time.time()
+        if retry.deadline < 1:
+            raise Etcd3Error('attempt_to_acquire_leader timeout')
+
+        ret = self._run_and_handle_exceptions(self._do_attempt_to_acquire_leader, retry, retry=None)
         if not ret:
             logger.info('Could not take out TTL lock')
         return ret
@@ -769,17 +807,32 @@ class Etcd3(AbstractEtcd):
         return self._client.put(self.status_path, value)
 
     @catch_etcd_errors
+    def _write_failsafe(self, value):
+        return self._client.put(self.failsafe_path, value)
+
+    @catch_return_false_exception
     def _update_leader(self):
-        if not self._lease:
-            self.refresh_lease()
-        elif self.retry(self._client.lease_keepalive, self._lease):
-            self._last_lease_refresh = time.time()
+        retry = self._retry.copy()
+
+        def _retry(*args, **kwargs):
+            kwargs['retry'] = retry
+            return retry(*args, **kwargs)
+
+        self._run_and_handle_exceptions(self._do_refresh_lease, True, retry=_retry)
 
         if self._lease:
             cluster = self.cluster
             leader_lease = cluster and isinstance(cluster.leader, Leader) and cluster.leader.session
             if leader_lease != self._lease:
-                self.take_leader()
+                retry.deadline = retry.stoptime - time.time()
+                if retry.deadline < 1:
+                    raise Etcd3Error('update_leader timeout')
+
+                try:
+                    self._run_and_handle_exceptions(self._client.put, self.leader_path,
+                                                    self._name, self._lease, retry=_retry)
+                except ReturnFalseException:
+                    pass
         return bool(self._lease)
 
     @catch_etcd_errors
@@ -798,7 +851,7 @@ class Etcd3(AbstractEtcd):
 
     @catch_etcd_errors
     def delete_cluster(self):
-        return self.retry(self._client.deleteprefix, self.cluster_prefix)
+        return self.retry(self._client.deleteprefix, self.client_path(''))
 
     @catch_etcd_errors
     def set_history_value(self, value):

patroni/dcs/exhibitor.py

@@ -19,7 +19,7 @@ class ExhibitorEnsembleProvider(object):
         self._uri_path = uri_path
         self._poll_interval = poll_interval
         self._exhibitors = hosts
-        self._master_exhibitors = hosts
+        self._boot_exhibitors = hosts
         self._zookeeper_hosts = ''
         self._next_poll = None
         while not self.poll():
@@ -32,7 +32,7 @@ class ExhibitorEnsembleProvider(object):
 
         json = self._query_exhibitors(self._exhibitors)
         if not json:
-            json = self._query_exhibitors(self._master_exhibitors)
+            json = self._query_exhibitors(self._boot_exhibitors)
 
         if isinstance(json, dict) and 'servers' in json and 'port' in json:
             self._next_poll = time.time() + self._poll_interval
@@ -64,11 +64,9 @@ class Exhibitor(ZooKeeper):
     def __init__(self, config):
         interval = config.get('poll_interval', 300)
         self._ensemble_provider = ExhibitorEnsembleProvider(config['hosts'], config['port'], poll_interval=interval)
-        config = config.copy()
-        config['hosts'] = self._ensemble_provider.zookeeper_hosts
-        super(Exhibitor, self).__init__(config)
+        super(Exhibitor, self).__init__({**config, 'hosts': self._ensemble_provider.zookeeper_hosts})
 
-    def _load_cluster(self):
+    def _load_cluster(self, path, loader):
         if self._ensemble_provider.poll():
             self._client.set_hosts(self._ensemble_provider.zookeeper_hosts)
-        return super(Exhibitor, self)._load_cluster()
+        return super(Exhibitor, self)._load_cluster(path, loader)

patroni/dcs/kubernetes.py

@@ -1,3 +1,5 @@
+import atexit
+import base64
 import datetime
 import functools
 import json
@@ -5,18 +7,20 @@ import logging
 import os
 import random
 import socket
-import six
-import sys
+import tempfile
 import time
 import urllib3
 import yaml
 
-from urllib3 import Timeout
-from urllib3.exceptions import HTTPError
-from six.moves.http_client import HTTPException
+from collections import defaultdict
+from copy import deepcopy
+from http.client import HTTPException
 from threading import Condition, Lock, Thread
+from typing import Any, Dict, List, Optional
+from urllib3.exceptions import HTTPError
 
-from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState, TimelineHistory
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
+        TimelineHistory, CITUS_COORDINATOR_GROUP_ID, citus_group_re
 from ..exceptions import DCSError
 from ..utils import deep_compare, iter_response_objects, keepalive_socket_options,\
         Retry, RetryFailedError, tzutc, uri, USER_AGENT
@@ -28,12 +32,34 @@ SERVICE_HOST_ENV_NAME = 'KUBERNETES_SERVICE_HOST'
 SERVICE_PORT_ENV_NAME = 'KUBERNETES_SERVICE_PORT'
 SERVICE_TOKEN_FILENAME = '/var/run/secrets/kubernetes.io/serviceaccount/token'
 SERVICE_CERT_FILENAME = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'
+__temp_files = []
 
 
 class KubernetesError(DCSError):
     pass
 
 
+def _cleanup_temp_files():
+    global __temp_files
+    for temp_file in __temp_files:
+        try:
+            os.remove(temp_file)
+        except OSError:
+            pass
+    __temp_files = []
+
+
+def _create_temp_file(content):
+    if len(__temp_files) == 0:
+        atexit.register(_cleanup_temp_files)
+
+    fd, name = tempfile.mkstemp()
+    os.write(fd, content)
+    os.close(fd)
+    __temp_files.append(name)
+    return name
+
+
 # this function does the same mapping of snake_case => camelCase for > 97% of cases as autogenerated swagger code
 def to_camel_case(value):
     reserved = {'api', 'apiv3', 'cidr', 'cpu', 'csi', 'id', 'io', 'ip', 'ipc', 'pid', 'tls', 'uri', 'url', 'uuid'}
@@ -93,6 +119,13 @@ class K8sConfig(object):
             if c['name'] == name:
                 return c[section]
 
+    def _pool_config_from_file_or_data(self, config, file_key_name, pool_key_name):
+        data_key_name = file_key_name + '-data'
+        if data_key_name in config:
+            self.pool_config[pool_key_name] = _create_temp_file(base64.b64decode(config[data_key_name]))
+        elif file_key_name in config:
+            self.pool_config[pool_key_name] = config[file_key_name]
+
     def load_kube_config(self, context=None):
         with open(os.path.expanduser(KUBE_CONFIG_DEFAULT_LOCATION)) as f:
             config = yaml.safe_load(f)
@@ -103,10 +136,9 @@ class K8sConfig(object):
 
         self._server = cluster['server'].rstrip('/')
         if self._server.startswith('https'):
-            self.pool_config.update({v: user[k] for k, v in {'client-certificate': 'cert_file',
-                                                             'client-key': 'key_file'}.items() if k in user})
-            if 'certificate-authority' in cluster:
-                self.pool_config['ca_certs'] = cluster['certificate-authority']
+            self._pool_config_from_file_or_data(user, 'client-certificate', 'cert_file')
+            self._pool_config_from_file_or_data(user, 'client-key', 'key_file')
+            self._pool_config_from_file_or_data(cluster, 'certificate-authority', 'ca_certs')
             self.pool_config['cert_reqs'] = 'CERT_NONE' if cluster.get('insecure-skip-tls-verify') else 'CERT_REQUIRED'
         if user.get('token'):
             self._make_headers(token=user['token'])
@@ -143,7 +175,7 @@ class K8sObject(object):
         if isinstance(value, dict):
             # we know that `annotations` and `labels` are dicts and therefore don't want to convert them into K8sObject
             return value if parent in {'annotations', 'labels'} and \
-                    all(isinstance(v, six.string_types) for v in value.values()) else cls(value)
+                    all(isinstance(v, str) for v in value.values()) else cls(value)
         elif isinstance(value, list):
             return [cls._wrap(None, v) for v in value]
         else:
@@ -187,7 +219,7 @@ class K8sClient(object):
 
         _API_URL_PREFIX = '/api/v1/namespaces/'
 
-        def __init__(self, bypass_api_service=False):
+        def __init__(self, bypass_api_service: Optional[bool] = False) -> None:
             self._bypass_api_service = bypass_api_service
             self.pool_manager = urllib3.PoolManager(**k8s_config.pool_config)
             self._base_uri = k8s_config.server
@@ -208,7 +240,7 @@ class K8sClient(object):
 
         def set_base_uri(self, value):
             logger.info('Selected new K8s API server endpoint %s', value)
-            # We will connect by IP of the master node which is not listed as alternative name
+            # We will connect by IP of the K8s master node which is not listed as alternative name
             self.pool_manager.connection_pool_kw['assert_hostname'] = False
             self._base_uri = value
 
@@ -343,7 +375,7 @@ class K8sClient(object):
             api_servers = len(api_servers_cache)
 
             if timeout:
-                if isinstance(timeout, six.integer_types + (float,)):
+                if isinstance(timeout, (int, float)):
                     timeout = urllib3.Timeout(total=timeout)
                 elif isinstance(timeout, tuple) and len(timeout) == 2:
                     timeout = urllib3.Timeout(connect=timeout[0], read=timeout[1])
@@ -455,11 +487,15 @@ class KubernetesRetriableException(k8s_client.rest.ApiException):
 
 
 class CoreV1ApiProxy(object):
+    """Proxy class to work with k8s_client.CoreV1Api() object"""
 
-    def __init__(self, use_endpoints=False, bypass_api_service=False):
+    _DEFAULT_RETRIABLE_HTTP_CODES = frozenset([500, 503, 504])
+
+    def __init__(self, use_endpoints: Optional[bool] = False, bypass_api_service: Optional[bool] = False) -> None:
         self._api_client = k8s_client.ApiClient(bypass_api_service)
         self._core_v1_api = k8s_client.CoreV1Api(self._api_client)
         self._use_endpoints = bool(use_endpoints)
+        self._retriable_http_codes = set(self._DEFAULT_RETRIABLE_HTTP_CODES)
 
     def configure_timeouts(self, loop_wait, retry_timeout, ttl):
         # Normally every loop_wait seconds we should have receive something from the socket.
@@ -471,10 +507,21 @@ class CoreV1ApiProxy(object):
         self._api_client.set_read_timeout(retry_timeout)
         self._api_client.set_api_servers_cache_ttl(loop_wait)
 
+    def configure_retriable_http_codes(self, retriable_http_codes: List[int]) -> None:
+        self._retriable_http_codes = self._DEFAULT_RETRIABLE_HTTP_CODES | set(retriable_http_codes)
+
     def refresh_api_servers_cache(self):
         self._api_client.refresh_api_servers_cache()
 
-    def __getattr__(self, func):
+    def __getattr__(self, func: str):
+        """Intercepts calls to `CoreV1Api` methods.
+
+        Handles two important cases:
+        1. Depending on whether Patroni is configured to work with `ConfigMaps` or `Endpoints`
+           it remaps "virtual" method names from `*_kind` to `*_endpoints` or `*_config_map`.
+        2. It handles HTTP error codes and raises `KubernetesRetriableException`
+           if the given error is supposed to be handled with retry."""
+
         if func.endswith('_kind'):
             func = func[:-4] + ('endpoints' if self._use_endpoints else 'config_map')
 
@@ -482,7 +529,7 @@ class CoreV1ApiProxy(object):
             try:
                 return getattr(self._core_v1_api, func)(*args, **kwargs)
             except k8s_client.rest.ApiException as e:
-                if e.status in (500, 503, 504) or e.headers and 'retry-after' in e.headers:  # XXX
+                if e.status in self._retriable_http_codes or e.headers and 'retry-after' in e.headers:
                     raise KubernetesRetriableException(e)
                 raise
         return wrapper
@@ -493,16 +540,10 @@ class CoreV1ApiProxy(object):
 
 
 def catch_kubernetes_errors(func):
-    def wrapper(*args, **kwargs):
+    def wrapper(self, *args, **kwargs):
         try:
-            return func(*args, **kwargs)
-        except k8s_client.rest.ApiException as e:
-            if e.status == 403:
-                logger.exception('Permission denied')
-            elif e.status != 409:  # Object exists or conflict in resource_version
-                logger.exception('Unexpected error from Kubernetes API')
-            return False
-        except (RetryFailedError, K8sException):
+            return self._run_and_handle_exceptions(func, self, *args, **kwargs)
+        except KubernetesError:
             return False
     return wrapper
 
@@ -533,7 +574,7 @@ class ObjectCache(Thread):
             raise
 
     def _watch(self, resource_version):
-        return self._func(_request_timeout=(self._retry.deadline, Timeout.DEFAULT_TIMEOUT),
+        return self._func(_request_timeout=(self._retry.deadline, urllib3.Timeout.DEFAULT_TIMEOUT),
                           _preload_content=False, watch=True, resource_version=resource_version)
 
     def set(self, name, value):
@@ -662,22 +703,26 @@ class ObjectCache(Thread):
 
 class Kubernetes(AbstractDCS):
 
+    _CITUS_LABEL = 'citus-group'
+
     def __init__(self, config):
-        self._labels = config['labels']
+        self._labels = deepcopy(config['labels'])
         self._labels[config.get('scope_label', 'cluster-name')] = config['scope']
         self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
         self._namespace = config.get('namespace') or 'default'
         self._role_label = config.get('role_label', 'role')
         self._ca_certs = os.environ.get('PATRONI_KUBERNETES_CACERT', config.get('cacert')) or SERVICE_CERT_FILENAME
-        config['namespace'] = ''
-        super(Kubernetes, self).__init__(config)
+        super(Kubernetes, self).__init__({**config, 'namespace': ''})
+        if self._citus_group:
+            self._labels[self._CITUS_LABEL] = self._citus_group
+
         self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
                             retry_exceptions=KubernetesRetriableException)
         self._ttl = None
         try:
             k8s_config.load_incluster_config(ca_certs=self._ca_certs)
         except k8s_config.ConfigException:
-            k8s_config.load_kube_config(context=config.get('context', 'local'))
+            k8s_config.load_kube_config(context=config.get('context', 'kind-kind'))
 
         self.__my_pod = None
         self.__ips = [] if config.get('patronictl') else [config.get('pod_ip')]
@@ -712,12 +757,25 @@ class Kubernetes(AbstractDCS):
         kwargs['_retry'] = retry
         return retry(*args, **kwargs)
 
+    @staticmethod
+    def _run_and_handle_exceptions(method, *args, **kwargs):
+        try:
+            return method(*args, **kwargs)
+        except k8s_client.rest.ApiException as e:
+            if e.status == 403:
+                logger.exception('Permission denied')
+            elif e.status != 409:  # Object exists or conflict in resource_version
+                logger.exception('Unexpected error from Kubernetes API')
+            return False
+        except (RetryFailedError, K8sException) as e:
+            raise KubernetesError(e)
+
     def client_path(self, path):
         return super(Kubernetes, self).client_path(path)[1:].replace('/', '-')
 
     @property
     def leader_path(self):
-        return self._base_path[1:] if self._api.use_endpoints else super(Kubernetes, self).leader_path
+        return super(Kubernetes, self).leader_path[:-7 if self._api.use_endpoints else None]
 
     def set_ttl(self, ttl):
         ttl = int(ttl)
@@ -731,10 +789,24 @@ class Kubernetes(AbstractDCS):
     def set_retry_timeout(self, retry_timeout):
         self._retry.deadline = retry_timeout
 
-    def reload_config(self, config):
+    def reload_config(self, config: Dict[str, Any]) -> None:
+        """Handles dynamic config changes.
+
+        Either cause by changes in the local configuration file + SIGHUP or by changes of dynamic configuration"""
+
         super(Kubernetes, self).reload_config(config)
         self._api.configure_timeouts(self.loop_wait, self._retry.deadline, self.ttl)
 
+        # retriable_http_codes supposed to be either int, list of integers or comma-separated string with integers.
+        retriable_http_codes = config.get('retriable_http_codes', [])
+        if not isinstance(retriable_http_codes, list):
+            retriable_http_codes = [c.strip() for c in str(retriable_http_codes).split(',')]
+
+        try:
+            self._api.configure_retriable_http_codes([int(c) for c in retriable_http_codes])
+        except Exception as e:
+            logger.warning('Invalid value of retriable_http_codes = %s: %r', config['retriable_http_codes'], e)
+
     @staticmethod
     def member(pod):
         annotations = pod.metadata.annotations or {}
@@ -749,88 +821,137 @@ class Kubernetes(AbstractDCS):
                 raise RetryFailedError('Exceeded retry deadline')
             self._condition.wait(timeout)
 
-    def _load_cluster(self):
+    def _cluster_from_nodes(self, group, nodes, pods):
+        members = [self.member(pod) for pod in pods]
+        path = self._base_path[1:] + '-'
+        if group:
+            path += group + '-'
+
+        config = nodes.get(path + self._CONFIG)
+        metadata = config and config.metadata
+        annotations = metadata and metadata.annotations or {}
+
+        # get initialize flag
+        initialize = annotations.get(self._INITIALIZE)
+
+        # get global dynamic configuration
+        config = ClusterConfig.from_node(metadata and metadata.resource_version,
+                                         annotations.get(self._CONFIG) or '{}',
+                                         metadata.resource_version if self._CONFIG in annotations else 0)
+
+        # get timeline history
+        history = TimelineHistory.from_node(metadata and metadata.resource_version,
+                                            annotations.get(self._HISTORY) or '[]')
+
+        leader_path = path[:-1] if self._api.use_endpoints else path + self._LEADER
+        leader = nodes.get(leader_path)
+        metadata = leader and leader.metadata
+        if leader_path == self.leader_path:  # We want to memorize leader_resource_version only for our cluster
+            self._leader_resource_version = metadata.resource_version if metadata else None
+        annotations = metadata and metadata.annotations or {}
+
+        # get last known leader lsn
+        last_lsn = annotations.get(self._OPTIME)
+        try:
+            last_lsn = 0 if last_lsn is None else int(last_lsn)
+        except Exception:
+            last_lsn = 0
+
+        # get permanent slots state (confirmed_flush_lsn)
+        slots = annotations.get('slots')
+        try:
+            slots = slots and json.loads(slots)
+        except Exception:
+            slots = None
+
+        # get failsafe topology
+        failsafe = annotations.get(self._FAILSAFE)
+        try:
+            failsafe = json.loads(failsafe) if failsafe else None
+        except Exception:
+            failsafe = None
+
+        # get leader
+        leader_record = {n: annotations.get(n) for n in (self._LEADER, 'acquireTime',
+                         'ttl', 'renewTime', 'transitions') if n in annotations}
+        # We want to memorize leader_observed_record and update leader_observed_time only for our cluster
+        if leader_path == self.leader_path and (leader_record or self._leader_observed_record)\
+                and leader_record != self._leader_observed_record:
+            self._leader_observed_record = leader_record
+            self._leader_observed_time = time.time()
+
+        leader = leader_record.get(self._LEADER)
+        try:
+            ttl = int(leader_record.get('ttl')) or self._ttl
+        except (TypeError, ValueError):
+            ttl = self._ttl
+
+        # We want to check validity of the leader record only for our own cluster
+        if leader_path == self.leader_path and\
+                not (metadata and self._leader_observed_time and self._leader_observed_time + ttl >= time.time()):
+            leader = None
+
+        if metadata:
+            member = Member(-1, leader, None, {})
+            member = ([m for m in members if m.name == leader] or [member])[0]
+            leader = Leader(metadata.resource_version, None, member)
+
+        # failover key
+        failover = nodes.get(path + self._FAILOVER)
+        metadata = failover and failover.metadata
+        failover = Failover.from_node(metadata and metadata.resource_version,
+                                      metadata and (metadata.annotations or {}).copy())
+
+        # get synchronization state
+        sync = nodes.get(path + self._SYNC)
+        metadata = sync and sync.metadata
+        sync = SyncState.from_node(metadata and metadata.resource_version,  metadata and metadata.annotations)
+
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+
+    def _cluster_loader(self, path):
+        return self._cluster_from_nodes(path['group'], path['nodes'], path['pods'])
+
+    def _citus_cluster_loader(self, path):
+        clusters = defaultdict(lambda: {'pods': [], 'nodes': {}})
+
+        for pod in path['pods']:
+            group = pod.metadata.labels.get(self._CITUS_LABEL)
+            if group and citus_group_re.match(group):
+                clusters[group]['pods'].append(pod)
+
+        for name, kind in path['nodes'].items():
+            group = kind.metadata.labels.get(self._CITUS_LABEL)
+            if group and citus_group_re.match(group):
+                clusters[group]['nodes'][name] = kind
+        return {int(group): self._cluster_from_nodes(group, value['nodes'], value['pods'])
+                for group, value in clusters.items()}
+
+    def __load_cluster(self, group, loader):
         stop_time = time.time() + self._retry.deadline
         self._api.refresh_api_servers_cache()
         try:
             with self._condition:
                 self._wait_caches(stop_time)
-
-                members = [self.member(pod) for pod in self._pods.copy().values()]
-                nodes = self._kinds.copy()
-
-            config = nodes.get(self.config_path)
-            metadata = config and config.metadata
-            annotations = metadata and metadata.annotations or {}
-
-            # get initialize flag
-            initialize = annotations.get(self._INITIALIZE)
-
-            # get global dynamic configuration
-            config = ClusterConfig.from_node(metadata and metadata.resource_version,
-                                             annotations.get(self._CONFIG) or '{}',
-                                             metadata.resource_version if self._CONFIG in annotations else 0)
-
-            # get timeline history
-            history = TimelineHistory.from_node(metadata and metadata.resource_version,
-                                                annotations.get(self._HISTORY) or '[]')
-
-            leader = nodes.get(self.leader_path)
-            metadata = leader and leader.metadata
-            self._leader_resource_version = metadata.resource_version if metadata else None
-            annotations = metadata and metadata.annotations or {}
-
-            # get last known leader lsn
-            last_lsn = annotations.get(self._OPTIME)
-            try:
-                last_lsn = 0 if last_lsn is None else int(last_lsn)
-            except Exception:
-                last_lsn = 0
-
-            # get permanent slots state (confirmed_flush_lsn)
-            slots = annotations.get('slots')
-            try:
-                slots = slots and json.loads(slots)
-            except Exception:
-                slots = None
-
-            # get leader
-            leader_record = {n: annotations.get(n) for n in (self._LEADER, 'acquireTime',
-                             'ttl', 'renewTime', 'transitions') if n in annotations}
-            if (leader_record or self._leader_observed_record) and leader_record != self._leader_observed_record:
-                self._leader_observed_record = leader_record
-                self._leader_observed_time = time.time()
-
-            leader = leader_record.get(self._LEADER)
-            try:
-                ttl = int(leader_record.get('ttl')) or self._ttl
-            except (TypeError, ValueError):
-                ttl = self._ttl
-
-            if not metadata or not self._leader_observed_time or self._leader_observed_time + ttl < time.time():
-                leader = None
-
-            if metadata:
-                member = Member(-1, leader, None, {})
-                member = ([m for m in members if m.name == leader] or [member])[0]
-                leader = Leader(metadata.resource_version, None, member)
-
-            # failover key
-            failover = nodes.get(self.failover_path)
-            metadata = failover and failover.metadata
-            failover = Failover.from_node(metadata and metadata.resource_version,
-                                          metadata and (metadata.annotations or {}).copy())
-
-            # get synchronization state
-            sync = nodes.get(self.sync_path)
-            metadata = sync and sync.metadata
-            sync = SyncState.from_node(metadata and metadata.resource_version,  metadata and metadata.annotations)
-
-            return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+                pods = [pod for pod in self._pods.copy().values()
+                        if not group or pod.metadata.labels.get(self._CITUS_LABEL) == group]
+                nodes = {name: kind for name, kind in self._kinds.copy().items()
+                         if not group or kind.metadata.labels.get(self._CITUS_LABEL) == group}
+            return loader({'group': group, 'pods': pods, 'nodes': nodes})
         except Exception:
             logger.exception('get_cluster')
             raise KubernetesError('Kubernetes API is not responding properly')
 
+    def _load_cluster(self, path, loader):
+        group = self._citus_group if path == self.client_path('') else None
+        return self.__load_cluster(group, loader)
+
+    def get_citus_coordinator(self):
+        try:
+            return self.__load_cluster(str(CITUS_COORDINATOR_GROUP_ID), self._cluster_loader)
+        except Exception as e:
+            logger.error('Failed to load Citus coordinator cluster from Kubernetes: %r', e)
+
     @staticmethod
     def compare_ports(p1, p2):
         return p1.name == p2.name and p1.port == p2.port and (p1.protocol or 'TCP') == (p2.protocol or 'TCP')
@@ -950,7 +1071,8 @@ class Kubernetes(AbstractDCS):
             if not self._api.create_namespaced_service(self._namespace, body):
                 return
         except Exception as e:
-            if not isinstance(e, k8s_client.rest.ApiException) or e.status != 409:  # Service already exists
+            # 409 - service already exists, 403 - creation forbidden
+            if not isinstance(e, k8s_client.rest.ApiException) or e.status not in (409, 403):
                 return logger.exception('create_config_service failed')
         self._should_create_config_service = False
 
@@ -960,6 +1082,9 @@ class Kubernetes(AbstractDCS):
     def _write_status(self, value):
         """Unused"""
 
+    def _write_failsafe(self, value):
+        """Unused"""
+
     def _update_leader(self):
         """Unused"""
 
@@ -978,16 +1103,19 @@ class Kubernetes(AbstractDCS):
             else:
                 logger.exception('Permission denied' if e.status == 403 else 'Unexpected error from Kubernetes API')
                 return False
-        except (RetryFailedError, K8sException):
-            return False
+        except (RetryFailedError, K8sException) as e:
+            raise KubernetesError(e)
 
+        # if we are here, that means update failed with 409
         retry.deadline = retry.stoptime - time.time()
         if retry.deadline < 1:
-            return False
+            return False  # No time for retry. Tell ha.py that we have to demote due to failed update.
 
         # Try to get the latest version directly from K8s API instead of relying on async cache
         try:
             kind = _retry(self._api.read_namespaced_kind, self.leader_path, self._namespace)
+        except (RetryFailedError, K8sException) as e:
+            raise KubernetesError(e)
         except Exception as e:
             logger.error('Failed to get the leader object "%s": %r', self.leader_path, e)
             return False
@@ -1005,9 +1133,10 @@ class Kubernetes(AbstractDCS):
         if kind and (kind_annotations.get(self._LEADER) != self._name or kind_resource_version == resource_version):
             return False
 
-        return self.patch_or_create(self.leader_path, annotations, kind_resource_version, ips=ips, retry=_retry)
+        return self._run_and_handle_exceptions(self._patch_or_create, self.leader_path, annotations,
+                                               kind_resource_version, ips=ips, retry=_retry)
 
-    def update_leader(self, last_lsn, slots=None):
+    def update_leader(self, last_lsn, slots=None, failsafe=None):
         kind = self._kinds.get(self.leader_path)
         kind_annotations = kind and kind.metadata.annotations or {}
 
@@ -1021,14 +1150,17 @@ class Kubernetes(AbstractDCS):
                        'transitions': leader_observed_record.get('transitions') or '0'}
         if last_lsn:
             annotations[self._OPTIME] = str(last_lsn)
-            annotations['slots'] = json.dumps(slots) if slots else None
+            annotations['slots'] = json.dumps(slots, separators=(',', ':')) if slots else None
+
+        if failsafe is not None:
+            annotations[self._FAILSAFE] = json.dumps(failsafe, separators=(',', ':')) if failsafe else None
 
         resource_version = kind and kind.metadata.resource_version
         return self._update_leader_with_retry(annotations, resource_version, self.__ips)
 
-    def attempt_to_acquire_leader(self, permanent=False):
+    def attempt_to_acquire_leader(self):
         now = datetime.datetime.now(tzutc).isoformat()
-        annotations = {self._LEADER: self._name, 'ttl': str(sys.maxsize if permanent else self._ttl),
+        annotations = {self._LEADER: self._name, 'ttl': str(self._ttl),
                        'renewTime': now, 'acquireTime': now, 'transitions': '0'}
         if self._leader_observed_record:
             try:
@@ -1042,7 +1174,19 @@ class Kubernetes(AbstractDCS):
                 annotations['acquireTime'] = self._leader_observed_record.get('acquireTime') or now
             annotations['transitions'] = str(transitions)
         ips = [] if self._api.use_endpoints else None
-        ret = self.patch_or_create(self.leader_path, annotations, self._leader_resource_version, ips=ips)
+
+        try:
+            ret = self._patch_or_create(self.leader_path, annotations,
+                                        self._leader_resource_version, retry=self.retry, ips=ips)
+        except k8s_client.rest.ApiException as e:
+            if e.status == 409 and self._leader_resource_version:  # Conflict in resource_version
+                # Terminate watchers, it could be a sign that K8s API is in a failed state
+                self._kinds.kill_stream()
+                self._pods.kill_stream()
+            ret = False
+        except (RetryFailedError, K8sException) as e:
+            raise KubernetesError(e)
+
         if not ret:
             logger.info('Could not take out TTL lock')
         return ret
@@ -1068,11 +1212,11 @@ class Kubernetes(AbstractDCS):
         return self.patch_or_create_config({self._CONFIG: value}, index, bool(self._config_resource_version), False)
 
     @catch_kubernetes_errors
-    def touch_member(self, data, permanent=False):
+    def touch_member(self, data):
         cluster = self.cluster
         if cluster and cluster.leader and cluster.leader.name == self._name:
             role = 'master'
-        elif data['state'] == 'running' and data['role'] != 'master':
+        elif data['state'] == 'running' and data['role'] not in ('master', 'primary'):
             role = data['role']
         else:
             role = None

patroni/dcs/raft.py

@@ -4,18 +4,24 @@ import os
 import threading
 import time
 
+from collections import defaultdict
 from pysyncobj import SyncObj, SyncObjConf, replicated, FAIL_REASON
 from pysyncobj.dns_resolver import globalDnsResolver
 from pysyncobj.node import TCPNode
 from pysyncobj.transport import TCPTransport, CONNECTION_STATE
 from pysyncobj.utility import TcpUtility
 
-from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
+from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory, citus_group_re
+from ..exceptions import DCSError
 from ..utils import validate_directory
 
 logger = logging.getLogger(__name__)
 
 
+class RaftError(DCSError):
+    pass
+
+
 class _TCPTransport(TCPTransport):
 
     def __init__(self, syncObj, selfNode, otherNodes):
@@ -39,9 +45,9 @@ setattr(TCPNode, 'ip', property(resolve_host))
 
 class SyncObjUtility(object):
 
-    def __init__(self, otherNodes, conf):
+    def __init__(self, otherNodes, conf, retry_timeout=10):
         self._nodes = otherNodes
-        self._utility = TcpUtility(conf.password)
+        self._utility = TcpUtility(conf.password, retry_timeout/max(1, len(otherNodes)))
 
     def executeCommand(self, command):
         try:
@@ -58,11 +64,11 @@ class SyncObjUtility(object):
 
 class DynMemberSyncObj(SyncObj):
 
-    def __init__(self, selfAddress, partnerAddrs, conf):
+    def __init__(self, selfAddress, partnerAddrs, conf, retry_timeout=10):
         self.__early_apply_local_log = selfAddress is not None
         self.applied_local_log = False
 
-        utility = SyncObjUtility(partnerAddrs, conf)
+        utility = SyncObjUtility(partnerAddrs, conf, retry_timeout)
         members = utility.getMembers()
         add_self = members and selfAddress not in members
 
@@ -97,7 +103,7 @@ class KVStoreTTL(DynMemberSyncObj):
         self.__on_set = on_set
         self.__on_delete = on_delete
         self.__limb = {}
-        self.__retry_timeout = None
+        self.set_retry_timeout(int(config.get('retry_timeout') or 10))
 
         self_addr = config.get('self_addr')
         partner_addrs = set(config.get('partner_addrs', []))
@@ -121,7 +127,7 @@ class KVStoreTTL(DynMemberSyncObj):
                            journalFile=(file_template + '.journal' if self_addr else None),
                            onReady=on_ready, dynamicMembershipChange=True)
 
-        super(KVStoreTTL, self).__init__(self_addr, partner_addrs, conf)
+        super(KVStoreTTL, self).__init__(self_addr, partner_addrs, conf, self.__retry_timeout)
         self.__data = {}
 
     @staticmethod
@@ -156,7 +162,7 @@ class KVStoreTTL(DynMemberSyncObj):
             elif deadline:
                 timeout = deadline - time.time()
                 if timeout <= 0:
-                    break
+                    raise RaftError('timeout')
             time.sleep(1)
         return False
 
@@ -175,7 +181,7 @@ class KVStoreTTL(DynMemberSyncObj):
             self.__on_set(key, value)
         return True
 
-    def set(self, key, value, ttl=None, **kwargs):
+    def set(self, key, value, ttl=None, handle_raft_error=True, **kwargs):
         old_value = self.__data.get(key, {})
         if not self.__check_requirements(old_value, **kwargs):
             return False
@@ -184,7 +190,12 @@ class KVStoreTTL(DynMemberSyncObj):
         value['created'] = old_value.get('created', value['updated'])
         if ttl:
             value['expire'] = value['updated'] + ttl
-        return self.retry(self._set, key, value, **kwargs)
+        try:
+            return self.retry(self._set, key, value, **kwargs)
+        except RaftError:
+            if not handle_raft_error:
+                raise
+            return False
 
     def __pop(self, key):
         self.__data.pop(key)
@@ -206,7 +217,10 @@ class KVStoreTTL(DynMemberSyncObj):
     def delete(self, key, recursive=False, **kwargs):
         if not recursive and not self.__check_requirements(self.__data.get(key, {}), **kwargs):
             return False
-        return self.retry(self._delete, key, recursive=recursive, **kwargs)
+        try:
+            return self.retry(self._delete, key, recursive=recursive, **kwargs)
+        except RaftError:
+            return False
 
     @staticmethod
     def __values_match(old, new):
@@ -275,7 +289,6 @@ class Raft(AbstractDCS):
                 break
             else:
                 logger.info('waiting on raft')
-        self.set_retry_timeout(int(config.get('retry_timeout') or 10))
 
     def _on_set(self, key, value):
         leader = (self._sync_obj.get(self.leader_path) or {}).get('value')
@@ -307,13 +320,7 @@ class Raft(AbstractDCS):
     def member(key, value):
         return Member.from_node(value['index'], os.path.basename(key), None, value['value'])
 
-    def _load_cluster(self):
-        prefix = self.client_path('')
-        response = self._sync_obj.get(prefix, recursive=True)
-        if not response:
-            return Cluster(None, None, None, None, [], None, None, None, None)
-        nodes = {os.path.relpath(key, prefix).replace('\\', '/'): value for key, value in response.items()}
-
+    def _cluster_from_nodes(self, nodes):
         # get initialize flag
         initialize = nodes.get(self._INITIALIZE)
         initialize = initialize and initialize['value']
@@ -364,7 +371,33 @@ class Raft(AbstractDCS):
         sync = nodes.get(self._SYNC)
         sync = SyncState.from_node(sync and sync['index'], sync and sync['value'])
 
-        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+        # get failsafe topology
+        failsafe = nodes.get(self._FAILSAFE)
+        try:
+            failsafe = json.loads(failsafe['value']) if failsafe else None
+        except Exception:
+            failsafe = None
+
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+
+    def _cluster_loader(self, path):
+        response = self._sync_obj.get(path, recursive=True)
+        if not response:
+            return Cluster.empty()
+        nodes = {key[len(path):]: value for key, value in response.items()}
+        return self._cluster_from_nodes(nodes)
+
+    def _citus_cluster_loader(self, path):
+        clusters = defaultdict(dict)
+        response = self._sync_obj.get(path, recursive=True)
+        for key, value in response.items():
+            key = key[len(path):].split('/', 1)
+            if len(key) == 2 and citus_group_re.match(key[0]):
+                clusters[int(key[0])][key[1]] = value
+        return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()}
+
+    def _load_cluster(self, path, loader):
+        return loader(path)
 
     def _write_leader_optime(self, last_lsn):
         return self._sync_obj.set(self.leader_optime_path, last_lsn, timeout=1)
@@ -372,15 +405,18 @@ class Raft(AbstractDCS):
     def _write_status(self, value):
         return self._sync_obj.set(self.status_path, value, timeout=1)
 
+    def _write_failsafe(self, value):
+        return self._sync_obj.set(self.failsafe_path, value, timeout=1)
+
     def _update_leader(self):
-        ret = self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl, prevValue=self._name)
+        ret = self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl,
+                                 handle_raft_error=False, prevValue=self._name)
         if not ret and self._sync_obj.get(self.leader_path) is None:
             ret = self.attempt_to_acquire_leader()
         return ret
 
-    def attempt_to_acquire_leader(self, permanent=False):
-        return self._sync_obj.set(self.leader_path, self._name, prevExist=False,
-                                  ttl=None if permanent else self._ttl)
+    def attempt_to_acquire_leader(self):
+        return self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl, handle_raft_error=False, prevExist=False)
 
     def set_failover_value(self, value, index=None):
         return self._sync_obj.set(self.failover_path, value, prevIndex=index)
@@ -388,9 +424,9 @@ class Raft(AbstractDCS):
     def set_config_value(self, value, index=None):
         return self._sync_obj.set(self.config_path, value, prevIndex=index)
 
-    def touch_member(self, data, permanent=False):
+    def touch_member(self, data):
         data = json.dumps(data, separators=(',', ':'))
-        return self._sync_obj.set(self.member_path, data, None if permanent else self._ttl, timeout=2)
+        return self._sync_obj.set(self.member_path, data, self._ttl, timeout=2)
 
     def take_leader(self):
         return self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl)

patroni/dcs/zookeeper.py

@@ -4,12 +4,13 @@ import select
 import time
 
 from kazoo.client import KazooClient, KazooState, KazooRetry
-from kazoo.exceptions import NoNodeError, NodeExistsError, SessionExpiredError
+from kazoo.exceptions import ConnectionClosedError, NoNodeError, NodeExistsError, SessionExpiredError
 from kazoo.handlers.threading import SequentialThreadingHandler
 from kazoo.protocol.states import KeeperState
+from kazoo.retry import RetryFailedError
 from kazoo.security import make_acl
 
-from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
+from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory, citus_group_re
 from ..exceptions import DCSError
 from ..utils import deep_compare
 
@@ -50,10 +51,18 @@ class PatroniSequentialThreadingHandler(SequentialThreadingHandler):
         return super(PatroniSequentialThreadingHandler, self).create_connection(*args, **kwargs)
 
     def select(self, *args, **kwargs):
-        """Python3 raises `ValueError` if socket is closed, because fd == -1"""
+        """
+        Python 3.XY may raise following exceptions if select/poll are called with an invalid socket:
+        - `ValueError`: because fd == -1
+        - `TypeError`: Invalid file descriptor: -1 (starting from kazoo 2.9)
+        Python 2.7 may raise the `IOError` instead of `socket.error` (starting from kazoo 2.9)
+
+        When it is appropriate we map these exceptions to `socket.error`.
+        """
+
         try:
             return super(PatroniSequentialThreadingHandler, self).select(*args, **kwargs)
-        except ValueError as e:
+        except (TypeError, ValueError) as e:
             raise select.error(9, str(e))
 
 
@@ -137,7 +146,11 @@ class ZooKeeper(AbstractDCS):
 
     def cluster_watcher(self, event):
         self._fetch_cluster = True
-        self.status_watcher(event)
+        if not event or event.state != KazooState.CONNECTED or event.path.startswith(self.client_path('')):
+            self.status_watcher(event)
+
+    def members_watcher(self, event):
+        self._fetch_cluster = True
 
     def reload_config(self, config):
         self.set_retry_timeout(config['retry_timeout'])
@@ -182,10 +195,10 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return None
 
-    def get_status(self, leader):
+    def get_status(self, path, leader):
         watch = self.status_watcher if not leader or leader.name != self._name else None
 
-        status = self.get_node(self.status_path, watch)
+        status = self.get_node(path + self._STATUS, watch)
         if status:
             try:
                 status = json.loads(status[0])
@@ -194,7 +207,7 @@ class ZooKeeper(AbstractDCS):
             except Exception:
                 slots = last_lsn = None
         else:
-            last_lsn = self.get_node(self.leader_optime_path, watch)
+            last_lsn = self.get_node(path + self._LEADER_OPTIME, watch)
             last_lsn = last_lsn and last_lsn[0]
             slots = None
 
@@ -216,69 +229,78 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return []
 
-    def load_members(self):
+    def load_members(self, path):
         members = []
-        for member in self.get_children(self.members_path, self.cluster_watcher):
-            data = self.get_node(self.members_path + member)
+        for member in self.get_children(path + self._MEMBERS, self.cluster_watcher):
+            data = self.get_node(path + self._MEMBERS + member)
             if data is not None:
                 members.append(self.member(member, *data))
         return members
 
-    def _inner_load_cluster(self):
+    def _cluster_loader(self, path):
         self._fetch_cluster = False
         self.event.clear()
-        nodes = set(self.get_children(self.client_path(''), self.cluster_watcher))
+        nodes = set(self.get_children(path, self.cluster_watcher))
         if not nodes:
             self._fetch_cluster = True
 
         # get initialize flag
-        initialize = (self.get_node(self.initialize_path) or [None])[0] if self._INITIALIZE in nodes else None
+        initialize = (self.get_node(path + self._INITIALIZE) or [None])[0] if self._INITIALIZE in nodes else None
 
         # get global dynamic configuration
-        config = self.get_node(self.config_path, watch=self.cluster_watcher) if self._CONFIG in nodes else None
+        config = self.get_node(path + self._CONFIG, watch=self.cluster_watcher) if self._CONFIG in nodes else None
         config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid)
 
         # get timeline history
-        history = self.get_node(self.history_path, watch=self.cluster_watcher) if self._HISTORY in nodes else None
+        history = self.get_node(path + self._HISTORY, watch=self.cluster_watcher) if self._HISTORY in nodes else None
         history = history and TimelineHistory.from_node(history[1].mzxid, history[0])
 
         # get synchronization state
-        sync = self.get_node(self.sync_path, watch=self.cluster_watcher) if self._SYNC in nodes else None
+        sync = self.get_node(path + self._SYNC, watch=self.cluster_watcher) if self._SYNC in nodes else None
         sync = SyncState.from_node(sync and sync[1].version, sync and sync[0])
 
         # get list of members
-        members = self.load_members() if self._MEMBERS[:-1] in nodes else []
+        members = self.load_members(path) if self._MEMBERS[:-1] in nodes else []
 
         # get leader
-        leader = self.get_node(self.leader_path) if self._LEADER in nodes else None
+        leader = self.get_node(path + self._LEADER) if self._LEADER in nodes else None
         if leader:
-            client_id = self._client.client_id
-            if not self._ctl and leader[0] == self._name and client_id is not None \
-                    and client_id[0] != leader[1].ephemeralOwner:
-                logger.info('I am leader but not owner of the session. Removing leader node')
-                self._client.delete(self.leader_path)
-                leader = None
-
-            if leader:
-                member = Member(-1, leader[0], None, {})
-                member = ([m for m in members if m.name == leader[0]] or [member])[0]
-                leader = Leader(leader[1].version, leader[1].ephemeralOwner, member)
-                self._fetch_cluster = member.index == -1
+            member = Member(-1, leader[0], None, {})
+            member = ([m for m in members if m.name == leader[0]] or [member])[0]
+            leader = Leader(leader[1].version, leader[1].ephemeralOwner, member)
+            self._fetch_cluster = member.index == -1
 
         # get last known leader lsn and slots
-        last_lsn, slots = self.get_status(leader)
+        last_lsn, slots = self.get_status(path, leader)
 
         # failover key
-        failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
+        failover = self.get_node(path + self._FAILOVER, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
         failover = failover and Failover.from_node(failover[1].version, failover[0])
 
-        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+        # get failsafe topology
+        failsafe = self.get_node(path + self._FAILSAFE, watch=self.cluster_watcher) if self._FAILSAFE in nodes else None
+        try:
+            failsafe = json.loads(failsafe[0]) if failsafe else None
+        except Exception:
+            failsafe = None
 
-    def _load_cluster(self):
-        cluster = self.cluster
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+
+    def _citus_cluster_loader(self, path):
+        fetch_cluster = False
+        ret = {}
+        for node in self.get_children(path, self.cluster_watcher):
+            if citus_group_re.match(node):
+                ret[int(node)] = self._cluster_loader(path + node + '/')
+                fetch_cluster = fetch_cluster or self._fetch_cluster
+        self._fetch_cluster = fetch_cluster
+        return ret
+
+    def _load_cluster(self, path, loader):
+        cluster = self.cluster if path == self._base_path + '/' else None
         if self._fetch_cluster or cluster is None:
             try:
-                cluster = self._client.retry(self._inner_load_cluster)
+                cluster = self._client.retry(loader, path)
             except Exception:
                 logger.exception('get_cluster')
                 self.cluster_watcher(None)
@@ -291,10 +313,12 @@ class ZooKeeper(AbstractDCS):
                 self.event.clear()
             else:
                 try:
-                    last_lsn, slots = self.get_status(cluster.leader)
+                    last_lsn, slots = self.get_status(self.client_path(''), cluster.leader)
                     self.event.clear()
-                    cluster = Cluster(cluster.initialize, cluster.config, cluster.leader, last_lsn,
-                                      cluster.members, cluster.failover, cluster.sync, cluster.history, slots)
+                    cluster = list(cluster)
+                    cluster[3] = last_lsn
+                    cluster[8] = slots
+                    cluster = Cluster(*cluster)
                 except Exception:
                     pass
         return cluster
@@ -313,11 +337,18 @@ class ZooKeeper(AbstractDCS):
             logger.exception('Failed to create %s', path)
         return False
 
-    def attempt_to_acquire_leader(self, permanent=False):
-        ret = self._create(self.leader_path, self._name.encode('utf-8'), retry=True, ephemeral=not permanent)
-        if not ret:
-            logger.info('Could not take out TTL lock')
-        return ret
+    def attempt_to_acquire_leader(self):
+        try:
+            self._client.retry(self._client.create, self.leader_path, self._name.encode('utf-8'),
+                               makepath=True, ephemeral=True)
+            return True
+        except (ConnectionClosedError, RetryFailedError) as e:
+            raise ZooKeeperError(e)
+        except Exception as e:
+            if not isinstance(e, NodeExistsError):
+                logger.error('Failed to create %s: %r', self.leader_path, e)
+        logger.info('Could not take out TTL lock')
+        return False
 
     def _set_or_create(self, key, value, index=None, retry=False, do_not_create_empty=False):
         value = value.encode('utf-8')
@@ -349,12 +380,15 @@ class ZooKeeper(AbstractDCS):
         return self._create(self.initialize_path, sysid, retry=True) if create_new \
             else self._client.retry(self._client.set, self.initialize_path, sysid)
 
-    def touch_member(self, data, permanent=False):
+    def touch_member(self, data):
         cluster = self.cluster
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
         member_data = self.__last_member_data or member and member.data
+        #  We want to notify leader if some important fields in the member key changed by removing ZNode
         if member and (self._client.client_id is not None and member.session != self._client.client_id[0] or
                        not (deep_compare(member_data.get('tags', {}), data.get('tags', {})) and
+                            (member_data.get('state') == data.get('state') or
+                                'running' not in (member_data.get('state'), data.get('state'))) and
                             member_data.get('version') == data.get('version') and
                             member_data.get('checkpoint_after_promote') == data.get('checkpoint_after_promote'))):
             try:
@@ -371,8 +405,7 @@ class ZooKeeper(AbstractDCS):
                 return True
         else:
             try:
-                self._client.create_async(self.member_path, encoded_data, makepath=True,
-                                          ephemeral=not permanent).get(timeout=1)
+                self._client.create_async(self.member_path, encoded_data, makepath=True, ephemeral=True).get(timeout=1)
                 self.__last_member_data = data
                 return True
             except Exception as e:
@@ -397,7 +430,32 @@ class ZooKeeper(AbstractDCS):
     def _write_status(self, value):
         return self._set_or_create(self.status_path, value)
 
+    def _write_failsafe(self, value):
+        return self._set_or_create(self.failsafe_path, value)
+
     def _update_leader(self):
+        cluster = self.cluster
+        session = cluster and isinstance(cluster.leader, Leader) and cluster.leader.session
+        if self._client.client_id and self._client.client_id[0] != session:
+            logger.warning('Recreating the leader ZNode due to ownership mismatch')
+            try:
+                self._client.retry(self._client.delete, self.leader_path)
+            except NoNodeError:
+                pass
+            except (ConnectionClosedError, RetryFailedError) as e:
+                raise ZooKeeperError(e)
+            except Exception as e:
+                logger.error('Failed to remove %s: %r', self.leader_path, e)
+                return False
+
+            try:
+                self._client.retry(self._client.create, self.leader_path,
+                                   self._name.encode('utf-8'), makepath=True, ephemeral=True)
+            except (ConnectionClosedError, RetryFailedError) as e:
+                raise ZooKeeperError(e)
+            except Exception as e:
+                logger.error('Failed to create %s: %r', self.leader_path, e)
+                return False
         return True
 
     def _delete_leader(self):

patroni/log.py

@@ -5,7 +5,7 @@ import sys
 from copy import deepcopy
 from logging.handlers import RotatingFileHandler
 from patroni.utils import deep_compare
-from six.moves.queue import Queue, Full
+from queue import Queue, Full
 from threading import Lock, Thread
 
 _LOGGER = logging.getLogger(__name__)

patroni/postgresql/__init__.py

@@ -3,7 +3,6 @@ import os
 import re
 import shlex
 import shutil
-import six
 import subprocess
 import time
 
@@ -13,29 +12,26 @@ from datetime import datetime
 from dateutil import tz
 from psutil import TimeoutExpired
 from threading import current_thread, Lock
+from typing import Optional
 
 from .bootstrap import Bootstrap
-from .callback_executor import CallbackExecutor
+from .callback_executor import CallbackAction, CallbackExecutor
 from .cancellable import CancellableSubprocess
 from .config import ConfigHandler, mtime
 from .connection import Connection, get_connection_cursor
+from .citus import CitusHandler
 from .misc import parse_history, parse_lsn, postgres_major_version_to_int
 from .postmaster import PostmasterProcess
 from .slots import SlotsHandler
+from .sync import SyncHandler
 from .. import psycopg
+from ..dcs import Member
 from ..exceptions import PostgresConnectionException
 from ..utils import Retry, RetryFailedError, polling_loop, data_directory_is_empty, parse_int
 
 
 logger = logging.getLogger(__name__)
 
-ACTION_ON_START = "on_start"
-ACTION_ON_STOP = "on_stop"
-ACTION_ON_RESTART = "on_restart"
-ACTION_ON_RELOAD = "on_reload"
-ACTION_ON_ROLE_CHANGE = "on_role_change"
-ACTION_NOOP = "noop"
-
 STATE_RUNNING = 'running'
 STATE_REJECT = 'rejecting connections'
 STATE_NO_RESPONSE = 'not responding'
@@ -54,7 +50,7 @@ class Postgresql(object):
     POSTMASTER_START_TIME = "pg_catalog.pg_postmaster_start_time()"
     TL_LSN = ("CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 "
               "ELSE ('x' || pg_catalog.substr(pg_catalog.pg_{0}file_name("
-              "pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END, "  # master timeline
+              "pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END, "  # primary timeline
               "CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 "
               "ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), '0/0')::bigint END, "  # write_lsn
               "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), '0/0')::bigint, "
@@ -75,6 +71,7 @@ class Postgresql(object):
 
         self._pending_restart = False
         self._connection = Connection()
+        self.citus_handler = CitusHandler(self, config.get('citus'))
         self.config = ConfigHandler(self, config)
         self.config.check_directories()
 
@@ -84,6 +81,7 @@ class Postgresql(object):
         self.__thread_ident = current_thread().ident
 
         self.slots_handler = SlotsHandler(self)
+        self.sync_handler = SyncHandler(self)
 
         self._callback_executor = CallbackExecutor()
         self.__cb_called = False
@@ -106,6 +104,7 @@ class Postgresql(object):
         self._cluster_info_state = {}
         self._has_permanent_logical_slots = True
         self._enforce_hot_standby_feedback = False
+        self._is_synchronous_mode = True
         self._cached_replica_timeline = None
 
         # Last known running process
@@ -121,7 +120,7 @@ class Postgresql(object):
             ident_saved = self.config.replace_pg_ident()
             if hba_saved or ident_saved:
                 self.reload()
-        elif self.role == 'master':
+        elif self.role in ('master', 'primary'):
             self.set_role('demoted')
 
     @property
@@ -158,11 +157,36 @@ class Postgresql(object):
 
     @property
     def cluster_info_query(self):
+        """Returns the monitoring query with a fixed number of fields.
+
+        The query text is constructed based on current state in DCS and PostgreSQL version:
+        1. function names depend on version. wal/lsn for v10+ and xlog/location for pre v10.
+        2. for primary we query timeline_id (extracted from pg_walfile_name()) and pg_current_wal_lsn()
+        3. for replicas we query pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn(), and  pg_is_wal_replay_paused()
+        4. for v9.6+ we query primary_slot_name and primary_conninfo from pg_stat_get_wal_receiver()
+        5. for v11+ with permanent logical slots we query from pg_replication_slots and aggregate the result
+        6. for standby_leader node running v9.6+ we also query pg_control_checkpoint to fetch timeline_id
+        7. if sync replication is enabled we query pg_stat_replication and aggregate the result.
+           In addition to that we get current values of synchronous_commit and synchronous_standby_names GUCs.
+
+        If some conditions are not satisfied we simply put static values instead. E.g., NULL, 0, '', and so on."""
+
+        extra = ", " + (("pg_catalog.current_setting('synchronous_commit'), " +
+                         "pg_catalog.current_setting('synchronous_standby_names'), "
+                         "(SELECT pg_catalog.json_agg(r.*) FROM (SELECT w.pid as pid, application_name, sync_state," +
+                         " pg_catalog.pg_{0}_{1}_diff(write_{1}, '0/0')::bigint AS write_lsn," +
+                         " pg_catalog.pg_{0}_{1}_diff(flush_{1}, '0/0')::bigint AS flush_lsn," +
+                         " pg_catalog.pg_{0}_{1}_diff(replay_{1}, '0/0')::bigint AS replay_lsn " +
+                         "FROM pg_catalog.pg_stat_get_wal_senders() w," +
+                         " pg_catalog.pg_stat_get_activity(w.pid)" +
+                         " WHERE w.state = 'streaming') r)").format(self.wal_name, self.lsn_name)
+                        if self._is_synchronous_mode and self.role in ('master', 'primary') else "'on', '', NULL")
+
         if self._major_version >= 90600:
-            extra = "(SELECT pg_catalog.json_agg(s.*) FROM (SELECT slot_name, slot_type as type, datoid::bigint, " +\
-                    "plugin, catalog_xmin, pg_catalog.pg_wal_lsn_diff(confirmed_flush_lsn, '0/0')::bigint" + \
-                    " AS confirmed_flush_lsn FROM pg_catalog.pg_get_replication_slots()) AS s)"\
-                            if self._has_permanent_logical_slots and self._major_version >= 110000 else "NULL"
+            extra = ("(SELECT pg_catalog.json_agg(s.*) FROM (SELECT slot_name, slot_type as type, datoid::bigint, " +
+                     "plugin, catalog_xmin, pg_catalog.pg_wal_lsn_diff(confirmed_flush_lsn, '0/0')::bigint" +
+                     " AS confirmed_flush_lsn FROM pg_catalog.pg_get_replication_slots()) AS s)"
+                     if self._has_permanent_logical_slots and self._major_version >= 110000 else "NULL") + extra
             extra = (", CASE WHEN latest_end_lsn IS NULL THEN NULL ELSE received_tli END,"
                      " slot_name, conninfo, {0} FROM pg_catalog.pg_stat_get_wal_receiver()").format(extra)
             if self.role == 'standby_leader':
@@ -170,14 +194,17 @@ class Postgresql(object):
             else:
                 extra = "0" + extra
         else:
-            extra = "0, NULL, NULL, NULL, NULL"
+            extra = "0, NULL, NULL, NULL, NULL" + extra
 
         return ("SELECT " + self.TL_LSN + ", {2}").format(self.wal_name, self.lsn_name, extra)
 
     def _version_file_exists(self):
         return not self.data_directory_empty() and os.path.isfile(self._version_file)
 
-    def get_major_version(self):
+    def get_major_version(self) -> int:
+        """Reads major version from PG_VERSION file
+
+        :returns: major PostgreSQL version in integer format or 0 in case of missing file or errors"""
         if self._version_file_exists():
             try:
                 with open(self._version_file) as f:
@@ -255,7 +282,8 @@ class Postgresql(object):
         return self._connection.get()
 
     def set_connection_kwargs(self, kwargs):
-        self._connection.set_conn_kwargs(kwargs)
+        self._connection.set_conn_kwargs(kwargs.copy())
+        self.citus_handler.set_conn_kwargs(kwargs.copy())
 
     def _query(self, sql, *params):
         """We are always using the same cursor, therefore this method is not thread-safe!!!
@@ -300,7 +328,8 @@ class Postgresql(object):
         return deepcopy(self.config.get(method, {}))
 
     def replica_method_can_work_without_replication_connection(self, method):
-        return method != 'basebackup' and self.replica_method_options(method).get('no_master')
+        return method != 'basebackup' and (self.replica_method_options(method).get('no_master') or
+                                           self.replica_method_options(method).get('no_leader'))
 
     def can_create_replica_without_replication_connection(self, replica_methods=None):
         """ go through the replication methods to see if there are ones
@@ -334,13 +363,16 @@ class Postgresql(object):
                 self._has_permanent_logical_slots or
                 cluster.should_enforce_hot_standby_feedback(self.name, nofailover, self.major_version))
 
+            self._is_synchronous_mode = cluster.is_synchronous_mode()
+
     def _cluster_info_state_get(self, name):
         if not self._cluster_info_state:
             try:
                 result = self._is_leader_retry(self._query, self.cluster_info_query).fetchone()
                 cluster_info_state = dict(zip(['timeline', 'wal_position', 'replayed_location',
                                                'received_location', 'replay_paused', 'pg_control_timeline',
-                                               'received_tli', 'slot_name', 'conninfo', 'slots'], result))
+                                               'received_tli', 'slot_name', 'conninfo', 'slots', 'synchronous_commit',
+                                               'synchronous_standby_names', 'pg_stat_replication'], result))
                 if self._has_permanent_logical_slots:
                     cluster_info_state['slots'] =\
                         self.slots_handler.process_permanent_slots(cluster_info_state['slots'])
@@ -373,12 +405,21 @@ class Postgresql(object):
     def received_timeline(self):
         return self._cluster_info_state_get('received_tli')
 
+    def synchronous_commit(self):
+        return self._cluster_info_state_get('synchronous_commit')
+
+    def synchronous_standby_names(self):
+        return self._cluster_info_state_get('synchronous_standby_names')
+
+    def pg_stat_replication(self):
+        return self._cluster_info_state_get('pg_stat_replication') or []
+
     def is_leader(self):
         try:
             return bool(self._cluster_info_state_get('timeline'))
         except PostgresConnectionException:
             logger.warning('Failed to determine PostgreSQL state from the connection, falling back to cached role')
-            return bool(self.is_running() and self.role == 'master')
+            return bool(self.is_running() and self.role in ('master', 'primary'))
 
     def replay_paused(self):
         return self._cluster_info_state_get('replay_paused')
@@ -428,11 +469,11 @@ class Postgresql(object):
                     # If the cluster is shutdown with archive_mode=on, WAL is switched before writing the checkpoint.
                     # In this case we want to take the LSN of previous record (switch) as the last known WAL location.
                     if parse_lsn(lsn) == prev and desc.strip() in ('xlog switch', 'SWITCH'):
-                        return str(prev)
+                        return prev
             except Exception as e:
                 logger.error('Exception when parsing WAL pg_%sdump output: %r', self.wal_name, e)
-            if isinstance(checkpoint_lsn, six.integer_types):
-                return str(checkpoint_lsn)
+            if isinstance(checkpoint_lsn, int):
+                return checkpoint_lsn
 
     def is_running(self):
         """Returns PostmasterProcess if one is running on the data directory or None. If most recently seen process
@@ -452,20 +493,22 @@ class Postgresql(object):
     def cb_called(self):
         return self.__cb_called
 
-    def call_nowait(self, cb_name):
-        """ pick a callback command and call it without waiting for it to finish """
+    def call_nowait(self, cb_type: CallbackAction) -> None:
+        """pick a callback command and call it without waiting for it to finish """
         if self.bootstrapping:
             return
-        if cb_name in (ACTION_ON_START, ACTION_ON_STOP, ACTION_ON_RESTART, ACTION_ON_ROLE_CHANGE):
+        if cb_type in (CallbackAction.ON_START, CallbackAction.ON_STOP,
+                       CallbackAction.ON_RESTART, CallbackAction.ON_ROLE_CHANGE):
             self.__cb_called = True
 
-        if self.callback and cb_name in self.callback:
-            cmd = self.callback[cb_name]
+        if self.callback and cb_type in self.callback:
+            cmd = self.callback[cb_type]
+            role = 'master' if self.role == 'promoted' else self.role
             try:
-                cmd = shlex.split(self.callback[cb_name]) + [cb_name, self.role, self.scope]
+                cmd = shlex.split(self.callback[cb_type]) + [cb_type, role, self.scope]
                 self._callback_executor.call(cmd)
             except Exception:
-                logger.exception('callback %s %s %s %s failed', cmd, cb_name, self.role, self.scope)
+                logger.exception('callback %s %r %s %s failed', cmd, cb_type, role, self.scope)
 
     @property
     def role(self):
@@ -512,13 +555,14 @@ class Postgresql(object):
         logger.warning("Timed out waiting for PostgreSQL to start")
         return False
 
-    def start(self, timeout=None, task=None, block_callbacks=False, role=None):
+    def start(self, timeout=None, task=None, block_callbacks=False, role=None, after_start=None):
         """Start PostgreSQL
 
         Waits for postmaster to open ports or terminate so pg_isready can be used to check startup completion
         or failure.
 
-        :returns: True if start was initiated and postmaster ports are open, False if start failed"""
+        :returns: True if start was initiated and postmaster ports are open,
+                  False if start failed, and None if postgres is still starting up"""
         # make sure we close all connections established against
         # the former node, otherwise, we might get a stalled one
         # after kill -9, which would report incorrect data to
@@ -531,7 +575,7 @@ class Postgresql(object):
             return True
 
         if not block_callbacks:
-            self.__cb_pending = ACTION_ON_START
+            self.__cb_pending = CallbackAction.ON_START
 
         self.set_role(role or self.get_postgres_role_from_data_directory())
 
@@ -539,8 +583,8 @@ class Postgresql(object):
         self._pending_restart = False
 
         try:
-            if not self._major_version:
-                self.configure_server_parameters()
+            if not self.ensure_major_version_is_known():
+                return None
             configuration = self.config.effective_configuration
         except Exception:
             return None
@@ -583,6 +627,8 @@ class Postgresql(object):
 
         ret = self.wait_for_startup(start_timeout)
         if ret is not None:
+            if ret and after_start:
+                after_start()
             return ret
         elif timeout is not None:
             return False
@@ -609,7 +655,7 @@ class Postgresql(object):
             return 'not accessible or not healty'
 
     def stop(self, mode='fast', block_callbacks=False, checkpoint=None,
-             on_safepoint=None, on_shutdown=None, stop_timeout=None):
+             on_safepoint=None, on_shutdown=None, before_shutdown=None, stop_timeout=None):
         """Stop PostgreSQL
 
         Supports a callback when a safepoint is reached. A safepoint is when no user backend can return a successful
@@ -618,24 +664,26 @@ class Postgresql(object):
 
         :param on_safepoint: This callback is called when no user backends are running.
         :param on_shutdown: is called when pg_controldata starts reporting `Database cluster state: shut down`
+        :param before_shutdown: is called after running optional CHECKPOINT and before running pg_ctl stop
         """
         if checkpoint is None:
             checkpoint = False if mode == 'immediate' else True
 
-        success, pg_signaled = self._do_stop(mode, block_callbacks, checkpoint, on_safepoint, on_shutdown, stop_timeout)
+        success, pg_signaled = self._do_stop(mode, block_callbacks, checkpoint, on_safepoint,
+                                             on_shutdown, before_shutdown, stop_timeout)
         if success:
             # block_callbacks is used during restart to avoid
             # running start/stop callbacks in addition to restart ones
             if not block_callbacks:
                 self.set_state('stopped')
                 if pg_signaled:
-                    self.call_nowait(ACTION_ON_STOP)
+                    self.call_nowait(CallbackAction.ON_STOP)
         else:
             logger.warning('pg_ctl stop failed')
             self.set_state('stop failed')
         return success
 
-    def _do_stop(self, mode, block_callbacks, checkpoint, on_safepoint, on_shutdown, stop_timeout):
+    def _do_stop(self, mode, block_callbacks, checkpoint, on_safepoint, on_shutdown, before_shutdown, stop_timeout):
         postmaster = self.is_running()
         if not postmaster:
             if on_safepoint:
@@ -648,6 +696,9 @@ class Postgresql(object):
         if not block_callbacks:
             self.set_state('stopping')
 
+        if before_shutdown:
+            before_shutdown()
+
         # Send signal to postmaster to stop
         success = postmaster.signal_stop(mode, self.pgcommand('pg_ctl'))
         if success is not None:
@@ -659,7 +710,7 @@ class Postgresql(object):
         if on_safepoint:
             # Wait for our connection to terminate so we can be sure that no new connections are being initiated
             self._wait_for_connection_close(postmaster)
-            postmaster.wait_for_user_backends_to_close()
+            postmaster.wait_for_user_backends_to_close(stop_timeout)
             on_safepoint()
 
         if on_shutdown and mode in ('fast', 'smart'):
@@ -668,7 +719,7 @@ class Postgresql(object):
             while postmaster.is_running():
                 data = self.controldata()
                 if data.get('Database cluster state', '') == 'shut down':
-                    on_shutdown(int(self.latest_checkpoint_location()))
+                    on_shutdown(self.latest_checkpoint_location())
                     break
                 elif data.get('Database cluster state', '').startswith('shut down'):  # shut down in recovery
                     break
@@ -718,7 +769,7 @@ class Postgresql(object):
     def reload(self, block_callbacks=False):
         ret = self.pg_ctl('reload')
         if ret and not block_callbacks:
-            self.call_nowait(ACTION_ON_RELOAD)
+            self.call_nowait(CallbackAction.ON_RELOAD)
         return ret
 
     def check_for_startup(self):
@@ -754,7 +805,7 @@ class Postgresql(object):
             self.config.save_configuration_files(True)
             # TODO: __cb_pending can be None here after PostgreSQL restarts on its own. Do we want to call the callback?
             # Previously we didn't even notice.
-            action = self.__cb_pending or ACTION_ON_START
+            action = self.__cb_pending or CallbackAction.ON_START
             self.call_nowait(action)
             self.__cb_pending = None
 
@@ -775,7 +826,8 @@ class Postgresql(object):
 
         return self.state == 'running'
 
-    def restart(self, timeout=None, task=None, block_callbacks=False, role=None):
+    def restart(self, timeout=None, task=None, block_callbacks=False,
+                role=None, before_shutdown=None, after_start=None):
         """Restarts PostgreSQL.
 
         When timeout parameter is set the call will block either until PostgreSQL has started, failed to start or
@@ -785,8 +837,9 @@ class Postgresql(object):
         """
         self.set_state('restarting')
         if not block_callbacks:
-            self.__cb_pending = ACTION_ON_RESTART
-        ret = self.stop(block_callbacks=True) and self.start(timeout, task, True, role)
+            self.__cb_pending = CallbackAction.ON_RESTART
+        ret = self.stop(block_callbacks=True, before_shutdown=before_shutdown)\
+            and self.start(timeout, task, True, role, after_start)
         if not ret and not self.is_starting():
             self.set_state('restart failed ({0})'.format(self.state))
         return ret
@@ -812,8 +865,7 @@ class Postgresql(object):
         # Don't try to call pg_controldata during backup restore
         if self._version_file_exists() and self.state != 'creating replica':
             try:
-                env = os.environ.copy()
-                env.update(LANG='C', LC_ALL='C')
+                env = {**os.environ, 'LANG': 'C', 'LC_ALL': 'C'}
                 data = subprocess.check_output([self.pgcommand('pg_controldata'), self._data_dir], env=env)
                 if data:
                     data = filter(lambda e: ':' in e, data.decode('utf-8').splitlines())
@@ -825,8 +877,7 @@ class Postgresql(object):
 
     def waldump(self, timeline, lsn, limit):
         cmd = self.pgcommand('pg_{0}dump'.format(self.wal_name))
-        env = os.environ.copy()
-        env.update(LANG='C', LC_ALL='C', PGDATA=self._data_dir)
+        env = {**os.environ, 'LANG': 'C', 'LC_ALL': 'C', 'PGDATA': self._data_dir}
         try:
             waldump = subprocess.Popen([cmd, '-t', str(timeline), '-s', str(lsn), '-n', str(limit)],
                                        stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
@@ -853,12 +904,13 @@ class Postgresql(object):
         except Exception:
             logger.exception('Can not fetch local timeline and lsn from replication connection')
 
-    def replica_cached_timeline(self, master_timeline):
-        if not self._cached_replica_timeline or not master_timeline or self._cached_replica_timeline != master_timeline:
+    def replica_cached_timeline(self, primary_timeline):
+        if not self._cached_replica_timeline or not primary_timeline\
+                or self._cached_replica_timeline != primary_timeline:
             self._cached_replica_timeline = self.get_replica_timeline()
         return self._cached_replica_timeline
 
-    def get_master_timeline(self):
+    def get_primary_timeline(self):
         return self._cluster_info_state_get('timeline')
 
     def get_history(self, timeline):
@@ -877,18 +929,38 @@ class Postgresql(object):
             except Exception:
                 logger.exception('Failed to read and parse %s', (history_path,))
 
-    def follow(self, member, role='replica', timeout=None, do_reload=False):
+    def follow(self,
+               member: Member,
+               role: Optional[str] = 'replica',
+               timeout: Optional[float] = None,
+               do_reload: Optional[bool] = False) -> Optional[bool]:
+        """Reconfigure postgres to follow a new member or use different recovery parameters.
+
+        Method may call `on_role_change` callback if role is changing.
+
+        :param member: The member to follow
+        :param role: The desired role, normally 'replica', but could also be a 'standby_leader'
+        :param timeout: start timeout, how long should the `start()` method wait for postgres accepting connections
+        :param do_reload: indicates that after updating postgresql.conf we just need to do a reload instead of restart
+
+        :returns: True - if restart/reload were successfully performed,
+                  False - if restart/reload failed
+                  None - if nothing was done or if Postgres is still in starting state after `timeout` seconds."""
+
+        if not self.ensure_major_version_is_known():
+            return None
+
         recovery_params = self.config.build_recovery_params(member)
         self.config.write_recovery_conf(recovery_params)
 
-        # When we demoting the master or standby_leader to replica or promoting replica to a standby_leader
+        # When we demoting the primary or standby_leader to replica or promoting replica to a standby_leader
         # and we know for sure that postgres was already running before, we will only execute on_role_change
         # callback and prevent execution of on_restart/on_start callback.
         # If the role remains the same (replica or standby_leader), we will execute on_start or on_restart
-        change_role = self.cb_called and (self.role in ('master', 'demoted') or
+        change_role = self.cb_called and (self.role in ('master', 'primary', 'demoted') or
                                           not {'standby_leader', 'replica'} - {self.role, role})
         if change_role:
-            self.__cb_pending = ACTION_NOOP
+            self.__cb_pending = CallbackAction.NOOP
 
         ret = True
         if self.is_running():
@@ -904,13 +976,14 @@ class Postgresql(object):
 
         if change_role:
             # TODO: postpone this until start completes, or maybe do even earlier
-            self.call_nowait(ACTION_ON_ROLE_CHANGE)
+            self.call_nowait(CallbackAction.ON_ROLE_CHANGE)
         return ret
 
     def _wait_promote(self, wait_seconds):
         for _ in polling_loop(wait_seconds):
             data = self.controldata()
             if data.get('Database cluster state') == 'in production':
+                self.set_role('master')
                 return True
 
     def _pre_promote(self):
@@ -928,8 +1001,8 @@ class Postgresql(object):
             logger.info('pre_promote script `%s` exited with %s', cmd, ret)
         return ret == 0
 
-    def promote(self, wait_seconds, task, on_success=None):
-        if self.role == 'master':
+    def promote(self, wait_seconds, task, before_promote=None, on_success=None):
+        if self.role in ('promoted', 'master', 'primary'):
             return True
 
         ret = self._pre_promote()
@@ -945,14 +1018,18 @@ class Postgresql(object):
             logger.info("PostgreSQL promote cancelled.")
             return False
 
+        if before_promote is not None:
+            before_promote()
+
         self.slots_handler.on_promote()
+        self.citus_handler.schedule_cache_rebuild()
 
         ret = self.pg_ctl('promote', '-W')
         if ret:
-            self.set_role('master')
+            self.set_role('promoted')
             if on_success is not None:
                 on_success()
-            self.call_nowait(ACTION_ON_ROLE_CHANGE)
+            self.call_nowait(CallbackAction.ON_ROLE_CHANGE)
             ret = self._wait_promote(wait_seconds)
         return ret
 
@@ -996,7 +1073,15 @@ class Postgresql(object):
     def configure_server_parameters(self):
         self._major_version = self.get_major_version()
         self.config.setup_server_parameters()
-        return True
+
+    def ensure_major_version_is_known(self) -> bool:
+        """Calls configure_server_parameters() if `_major_version` is not known
+
+        :returns: `True` if `_major_version` is set, otherwise `False`"""
+
+        if not self._major_version:
+            self.configure_server_parameters()
+        return self._major_version > 0
 
     def pg_wal_realpath(self):
         """Returns a dict containing the symlink (key) and target (value) for the wal directory"""
@@ -1023,13 +1108,15 @@ class Postgresql(object):
     def move_data_directory(self):
         if os.path.isdir(self._data_dir) and not self.is_running():
             try:
-                postfix = time.strftime('%Y-%m-%d-%H-%M-%S')
+                postfix = 'failed'
 
                 # let's see if the wal directory is a symlink, in this case we
                 # should move the target
                 for (source, pg_wal_realpath) in self.pg_wal_realpath().items():
                     logger.info('renaming WAL directory and updating symlink: %s', pg_wal_realpath)
-                    new_name = '{0}_{1}'.format(pg_wal_realpath, postfix)
+                    new_name = '{0}.{1}'.format(pg_wal_realpath, postfix)
+                    if os.path.exists(new_name):
+                        shutil.rmtree(new_name)
                     os.rename(pg_wal_realpath, new_name)
                     os.unlink(source)
                     os.symlink(new_name, source)
@@ -1037,13 +1124,17 @@ class Postgresql(object):
                 # Move user defined tablespace directory
                 for (source, pg_tsp_rpath) in self.pg_tblspc_realpaths().items():
                     logger.info('renaming user defined tablespace directory and updating symlink: %s', pg_tsp_rpath)
-                    new_name = '{0}_{1}'.format(pg_tsp_rpath, postfix)
+                    new_name = '{0}.{1}'.format(pg_tsp_rpath, postfix)
+                    if os.path.exists(new_name):
+                        shutil.rmtree(new_name)
                     os.rename(pg_tsp_rpath, new_name)
                     os.unlink(source)
                     os.symlink(new_name, source)
 
-                new_name = '{0}_{1}'.format(self._data_dir, postfix)
+                new_name = '{0}.{1}'.format(self._data_dir, postfix)
                 logger.info('renaming data directory to %s', new_name)
+                if os.path.exists(new_name):
+                    shutil.rmtree(new_name)
                 os.rename(self._data_dir, new_name)
             except OSError:
                 logger.exception("Could not rename data directory %s", self._data_dir)
@@ -1076,58 +1167,6 @@ class Postgresql(object):
             logger.exception('Could not remove data directory %s', self._data_dir)
             self.move_data_directory()
 
-    def _get_synchronous_commit_param(self):
-        return self.query("SHOW synchronous_commit").fetchone()[0]
-
-    def pick_synchronous_standby(self, cluster, sync_node_count=1, sync_node_maxlag=-1):
-        """Finds the best candidate to be the synchronous standby.
-
-        Current synchronous standby is always preferred, unless it has disconnected or does not want to be a
-        synchronous standby any longer.
-        Parameter sync_node_maxlag(maximum_lag_on_syncnode) would help swapping unhealthy sync replica in case
-        if it stops responding (or hung). Please set the value high enough so it won't unncessarily swap sync
-        standbys during high loads. Any less or equal of 0 value keep the behavior backward compatible and
-        will not swap. Please note that it will not also swap sync standbys in case where all replicas are hung.
-
-        :returns tuple of candidates list and synchronous standby list.
-        """
-        if self._major_version < 90600:
-            sync_node_count = 1
-        members = {m.name.lower(): m for m in cluster.members}
-        candidates = []
-        sync_nodes = []
-        replica_list = []
-        # Pick candidates based on who has higher replay/remote_write/flush lsn.
-        sync_commit_par = self._get_synchronous_commit_param()
-        sort_col = {'remote_apply': 'replay', 'remote_write': 'write'}.get(sync_commit_par, 'flush')
-        # pg_stat_replication.sync_state has 4 possible states - async, potential, quorum, sync.
-        # Sort clause "ORDER BY sync_state DESC" is to get the result in required order and to keep
-        # the result consistent in case if a synchronous standby member is slowed down OR async node
-        # receiving changes faster than the sync member (very rare but possible). Such cases would
-        # trigger sync standby member swapping frequently and the sort on sync_state desc should
-        # help in keeping the query result consistent.
-        for app_name, sync_state, replica_lsn in self.query(
-                "SELECT pg_catalog.lower(application_name), sync_state, pg_{2}_{1}_diff({0}_{1}, '0/0')::bigint"
-                " FROM pg_catalog.pg_stat_replication"
-                " WHERE state = 'streaming' AND {0}_{1} IS NOT NULL"
-                " ORDER BY sync_state DESC, {0}_{1} DESC".format(sort_col, self.lsn_name, self.wal_name)):
-            member = members.get(app_name)
-            if member and not member.tags.get('nosync', False):
-                replica_list.append((member.name, sync_state, replica_lsn, bool(member.nofailover)))
-
-        max_lsn = max(replica_list, key=lambda x: x[2])[2] if len(replica_list) > 1 else int(str(self.last_operation()))
-
-        # Prefer members without nofailover tag. We are relying on the fact that sorts are guaranteed to be stable.
-        for app_name, sync_state, replica_lsn, _ in sorted(replica_list, key=lambda x: x[3]):
-            if sync_node_maxlag <= 0 or max_lsn - replica_lsn <= sync_node_maxlag:
-                candidates.append(app_name)
-                if sync_state == 'sync':
-                    sync_nodes.append(app_name)
-            if len(candidates) >= sync_node_count:
-                break
-
-        return candidates, sync_nodes
-
     def schedule_sanity_checks_after_pause(self):
         """
             After coming out of pause we have to:
@@ -1135,7 +1174,7 @@ class Postgresql(object):
             2. sync replication slots, because it might happen that slots were removed
             3. get new 'Database system identifier' to make sure that it wasn't changed
         """
-        if not self._major_version:
-            self.configure_server_parameters()
+        self.ensure_major_version_is_known()
         self.slots_handler.schedule()
+        self.citus_handler.schedule_cache_rebuild()
         self._sysid = None

patroni/postgresql/bootstrap.py

@@ -4,8 +4,6 @@ import shlex
 import tempfile
 import time
 
-from six import string_types
-
 from ..dcs import RemoteMember
 from ..psycopg import quote_ident, quote_literal
 from ..utils import deep_compare
@@ -43,11 +41,11 @@ class Bootstrap(object):
                     user_options.append('--{0}={1}'.format(k, v))
         elif isinstance(options, list):
             for opt in options:
-                if isinstance(opt, string_types) and option_is_allowed(opt):
+                if isinstance(opt, str) and option_is_allowed(opt):
                     user_options.append('--{0}'.format(opt))
                 elif isinstance(opt, dict):
                     keys = list(opt.keys())
-                    if len(keys) != 1 or not isinstance(opt[keys[0]], string_types) or not option_is_allowed(keys[0]):
+                    if len(keys) != 1 or not isinstance(opt[keys[0]], str) or not option_is_allowed(keys[0]):
                         error_handler('Error when parsing {0} key-value option {1}: only one key-value is allowed'
                                       ' and value should be a string'.format(tool, opt[keys[0]]))
                     user_options.append('--{0}={1}'.format(keys[0], opt[keys[0]]))
@@ -59,7 +57,7 @@ class Bootstrap(object):
         return user_options
 
     def _initdb(self, config):
-        self._postgresql.set_state('initalizing new cluster')
+        self._postgresql.set_state('initializing new cluster')
         not_allowed_options = ('pgdata', 'nosync', 'pwfile', 'sync-only', 'version')
 
         def error_handler(e):
@@ -155,12 +153,12 @@ class Bootstrap(object):
         self._postgresql.set_state('creating replica')
         self._postgresql.schedule_sanity_checks_after_pause()
 
-        is_remote_master = isinstance(clone_member, RemoteMember)
+        is_remote_member = isinstance(clone_member, RemoteMember)
 
         # get list of replica methods either from clone member or from
         # the config. If there is no configuration key, or no value is
         # specified, use basebackup
-        replica_methods = (clone_member.create_replica_methods if is_remote_master
+        replica_methods = (clone_member.create_replica_methods if is_remote_member
                            else self._postgresql.create_replica_methods) or ['basebackup']
 
         if clone_member and clone_member.conn_url:
@@ -212,7 +210,7 @@ class Bootstrap(object):
                                           "datadir": self._postgresql.data_dir,
                                           "connstring": connstring})
                 else:
-                    for param in ('no_params', 'no_master', 'keep_data'):
+                    for param in ('no_params', 'no_master', 'no_leader', 'keep_data'):
                         method_config.pop(param, None)
                 params = ["--{0}={1}".format(arg, val) for arg, val in method_config.items()]
                 try:
@@ -269,7 +267,7 @@ class Bootstrap(object):
 
     def clone(self, clone_member):
         """
-             - initialize the replica from an existing member (master or replica)
+             - initialize the replica from an existing member (primary or replica)
              - initialize the replica using the replica creation method that
                works without the replication connection (i.e. restore from on-disk
                base backup)
@@ -315,12 +313,14 @@ END;$$""".format(quote_literal(name), quote_ident(name, self._postgresql.connect
         self._postgresql.query('SET log_statement TO none')
         self._postgresql.query('SET log_min_duration_statement TO -1')
         self._postgresql.query("SET log_min_error_statement TO 'log'")
+        self._postgresql.query("SET pg_stat_statements.track_utility to 'off'")
         try:
             self._postgresql.query(sql)
         finally:
             self._postgresql.query('RESET log_min_error_statement')
             self._postgresql.query('RESET log_min_duration_statement')
             self._postgresql.query('RESET log_statement')
+            self._postgresql.query('RESET pg_stat_statements.track_utility')
 
     def post_bootstrap(self, config, task):
         try:
@@ -343,7 +343,7 @@ END;$$""".format(quote_literal(name), quote_ident(name, self._postgresql.connect
 BEGIN
     SET local synchronous_commit = 'local';
     GRANT EXECUTE ON function pg_catalog.{0} TO {1};
-END;$$""".format(f, quote_ident(rewind['username'], self._postgresql.connection()))
+END;$$""".format(f, quote_ident(rewind['username'], postgresql.connection()))
                         postgresql.query(sql)
 
                 for name, value in (config.get('users') or {}).items():
@@ -375,6 +375,9 @@ END;$$""".format(f, quote_ident(rewind['username'], self._postgresql.connection(
                             postgresql.reload()
                             time.sleep(1)  # give a time to postgres to "reload" configuration files
                             postgresql.connection().close()  # close connection to reconnect with a new password
+                else:  # initdb
+                    # We may want create database and extension for citus
+                    self._postgresql.citus_handler.bootstrap()
         except Exception:
             logger.exception('post_bootstrap')
             task.complete(False)

patroni/postgresql/callback_executor.py

@@ -1,22 +1,60 @@
 import logging
 
-from patroni.postgresql.cancellable import CancellableExecutor
+from enum import Enum
 from threading import Condition, Thread
+from typing import List
+
+from .cancellable import CancellableExecutor, CancellableSubprocess
 
 logger = logging.getLogger(__name__)
 
 
+class CallbackAction(str, Enum):
+    NOOP = "noop"
+    ON_START = "on_start"
+    ON_STOP = "on_stop"
+    ON_RESTART = "on_restart"
+    ON_RELOAD = "on_reload"
+    ON_ROLE_CHANGE = "on_role_change"
+
+    def __repr__(self):
+        return self.value
+
+
+class OnReloadExecutor(CancellableSubprocess):
+
+    def call_nowait(self, cmd: List[str]) -> None:
+        """Run one `on_reload` callback at most.
+
+        To achieve it we always kill already running command including child processes."""
+        self.cancel(kill=True)
+        self._kill_children()
+        with self._lock:
+            self._start_process(cmd, close_fds=True)
+
+
 class CallbackExecutor(CancellableExecutor, Thread):
 
     def __init__(self):
         CancellableExecutor.__init__(self)
         Thread.__init__(self)
         self.daemon = True
+        self._on_reload_executor = OnReloadExecutor()
         self._cmd = None
         self._condition = Condition()
         self.start()
 
-    def call(self, cmd):
+    def call(self, cmd: List[str]) -> None:
+        """Executes one callback at a time.
+
+        Already running command is killed (including child processes).
+        If it couldn't be killed we wait until it finishes.
+
+        :param cmd: command to be executed"""
+
+        if cmd[-3] == CallbackAction.ON_RELOAD:
+            return self._on_reload_executor.call_nowait(cmd)
+
         self._kill_process()
         with self._condition:
             self._cmd = cmd

patroni/postgresql/citus.py

@@ -0,0 +1,389 @@
+import logging
+import re
+import time
+
+from threading import Condition, Event, Thread
+from urllib.parse import urlparse
+
+from .connection import Connection
+from ..dcs import CITUS_COORDINATOR_GROUP_ID
+from ..psycopg import connect, quote_ident
+
+CITUS_SLOT_NAME_RE = re.compile(r'^citus_shard_(move|split)_slot(_[1-9][0-9]*){2,3}$')
+logger = logging.getLogger(__name__)
+
+
+class PgDistNode(object):
+    """Represents a single row in the `pg_dist_node` table"""
+
+    def __init__(self, group, host, port, event, nodeid=None, timeout=None, cooldown=None):
+        self.group = group
+        # A weird way of pausing client connections by adding the `-demoted` suffix to the hostname
+        self.host = host + ('-demoted' if event == 'before_demote' else '')
+        self.port = port
+        # Event that is trying to change or changed the given row.
+        # Possible values: before_demote, before_promote, after_promote.
+        self.event = event
+        self.nodeid = nodeid
+
+        # If transaction was started, we need to COMMIT/ROLLBACK before the deadline
+        self.timeout = timeout
+        self.cooldown = cooldown or 10000  # 10s by default
+        self.deadline = 0
+
+        # All changes in the pg_dist_node are serialized on the Patroni
+        # side by performing them from a thread. The thread, that is
+        # requested a change, sometimes needs to wait for a result.
+        # For example, we want to pause client connections before demoting
+        # the worker, and once it is done notify the calling thread.
+        self._event = Event()
+
+    def wait(self):
+        self._event.wait()
+
+    def wakeup(self):
+        self._event.set()
+
+    def __eq__(self, other):
+        return isinstance(other, PgDistNode) and self.event == other.event\
+            and self.host == other.host and self.port == other.port
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __str__(self):
+        return ('PgDistNode(nodeid={0},group={1},host={2},port={3},event={4})'
+                .format(self.nodeid, self.group, self.host, self.port, self.event))
+
+    def __repr__(self):
+        return str(self)
+
+
+class CitusHandler(Thread):
+
+    def __init__(self, postgresql, config):
+        super(CitusHandler, self).__init__()
+        self.daemon = True
+        self._postgresql = postgresql
+        self._config = config
+        self._connection = Connection()
+        self._pg_dist_node = {}  # Cache of pg_dist_node: {groupid: PgDistNode()}
+        self._tasks = []  # Requests to change pg_dist_node, every task is a `PgDistNode`
+        self._condition = Condition()  # protects _pg_dist_node, _tasks, and _schedule_load_pg_dist_node
+        self._in_flight = None  # Reference to the `PgDistNode` if there is a transaction in progress changing it
+        self.schedule_cache_rebuild()
+
+    def is_enabled(self):
+        return isinstance(self._config, dict)
+
+    def group(self):
+        return self._config['group']
+
+    def is_coordinator(self):
+        return self.is_enabled() and self.group() == CITUS_COORDINATOR_GROUP_ID
+
+    def is_worker(self):
+        return self.is_enabled() and not self.is_coordinator()
+
+    def set_conn_kwargs(self, kwargs):
+        if self.is_enabled():
+            kwargs.update({'dbname': self._config['database'],
+                           'options': '-c statement_timeout=0 -c idle_in_transaction_session_timeout=0'})
+            self._connection.set_conn_kwargs(kwargs)
+
+    def schedule_cache_rebuild(self):
+        with self._condition:
+            self._schedule_load_pg_dist_node = True
+
+    def on_demote(self):
+        with self._condition:
+            self._pg_dist_node.clear()
+            self._tasks[:] = []
+            self._in_flight = None
+
+    def query(self, sql, *params):
+        try:
+            logger.debug('query(%s, %s)', sql, params)
+            cursor = self._connection.cursor()
+            cursor.execute(sql, params or None)
+            return cursor
+        except Exception as e:
+            logger.error('Exception when executing query "%s", (%s): %r', sql, params, e)
+            self._connection.close()
+            self._in_flight = None
+            self.schedule_cache_rebuild()
+            raise e
+
+    def load_pg_dist_node(self):
+        """Read from the `pg_dist_node` table and put it into the local cache"""
+
+        with self._condition:
+            if not self._schedule_load_pg_dist_node:
+                return True
+            self._schedule_load_pg_dist_node = False
+
+        try:
+            cursor = self.query("SELECT nodeid, groupid, nodename, nodeport, noderole"
+                                " FROM pg_catalog.pg_dist_node WHERE noderole = 'primary'")
+        except Exception:
+            return False
+
+        with self._condition:
+            self._pg_dist_node = {r[1]: PgDistNode(r[1], r[2], r[3], 'after_promote', r[0]) for r in cursor}
+        return True
+
+    def sync_pg_dist_node(self, cluster):
+        """Maintain the `pg_dist_node` from the coordinator leader every heartbeat loop.
+
+        We can't always rely on REST API calls from worker nodes in order
+        to maintain `pg_dist_node`, therefore at least once per heartbeat
+        loop we make sure that workes registered in `self._pg_dist_node`
+        cache are matching the cluster view from DCS by creating tasks
+        the same way as it is done from the REST API."""
+
+        if not self.is_coordinator():
+            return
+
+        with self._condition:
+            if not self.is_alive():
+                self.start()
+
+        self.add_task('after_promote', CITUS_COORDINATOR_GROUP_ID, self._postgresql.connection_string)
+
+        for group, worker in cluster.workers.items():
+            leader = worker.leader
+            if leader and leader.conn_url\
+                    and leader.data.get('role') in ('master', 'primary') and leader.data.get('state') == 'running':
+                self.add_task('after_promote', group, leader.conn_url)
+
+    def find_task_by_group(self, group):
+        for i, task in enumerate(self._tasks):
+            if task.group == group:
+                return i
+
+    def pick_task(self):
+        """Returns the tuple(i, task), where `i` - is the task index in the self._tasks list
+
+        Tasks are picked by following priorities:
+        1. If there is already a transaction in progress, pick a task
+           that that will change already affected worker primary.
+        2. If the coordinator address should be changed - pick a task
+           with group=0 (coordinators are always in group 0).
+        3. Pick a task that is the oldest (first from the self._tasks)"""
+
+        with self._condition:
+            if self._in_flight:
+                i = self.find_task_by_group(self._in_flight.group)
+            else:
+                while True:
+                    i = self.find_task_by_group(CITUS_COORDINATOR_GROUP_ID)  # set_coordinator
+                    if i is None and self._tasks:
+                        i = 0
+                    if i is None:
+                        break
+                    task = self._tasks[i]
+                    if task == self._pg_dist_node.get(task.group):
+                        self._tasks.pop(i)  # nothing to do because cached version of pg_dist_node already matches
+                    else:
+                        break
+            task = self._tasks[i] if i is not None else None
+
+            # When tasks are added it could happen that self._pg_dist_node
+            # wasn't ready (self._schedule_load_pg_dist_node is False)
+            # and hence the nodeid wasn't filled.
+            if task and task.group in self._pg_dist_node:
+                task.nodeid = self._pg_dist_node[task.group].nodeid
+            return i, task
+
+    def update_node(self, task):
+        if task.nodeid is not None:
+            self.query('SELECT pg_catalog.citus_update_node(%s, %s, %s, true, %s)',
+                       task.nodeid, task.host, task.port, task.cooldown)
+        elif task.event != 'before_demote':
+            task.nodeid = self.query("SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default')",
+                                     task.host, task.port, task.group).fetchone()[0]
+
+    def process_task(self, task):
+        """Updates a single row in `pg_dist_node` table, optionally in a transaction.
+
+        The transaction is started if we do a demote of the worker node
+        or before promoting the other worker if there is not transaction
+        in progress. And, the transaction it is committed when the
+        switchover/failover completed.
+
+        This method returns `True` if node was updated (optionally,
+        transaction was committed) as an indicator that
+        the `self._pg_dist_node` cache should be updated.
+
+        The maximum lifetime of the transaction in progress
+        is controlled outside of this method."""
+
+        if task.event == 'after_promote':
+            # The after_promote may happen without previous before_demote and/or
+            # before_promore.  In this case we just call self.update_node() method.
+            # If there is a transaction in progress, it could be that it already did
+            # required changes and we can simply COMMIT.
+            if not self._in_flight or self._in_flight.host != task.host or self._in_flight.port != task.port:
+                self.update_node(task)
+            if self._in_flight:
+                self.query('COMMIT')
+                self._in_flight = None
+            return True
+        else:  # before_demote, before_promote
+            if task.timeout:
+                task.deadline = time.time() + task.timeout
+            if not self._in_flight:
+                self.query('BEGIN')
+            self.update_node(task)
+            self._in_flight = task
+        return False
+
+    def process_tasks(self):
+        while True:
+            if not self._in_flight and not self.load_pg_dist_node():
+                break
+
+            i, task = self.pick_task()
+            if not task:
+                break
+            try:
+                update_cache = self.process_task(task)
+            except Exception as e:
+                logger.error('Exception when working with pg_dist_node: %r', e)
+                update_cache = False
+            with self._condition:
+                if self._tasks:
+                    if update_cache:
+                        self._pg_dist_node[task.group] = task
+                    if id(self._tasks[i]) == id(task):
+                        self._tasks.pop(i)
+            task.wakeup()
+
+    def run(self):
+        while True:
+            try:
+                with self._condition:
+                    if self._schedule_load_pg_dist_node:
+                        timeout = -1
+                    elif self._in_flight:
+                        timeout = self._in_flight.deadline - time.time() if self._tasks else None
+                    else:
+                        timeout = -1 if self._tasks else None
+
+                    if timeout is None or timeout > 0:
+                        self._condition.wait(timeout)
+                    elif self._in_flight:
+                        logger.warning('Rolling back transaction. Last known status: %s', self._in_flight)
+                        self.query('ROLLBACK')
+                        self._in_flight = None
+                self.process_tasks()
+            except Exception:
+                logger.exception('run')
+
+    def _add_task(self, task):
+        with self._condition:
+            i = self.find_task_by_group(task.group)
+
+            # task.timeout is None is an indicator that it was scheduled
+            # from the sync_pg_dist_node() and we don't want to override
+            # already existing task created from REST API.
+            if task.timeout is None and (i is not None or self._in_flight and self._in_flight.group == task.group):
+                return False
+
+            # Override already existing task for the same worker group
+            if i is not None:
+                if task != self._tasks[i]:
+                    logger.debug('Overriding existing task: %s != %s', self._tasks[i], task)
+                    self._tasks[i] = task
+                    self._condition.notify()
+                    return True
+            # Add the task to the list if Worker node state is different from the cached `pg_dist_node`
+            elif self._schedule_load_pg_dist_node or task != self._pg_dist_node.get(task.group)\
+                    or self._in_flight and task.group == self._in_flight.group:
+                logger.debug('Adding the new task: %s', task)
+                self._tasks.append(task)
+                self._condition.notify()
+                return True
+        return False
+
+    def add_task(self, event, group, conn_url, timeout=None, cooldown=None):
+        try:
+            r = urlparse(conn_url)
+        except Exception as e:
+            return logger.error('Failed to parse connection url %s: %r', conn_url, e)
+        host = r.hostname
+        port = r.port or 5432
+        task = PgDistNode(group, host, port, event, timeout=timeout, cooldown=cooldown)
+        return task if self._add_task(task) else None
+
+    def handle_event(self, cluster, event):
+        if not self.is_alive():
+            return
+
+        cluster = cluster.workers.get(event['group'])
+        if not (cluster and cluster.leader and cluster.leader.name == event['leader'] and cluster.leader.conn_url):
+            return
+
+        task = self.add_task(event['type'], event['group'],
+                             cluster.leader.conn_url,
+                             event['timeout'], event['cooldown']*1000)
+        if task and event['type'] == 'before_demote':
+            task.wait()
+
+    def bootstrap(self):
+        if not self.is_enabled():
+            return
+
+        conn_kwargs = self._postgresql.config.local_connect_kwargs
+        conn_kwargs['options'] = '-c synchronous_commit=local -c statement_timeout=0'
+        if self._config['database'] != self._postgresql.database:
+            conn = connect(**conn_kwargs)
+            try:
+                with conn.cursor() as cur:
+                    cur.execute('CREATE DATABASE {0}'.format(quote_ident(self._config['database'], conn)))
+            finally:
+                conn.close()
+
+        conn_kwargs['dbname'] = self._config['database']
+        conn = connect(**conn_kwargs)
+        try:
+            with conn.cursor() as cur:
+                cur.execute('CREATE EXTENSION citus')
+
+                superuser = self._postgresql.config.superuser
+                params = {k: superuser[k] for k in ('password', 'sslcert', 'sslkey') if k in superuser}
+                if params:
+                    cur.execute("INSERT INTO pg_catalog.pg_dist_authinfo VALUES"
+                                "(0, pg_catalog.current_user(), %s)",
+                                (self._postgresql.config.format_dsn(params),))
+
+                if self.is_coordinator():
+                    r = urlparse(self._postgresql.connection_string)
+                    cur.execute("SELECT pg_catalog.citus_set_coordinator_host(%s, %s, 'primary', 'default')",
+                                (r.hostname, r.port or 5432))
+        finally:
+            conn.close()
+
+    def adjust_postgres_gucs(self, parameters):
+        if not self.is_enabled():
+            return
+
+        # citus extension must be on the first place in shared_preload_libraries
+        shared_preload_libraries = list(filter(
+            lambda el: el and el != 'citus',
+            [p.strip() for p in parameters.get('shared_preload_libraries', '').split(',')]))
+        parameters['shared_preload_libraries'] = ','.join(['citus'] + shared_preload_libraries)
+
+        # if not explicitly set Citus overrides max_prepared_transactions to max_connections*2
+        if parameters.get('max_prepared_transactions') == 0:
+            parameters['max_prepared_transactions'] = parameters['max_connections'] * 2
+
+        # Resharding in Citus implemented using logical replication
+        parameters['wal_level'] = 'logical'
+
+    def ignore_replication_slot(self, slot):
+        if self.is_enabled() and self._postgresql.is_leader() and\
+                slot['type'] == 'logical' and slot['database'] == self._config['database']:
+            m = CITUS_SLOT_NAME_RE.match(slot['name'])
+            return m and {'move': 'pgoutput', 'split': 'citus'}.get(m.group(1)) == slot['plugin']
+        return False

patroni/postgresql/config.py

@@ -6,27 +6,20 @@ import socket
 import stat
 import time
 
-from six.moves.urllib_parse import urlparse, parse_qsl, unquote
+from urllib.parse import urlparse, parse_qsl, unquote
 
 from .validator import CaseInsensitiveDict, recovery_parameters,\
         transform_postgresql_parameter_value, transform_recovery_parameter_value
 from ..dcs import slot_name_from_member_name, RemoteMember
 from ..exceptions import PatroniFatalException
-from ..psycopg import quote_ident as _quote_ident
 from ..utils import compare_values, parse_bool, parse_int, split_host_port, uri, \
         validate_directory, is_subpath
 
 logger = logging.getLogger(__name__)
 
-SYNC_STANDBY_NAME_RE = re.compile(r'^[A-Za-z_][A-Za-z_0-9\$]*$')
 PARAMETER_RE = re.compile(r'([a-z_]+)\s*=\s*')
 
 
-def quote_ident(value):
-    """Very simplified version of quote_ident"""
-    return value if SYNC_STANDBY_NAME_RE.match(value) else _quote_ident(value)
-
-
 def conninfo_uri_parse(dsn):
     ret = {}
     r = urlparse(dsn)
@@ -536,24 +529,24 @@ class ConfigHandler(object):
             recovery_params.update({'recovery_target': '', 'recovery_target_name': '', 'recovery_target_time': '',
                                     'recovery_target_xid': '', 'recovery_target_lsn': ''})
 
-        is_remote_master = isinstance(member, RemoteMember)
+        is_remote_member = isinstance(member, RemoteMember)
         primary_conninfo = self.primary_conninfo_params(member)
         if primary_conninfo:
             use_slots = self.get('use_slots', True) and self._postgresql.major_version >= 90400
-            if use_slots and not (is_remote_master and member.no_replication_slot):
-                primary_slot_name = member.primary_slot_name if is_remote_master else self._postgresql.name
+            if use_slots and not (is_remote_member and member.no_replication_slot):
+                primary_slot_name = member.primary_slot_name if is_remote_member else self._postgresql.name
                 recovery_params['primary_slot_name'] = slot_name_from_member_name(primary_slot_name)
                 # We are a standby leader and are using a replication slot. Make sure we connect to
                 # the leader of the main cluster (in case more than one host is specified in the
                 # connstr) by adding 'target_session_attrs=read-write' to primary_conninfo.
-                if is_remote_master and 'target_sesions_attrs' not in primary_conninfo and\
+                if is_remote_member and 'target_sesions_attrs' not in primary_conninfo and\
                         self._postgresql.major_version >= 100000:
                     primary_conninfo['target_session_attrs'] = 'read-write'
             recovery_params['primary_conninfo'] = primary_conninfo
 
         # standby_cluster config might have different parameters, we want to override them
         standby_cluster_params = ['restore_command', 'archive_cleanup_command']\
-            + (['recovery_min_apply_delay'] if is_remote_master else [])
+            + (['recovery_min_apply_delay'] if is_remote_member else [])
         recovery_params.update({p: member.data.get(p) for p in standby_cluster_params if member and member.data.get(p)})
         return recovery_params
 
@@ -775,9 +768,7 @@ class ConfigHandler(object):
             os.chmod(self._pgpass, stat.S_IWRITE | stat.S_IREAD)
             f.write(line)
 
-        env = os.environ.copy()
-        env['PGPASSFILE'] = self._pgpass
-        return env
+        return {**os.environ, 'PGPASSFILE': self._pgpass}
 
     def write_recovery_conf(self, recovery_params):
         self._recovery_params = recovery_params
@@ -869,6 +860,9 @@ class ConfigHandler(object):
         elif self._postgresql.major_version:
             wal_keep_size = parse_int(parameters.pop('wal_keep_size', self.CMDLINE_OPTIONS['wal_keep_size'][0]), 'MB')
             parameters.setdefault('wal_keep_segments', int((wal_keep_size + 8) / 16))
+
+        self._postgresql.citus_handler.adjust_postgres_gucs(parameters)
+
         ret = CaseInsensitiveDict({k: v for k, v in parameters.items() if not self._postgresql.major_version or
                                    self._postgresql.major_version >= self.CMDLINE_OPTIONS.get(k, (0, 1, 90100))[2]})
         ret.update({k: os.path.join(self._config_dir, ret[k]) for k in ('hba_file', 'ident_file') if k in ret})
@@ -1017,6 +1011,9 @@ class ConfigHandler(object):
         if not local_connection_address_changed:
             self.resolve_connection_addresses()
 
+        proxy_addr = config.get('proxy_address')
+        self._postgresql.proxy_url = uri('postgres', proxy_addr, self._postgresql.database) if proxy_addr else None
+
         if conf_changed:
             self.write_postgresql_conf()
 
@@ -1041,23 +1038,19 @@ class ConfigHandler(object):
         else:
             logger.info('No PostgreSQL configuration items changed, nothing to reload.')
 
-    def set_synchronous_standby(self, sync_members):
-        """Sets a node to be synchronous standby and if changed does a reload for PostgreSQL."""
-        if sync_members and sync_members != ['*']:
-            sync_members = [quote_ident(x) for x in sync_members]
-        if self._postgresql.major_version >= 90600 and len(sync_members) > 1:
-            sync_param = '{0} ({1})'.format(len(sync_members), ','.join(sync_members))
-        else:
-            sync_param = next(iter(sync_members), None)
-        if sync_param != self._synchronous_standby_names:
-            if sync_param is None:
+    def set_synchronous_standby_names(self, value):
+        """Updates synchronous_standby_names and reloads if necessary.
+        :returns: True if value was updated."""
+        if value != self._synchronous_standby_names:
+            if value is None:
                 self._server_parameters.pop('synchronous_standby_names', None)
             else:
-                self._server_parameters['synchronous_standby_names'] = sync_param
-            self._synchronous_standby_names = sync_param
+                self._server_parameters['synchronous_standby_names'] = value
+            self._synchronous_standby_names = value
             if self._postgresql.state == 'running':
                 self.write_postgresql_conf()
                 self._postgresql.reload()
+            return True
 
     @property
     def effective_configuration(self):
@@ -1070,7 +1063,7 @@ class ConfigHandler(object):
         As a workaround we will start it with the values from controldata and set `pending_restart`
         to true as an indicator that current values of parameters are not matching expectations."""
 
-        if self._postgresql.role == 'master':
+        if self._postgresql.role in ('master', 'primary'):
             return self._server_parameters
 
         options_mapping = {
@@ -1099,12 +1092,22 @@ class ConfigHandler(object):
                 effective_configuration[name] = cvalue
                 self._postgresql.set_pending_restart(True)
 
-        # If we are using custom bootstrap with PITR it could fail when values
-        # like max_connections are increased, therefore we disable hot_standby.
-        if self._postgresql.bootstrap.running_custom_bootstrap and \
-                (self._postgresql.bootstrap.keep_existing_recovery_conf or self._recovery_conf):
-            effective_configuration['hot_standby'] = 'off'
-            self._postgresql.set_pending_restart(True)
+        # If we are using custom bootstrap with PITR it could fail when values like max_connections
+        # are increased, therefore we disable hot_standby if recovery_target_action == 'promote'.
+        if self._postgresql.bootstrap.running_custom_bootstrap:
+            disable_hot_standby = False
+            if self._postgresql.bootstrap.keep_existing_recovery_conf:
+                disable_hot_standby = True  # trust that pgBackRest does the right thing
+            # `pause_at_recovery_target` has no effect if hot_standby is not enabled, therefore we consider only 9.5+
+            elif self._postgresql.major_version >= 90500 and self._recovery_params:
+                pause_at_recovery_target = parse_bool(self._recovery_params.get('pause_at_recovery_target'))
+                recovery_target_action = self._recovery_params.get(
+                    'recovery_target_action', 'promote' if pause_at_recovery_target is False else 'pause')
+                disable_hot_standby = recovery_target_action == 'promote'
+
+            if disable_hot_standby:
+                effective_configuration['hot_standby'] = 'off'
+                self._postgresql.set_pending_restart(True)
 
         return effective_configuration
 

patroni/postgresql/connection.py

@@ -22,7 +22,6 @@ class Connection(object):
         with self._lock:
             if not self._connection or self._connection.closed != 0:
                 self._connection = psycopg.connect(**self._conn_kwargs)
-                self._connection.autocommit = True
                 self.server_version = self._connection.server_version
         return self._connection
 
@@ -42,7 +41,6 @@ class Connection(object):
 @contextmanager
 def get_connection_cursor(**kwargs):
     conn = psycopg.connect(**kwargs)
-    conn.autocommit = True
     with conn.cursor() as cur:
         yield cur
     conn.close()

patroni/postgresql/misc.py

@@ -1,11 +1,13 @@
+import errno
 import logging
+import os
 
 from patroni.exceptions import PostgresException
 
 logger = logging.getLogger(__name__)
 
 
-def postgres_version_to_int(pg_version):
+def postgres_version_to_int(pg_version: str) -> int:
     """Convert the server_version to integer
 
     >>> postgres_version_to_int('9.5.3')
@@ -43,7 +45,7 @@ def postgres_version_to_int(pg_version):
     return int(''.join('{0:02d}'.format(c) for c in components))
 
 
-def postgres_major_version_to_int(pg_version):
+def postgres_major_version_to_int(pg_version: str) -> int:
     """
     >>> postgres_major_version_to_int('10')
     100000
@@ -73,3 +75,16 @@ def parse_history(data):
 def format_lsn(lsn, full=False):
     template = '{0:X}/{1:08X}' if full else '{0:X}/{1:X}'
     return template.format(lsn >> 32, lsn & 0xFFFFFFFF)
+
+
+def fsync_dir(path):
+    if os.name != 'nt':
+        fd = os.open(path, os.O_DIRECTORY)
+        try:
+            os.fsync(fd)
+        except OSError as e:
+            # Some filesystems don't like fsyncing directories and raise EINVAL. Ignoring it is usually safe.
+            if e.errno != errno.EINVAL:
+                raise
+        finally:
+            os.close(fd)

patroni/postgresql/postmaster.py

@@ -171,8 +171,8 @@ class PostmasterProcess(psutil.Process):
         else:
             return not self.is_running()
 
-    def wait_for_user_backends_to_close(self):
-        # These regexps are cross checked against versions PostgreSQL 9.1 .. 11
+    def wait_for_user_backends_to_close(self, stop_timeout):
+        # These regexps are cross checked against versions PostgreSQL 9.1 .. 15
         aux_proc_re = re.compile("(?:postgres:)( .*:)? (?:(?:archiver|startup|autovacuum launcher|autovacuum worker|"
                                  "checkpointer|logger|stats collector|wal receiver|wal writer|writer)(?: process  )?|"
                                  "walreceiver|wal sender process|walsender|walwriter|background writer|"
@@ -184,19 +184,23 @@ class PostmasterProcess(psutil.Process):
             return logger.debug('Failed to get list of postmaster children')
 
         user_backends = []
-        user_backends_cmdlines = []
+        user_backends_cmdlines = {}
         for child in children:
             try:
                 cmdline = child.cmdline()
                 if cmdline and not aux_proc_re.match(cmdline[0]):
                     user_backends.append(child)
-                    user_backends_cmdlines.append(cmdline[0])
+                    user_backends_cmdlines[child.pid] = cmdline[0]
             except psutil.NoSuchProcess:
                 pass
         if user_backends:
-            logger.debug('Waiting for user backends %s to close', ', '.join(user_backends_cmdlines))
-            psutil.wait_procs(user_backends)
-        logger.debug("Backends closed")
+            logger.debug('Waiting for user backends %s to close', ', '.join(user_backends_cmdlines.values()))
+            gone, live = psutil.wait_procs(user_backends, stop_timeout)
+            if stop_timeout and live:
+                live = [user_backends_cmdlines[b.pid] for b in live]
+                logger.warning('Backends still alive after %s: %s', stop_timeout, ', '.join(live))
+            else:
+                logger.debug("Backends closed")
 
     @staticmethod
     def start(pgcommand, data_dir, conf, options):

patroni/postgresql/rewind.py

@@ -1,13 +1,14 @@
 import logging
 import os
+import re
 import shlex
-import six
+import shutil
 import subprocess
 
 from threading import Lock, Thread
 
 from .connection import get_connection_cursor
-from .misc import format_lsn, parse_history, parse_lsn
+from .misc import format_lsn, fsync_dir, parse_history, parse_lsn
 from ..async_executor import CriticalTask
 from ..dcs import Leader
 
@@ -124,7 +125,7 @@ class Rewind(object):
                 in_recovery = True
                 lsn = data.get('Minimum recovery ending location')
                 timeline = int(data.get("Min recovery ending loc's timeline"))
-                if lsn == '0/0' or timeline == 0:  # it was a master when it crashed
+                if lsn == '0/0' or timeline == 0:  # it was a primary when it crashed
                     data['Database cluster state'] = 'shut down'
             if data.get('Database cluster state') == 'shut down':
                 in_recovery = False
@@ -150,12 +151,12 @@ class Rewind(object):
         else:  # otherwise analyze pg_controldata output
             in_recovery, timeline, lsn = self._get_local_timeline_lsn_from_controldata()
 
-        log_lsn = format_lsn(lsn) if isinstance(lsn, six.integer_types) else lsn
+        log_lsn = format_lsn(lsn) if isinstance(lsn, int) else lsn
         logger.info('Local timeline=%s lsn=%s', timeline, log_lsn)
         return in_recovery, timeline, lsn
 
     @staticmethod
-    def _log_master_history(history, i):
+    def _log_primary_history(history, i):
         start = max(0, i - 3)
         end = None if i + 4 >= len(history) else i + 2
         history_show = []
@@ -170,7 +171,7 @@ class Rewind(object):
             history_show.append('...')
             history_show.append(format_history_line(history[-1]))
 
-        logger.info('master: history=%s', '\n'.join(history_show))
+        logger.info('primary: history=%s', '\n'.join(history_show))
 
     def _conn_kwargs(self, member, auth):
         ret = member.conn_kwargs(auth)
@@ -187,7 +188,7 @@ class Rewind(object):
         if local_timeline is None or local_lsn is None:
             return
 
-        if isinstance(leader, Leader) and leader.member.data.get('role') != 'master':
+        if isinstance(leader, Leader) and leader.member.data.get('role') not in ('master', 'primary'):
             return
 
         # We want to use replication credentials when connecting to the "postgres" database in case if
@@ -204,20 +205,20 @@ class Rewind(object):
         try:
             with self._postgresql.get_replication_connection_cursor(**leader.conn_kwargs()) as cur:
                 cur.execute('IDENTIFY_SYSTEM')
-                master_timeline = cur.fetchone()[1]
-                logger.info('master_timeline=%s', master_timeline)
-                if local_timeline > master_timeline:  # Not always supported by pg_rewind
+                primary_timeline = cur.fetchone()[1]
+                logger.info('primary_timeline=%s', primary_timeline)
+                if local_timeline > primary_timeline:  # Not always supported by pg_rewind
                     need_rewind = True
-                elif local_timeline == master_timeline:
+                elif local_timeline == primary_timeline:
                     need_rewind = False
-                elif master_timeline > 1:
-                    cur.execute('TIMELINE_HISTORY {0}'.format(master_timeline))
+                elif primary_timeline > 1:
+                    cur.execute('TIMELINE_HISTORY {0}'.format(primary_timeline))
                     history = cur.fetchone()[1]
-                    if not isinstance(history, six.string_types):
+                    if not isinstance(history, str):
                         history = bytes(history).decode('utf-8')
-                    logger.debug('master: history=%s', history)
+                    logger.debug('primary: history=%s', history)
         except Exception:
-            return logger.exception('Exception when working with master via replication connection')
+            return logger.exception('Exception when working with primary via replication connection')
 
         if history is not None:
             history = list(parse_history(history))
@@ -238,7 +239,7 @@ class Rewind(object):
                     break
             else:
                 need_rewind = True
-            self._log_master_history(history, i)
+            self._log_primary_history(history, i)
 
         self._state = need_rewind and REWIND_STATUS.NEED or REWIND_STATUS.NOT_NEED
 
@@ -268,7 +269,7 @@ class Rewind(object):
                         if self._checkpoint_task.result is not None:
                             self._state = REWIND_STATUS.CHECKPOINT
                             self._checkpoint_task = None
-                elif self._postgresql.get_master_timeline() == self._postgresql.pg_control_timeline():
+                elif self._postgresql.get_primary_timeline() == self._postgresql.pg_control_timeline():
                     self._state = REWIND_STATUS.CHECKPOINT
                 else:
                     self._checkpoint_task = CriticalTask()
@@ -277,28 +278,37 @@ class Rewind(object):
     def checkpoint_after_promote(self):
         return self._state == REWIND_STATUS.CHECKPOINT
 
-    def _fetch_missing_wal(self, restore_command, wal_filename):
+    def _buid_archiver_command(self, command, wal_filename):
+        """Replace placeholders in the given archiver command's template.
+        Applicable for archive_command and restore_command.
+        Can also be used for archive_cleanup_command and recovery_end_command,
+        however %r value is always set to 000000010000000000000001."""
         cmd = ''
-        length = len(restore_command)
+        length = len(command)
         i = 0
         while i < length:
-            if restore_command[i] == '%' and i + 1 < length:
+            if command[i] == '%' and i + 1 < length:
                 i += 1
-                if restore_command[i] == 'p':
+                if command[i] == 'p':
                     cmd += os.path.join(self._postgresql.wal_dir, wal_filename)
-                elif restore_command[i] == 'f':
+                elif command[i] == 'f':
                     cmd += wal_filename
-                elif restore_command[i] == 'r':
+                elif command[i] == 'r':
                     cmd += '000000010000000000000001'
-                elif restore_command[i] == '%':
+                elif command[i] == '%':
                     cmd += '%'
                 else:
                     cmd += '%'
                     i -= 1
             else:
-                cmd += restore_command[i]
+                cmd += command[i]
             i += 1
 
+        return cmd
+
+    def _fetch_missing_wal(self, restore_command, wal_filename):
+        cmd = self._buid_archiver_command(restore_command, wal_filename)
+
         logger.info('Trying to fetch the missing wal: %s', cmd)
         return self._postgresql.cancellable.call(shlex.split(cmd)) == 0
 
@@ -315,6 +325,54 @@ class Rewind(object):
                     if waldir.endswith('/pg_' + self._postgresql.wal_name) and len(wal_filename) == 24:
                         return wal_filename
 
+    def _archive_ready_wals(self):
+        """Try to archive WALs that have .ready files just in case
+        archive_mode was not set to 'always' before promote, while
+        after it the WALs were recycled on the promoted replica.
+        With this we prevent the entire loss of such WALs and the
+        consequent old leader's start failure."""
+        archive_mode = self._postgresql.get_guc_value('archive_mode')
+        archive_cmd = self._postgresql.get_guc_value('archive_command')
+        if archive_mode not in ('on', 'always') or not archive_cmd:
+            return
+
+        walseg_regex = re.compile(r'^[0-9A-F]{24}(\.partial){0,1}\.ready$')
+        status_dir = os.path.join(self._postgresql.wal_dir, 'archive_status')
+        try:
+            wals_to_archive = [f[:-6] for f in os.listdir(status_dir) if walseg_regex.match(f)]
+        except OSError as e:
+            return logger.error('Unable to list %s: %r', status_dir, e)
+
+        # skip fsync, as postgres --single or pg_rewind will anyway run it
+        for wal in sorted(wals_to_archive):
+            old_name = os.path.join(status_dir, wal + '.ready')
+            # wal file might have alredy been archived
+            if os.path.isfile(old_name) and os.path.isfile(os.path.join(self._postgresql.wal_dir, wal)):
+                cmd = self._buid_archiver_command(archive_cmd, wal)
+                # it is the author of archive_command, who is responsible
+                # for not overriding the WALs already present in archive
+                logger.info('Trying to archive %s: %s', wal, cmd)
+                if self._postgresql.cancellable.call(shlex.split(cmd)) == 0:
+                    new_name = os.path.join(status_dir, wal + '.done')
+                    try:
+                        shutil.move(old_name, new_name)
+                    except Exception as e:
+                        logger.error('Unable to rename %s to %s: %r', old_name, new_name, e)
+                else:
+                    logger.info('Failed to archive WAL segment %s', wal)
+
+    def _maybe_clean_pg_replslot(self):
+        """Clean pg_replslot directory if pg version is less then 11
+        (pg_rewind deletes $PGDATA/pg_replslot content only since pg11)."""
+        if self._postgresql.major_version < 110000:
+            replslot_dir = self._postgresql.slots_handler.pg_replslot_dir
+            try:
+                for f in os.listdir(replslot_dir):
+                    shutil.rmtree(os.path.join(replslot_dir, f))
+                fsync_dir(replslot_dir)
+            except Exception as e:
+                logger.warning('Unable to clean %s: %r', replslot_dir, e)
+
     def pg_rewind(self, r):
         # prepare pg_rewind connection
         env = self._postgresql.config.write_pgpass(r)
@@ -367,15 +425,17 @@ class Rewind(object):
         if self._postgresql.is_running() and not self._postgresql.stop(checkpoint=False):
             return logger.warning('Can not run pg_rewind because postgres is still running')
 
+        self._archive_ready_wals()
+
         # prepare pg_rewind connection
         r = self._conn_kwargs(leader, self._postgresql.config.rewind_credentials)
 
-        # 1. make sure that we are really trying to rewind from the master
+        # 1. make sure that we are really trying to rewind from the primary
         # 2. make sure that pg_control contains the new timeline by:
         #   running a checkpoint or
-        #   waiting until Patroni on the master will expose checkpoint_after_promote=True
+        #   waiting until Patroni on the primary will expose checkpoint_after_promote=True
         checkpoint_status = leader.checkpoint_after_promote if isinstance(leader, Leader) else None
-        if checkpoint_status is None:  # we are the standby-cluster leader or master still runs the old Patroni
+        if checkpoint_status is None:  # we are the standby-cluster leader or primary still runs the old Patroni
             # superuser credentials match rewind_credentials if the latter are not provided or we run 10 or older
             if self._postgresql.config.superuser == self._postgresql.config.rewind_credentials:
                 leader_status = self._postgresql.checkpoint(
@@ -390,14 +450,15 @@ class Rewind(object):
             return
 
         if self.pg_rewind(r):
+            self._maybe_clean_pg_replslot()
             self._state = REWIND_STATUS.SUCCESS
         else:
             if not self.check_leader_is_not_in_recovery(r):
-                logger.warning('Failed to rewind because master %s become unreachable', leader.name)
+                logger.warning('Failed to rewind because primary %s become unreachable', leader.name)
                 if not self.can_rewind:  # It is possible that the previous attempt damaged pg_control file!
                     self._state = REWIND_STATUS.FAILED
             else:
-                logger.error('Failed to rewind from healty master: %s', leader.name)
+                logger.error('Failed to rewind from healty primary: %s', leader.name)
                 self._state = REWIND_STATUS.FAILED
 
             if self.failed:
@@ -465,6 +526,7 @@ class Rewind(object):
             logger.exception('Unable to list %s', status_dir)
 
     def ensure_clean_shutdown(self):
+        self._archive_ready_wals()
         self.cleanup_archive_status()
 
         # Start in a single user mode and stop to produce a clean shutdown

patroni/postgresql/slots.py

@@ -1,13 +1,13 @@
-import errno
 import logging
 import os
 import shutil
 
 from collections import defaultdict
 from contextlib import contextmanager
+from threading import Condition, Thread
 
 from .connection import get_connection_cursor
-from .misc import format_lsn
+from .misc import format_lsn, fsync_dir
 from ..psycopg import OperationalError
 
 logger = logging.getLogger(__name__)
@@ -18,25 +18,97 @@ def compare_slots(s1, s2, dbid='database'):
                                          s1.get(dbid) == s2.get(dbid) and s1['plugin'] == s2['plugin'])
 
 
-def fsync_dir(path):
-    if os.name != 'nt':
-        fd = os.open(path, os.O_DIRECTORY)
+class SlotsAdvanceThread(Thread):
+
+    def __init__(self, slots_handler):
+        super(SlotsAdvanceThread, self).__init__()
+        self.daemon = True
+        self._slots_handler = slots_handler
+
+        # _copy_slots and _failed are used to asynchronously give some feedback to the main thread
+        self._copy_slots = []
+        self._failed = False
+
+        self._scheduled = defaultdict(dict)  # {'dbname1': {'slot1': 100, 'slot2': 100}, 'dbname2': {'slot3': 100}}
+        self._condition = Condition()  # protect self._scheduled from concurrent access and to wakeup the run() method
+
+        self.start()
+
+    def sync_slot(self, cur, database, slot, lsn):
+        failed = copy = False
         try:
-            os.fsync(fd)
-        except OSError as e:
-            # Some filesystems don't like fsyncing directories and raise EINVAL. Ignoring it is usually safe.
-            if e.errno != errno.EINVAL:
-                raise
-        finally:
-            os.close(fd)
+            cur.execute("SELECT pg_catalog.pg_replication_slot_advance(%s, %s)", (slot, format_lsn(lsn)))
+        except Exception as e:
+            logger.error("Failed to advance logical replication slot '%s': %r", slot, e)
+            failed = True
+            copy = isinstance(e, OperationalError) and e.diag.sqlstate == '58P01'  # WAL file is gone
+        with self._condition:
+            if self._scheduled and failed:
+                if copy and slot not in self._copy_slots:
+                    self._copy_slots.append(slot)
+                self._failed = True
+
+            new_lsn = self._scheduled.get(database, {}).get(slot, 0)
+            # remove slot from the self._scheduled structure only if it wasn't changed
+            if new_lsn == lsn and database in self._scheduled:
+                self._scheduled[database].pop(slot)
+                if not self._scheduled[database]:
+                    self._scheduled.pop(database)
+
+    def sync_slots_in_database(self, database, slots):
+        with self._slots_handler.get_local_connection_cursor(dbname=database, options='-c statement_timeout=0') as cur:
+            for slot in slots:
+                with self._condition:
+                    lsn = self._scheduled.get(database, {}).get(slot, 0)
+                if lsn:
+                    self.sync_slot(cur, database, slot, lsn)
+
+    def sync_slots(self):
+        with self._condition:
+            databases = list(self._scheduled.keys())
+        for database in databases:
+            with self._condition:
+                slots = list(self._scheduled.get(database, {}).keys())
+            if slots:
+                try:
+                    self.sync_slots_in_database(database, slots)
+                except Exception as e:
+                    logger.error('Failed to advance replication slots in database %s: %r', database, e)
+
+    def run(self):
+        while True:
+            with self._condition:
+                if not self._scheduled:
+                    self._condition.wait()
+
+            self.sync_slots()
+
+    def schedule(self, advance_slots):
+        with self._condition:
+            for database, values in advance_slots.items():
+                self._scheduled[database].update(values)
+            ret = (self._failed, self._copy_slots)
+            self._copy_slots = []
+            self._failed = False
+            self._condition.notify()
+
+        return ret
+
+    def on_promote(self):
+        with self._condition:
+            self._scheduled.clear()
+            self._failed = False
+            self._copy_slots = []
 
 
 class SlotsHandler(object):
 
     def __init__(self, postgresql):
         self._postgresql = postgresql
+        self._advance = None
         self._replication_slots = {}  # already existing replication slots
         self._unready_logical_slots = {}
+        self.pg_replslot_dir = os.path.join(self._postgresql.data_dir, 'pg_replslot')
         self.schedule()
 
     def _query(self, sql, *params):
@@ -104,26 +176,36 @@ class SlotsHandler(object):
             if ((matcher.get("name") is None or matcher["name"] == name)
                and all(not matcher.get(a) or matcher[a] == slot.get(a) for a in ('database', 'plugin', 'type'))):
                 return True
-        return False
+        return self._postgresql.citus_handler.ignore_replication_slot(slot)
 
     def drop_replication_slot(self, name):
-        cursor = self._query(('SELECT pg_catalog.pg_drop_replication_slot(%s) WHERE EXISTS (SELECT 1 ' +
-                              'FROM pg_catalog.pg_replication_slots WHERE slot_name = %s AND NOT active)'), name, name)
-        # In normal situation rowcount should be 1, otherwise either slot doesn't exists or it is still active
-        return cursor.rowcount == 1
+        """Returns a tuple(active, dropped)"""
+        cursor = self._query(('WITH slots AS (SELECT slot_name, active' +
+                              ' FROM pg_catalog.pg_replication_slots WHERE slot_name = %s),' +
+                              ' dropped AS (SELECT pg_catalog.pg_drop_replication_slot(slot_name),' +
+                              ' true AS dropped FROM slots WHERE not active) ' +
+                              'SELECT active, COALESCE(dropped, false) FROM slots' +
+                              ' FULL OUTER JOIN dropped ON true'), name)
+        return cursor.fetchone() if cursor.rowcount == 1 else (False, False)
 
-    def _drop_incorrect_slots(self, cluster, slots):
+    def _drop_incorrect_slots(self, cluster, slots, paused):
         # drop old replication slots which are not presented in desired slots
         for name in set(self._replication_slots) - set(slots):
-            if not self.ignore_replication_slot(cluster, name) and not self.drop_replication_slot(name):
-                logger.error("Failed to drop replication slot '%s'", name)
-                self._schedule_load_slots = True
-
+            if not paused and not self.ignore_replication_slot(cluster, name):
+                active, dropped = self.drop_replication_slot(name)
+                if dropped:
+                    logger.info("Dropped unknown replication slot '%s'", name)
+                else:
+                    self._schedule_load_slots = True
+                    if active:
+                        logger.debug("Unable to drop unknown replication slot '%s', slot is still active", name)
+                    else:
+                        logger.error("Failed to drop replication slot '%s'", name)
         for name, value in slots.items():
             if name in self._replication_slots and not compare_slots(value, self._replication_slots[name]):
                 logger.info("Trying to drop replication slot '%s' because value is changing from %s to %s",
                             name, self._replication_slots[name], value)
-                if self.drop_replication_slot(name):
+                if self.drop_replication_slot(name) == (False, True):
                     self._replication_slots.pop(name)
                 else:
                     logger.error("Failed to drop replication slot '%s'", name)
@@ -143,7 +225,7 @@ class SlotsHandler(object):
                 self._schedule_load_slots = True
 
     @contextmanager
-    def _get_local_connection_cursor(self, **kwargs):
+    def get_local_connection_cursor(self, **kwargs):
         conn_kwargs = self._postgresql.config.local_connect_kwargs
         conn_kwargs.update(kwargs)
         with get_connection_cursor(**conn_kwargs) as cur:
@@ -162,7 +244,7 @@ class SlotsHandler(object):
 
         # Create new logical slots
         for database, values in logical_slots.items():
-            with self._get_local_connection_cursor(dbname=database) as cur:
+            with self.get_local_connection_cursor(dbname=database) as cur:
                 for name, value in values.items():
                     try:
                         cur.execute("SELECT pg_catalog.pg_create_logical_replication_slot(%s, %s)" +
@@ -175,6 +257,11 @@ class SlotsHandler(object):
                         slots.pop(name)
                     self._schedule_load_slots = True
 
+    def schedule_advance_slots(self, slots):
+        if not self._advance:
+            self._advance = SlotsAdvanceThread(self)
+        return self._advance.schedule(slots)
+
     def _ensure_logical_slots_replica(self, cluster, slots):
         advance_slots = defaultdict(dict)  # Group logical slots to be advanced by database name
         create_slots = []  # And collect logical slots to be created on the replica
@@ -186,27 +273,18 @@ class SlotsHandler(object):
                     if name in cluster.slots:
                         try:  # Skip slots that doesn't need to be advanced
                             if value['confirmed_flush_lsn'] < int(cluster.slots[name]):
-                                advance_slots[value['database']][name] = value
+                                advance_slots[value['database']][name] = int(cluster.slots[name])
                         except Exception as e:
                             logger.error('Failed to parse "%s": %r', cluster.slots[name], e)
                 elif name in cluster.slots:  # We want to copy only slots with feedback in a DCS
                     create_slots.append(name)
 
-        # Advance logical slots
-        for database, values in advance_slots.items():
-            with self._get_local_connection_cursor(dbname=database, options='-c statement_timeout=0') as cur:
-                for name, value in values.items():
-                    try:
-                        cur.execute("SELECT pg_catalog.pg_replication_slot_advance(%s, %s)",
-                                    (name, format_lsn(int(cluster.slots[name]))))
-                    except Exception as e:
-                        logger.error("Failed to advance logical replication slot '%s': %r", name, e)
-                        if isinstance(e, OperationalError) and e.diag.sqlstate == '58P01':  # WAL file is gone
-                            create_slots.append(name)
-                        self._schedule_load_slots = True
-        return create_slots
+        error, copy_slots = self.schedule_advance_slots(advance_slots)
+        if error:
+            self._schedule_load_slots = True
+        return create_slots + copy_slots
 
-    def sync_replication_slots(self, cluster, nofailover, replicatefrom=None):
+    def sync_replication_slots(self, cluster, nofailover, replicatefrom=None, paused=False):
         ret = None
         if self._postgresql.major_version >= 90400 and cluster.config:
             try:
@@ -215,7 +293,7 @@ class SlotsHandler(object):
                 slots = cluster.get_replication_slots(self._postgresql.name, self._postgresql.role,
                                                       nofailover, self._postgresql.major_version, True)
 
-                self._drop_incorrect_slots(cluster, slots)
+                self._drop_incorrect_slots(cluster, slots, paused)
 
                 self._ensure_physical_slots(slots)
 
@@ -264,10 +342,11 @@ class SlotsHandler(object):
                 try:
                     cur = self._query("SELECT pg_catalog.current_setting('hot_standby_feedback')::boolean")
                     if not cur.fetchone()[0]:
-                        return logger.error('Logical slot failover requires "hot_standby_feedback".'
-                                            ' Please check postgresql.auto.conf')
+                        logger.error('Logical slot failover requires "hot_standby_feedback".'
+                                     ' Please check postgresql.auto.conf')
                 except Exception as e:
-                    return logger.error('Failed to check the hot_standby_feedback setting: %r', e)
+                    logger.error('Failed to check the hot_standby_feedback setting: %r', e)
+                return  # since `catalog_xmin` isn't valid further checks don't make any sense
 
         for name in list(self._unready_logical_slots):
             value = self._replication_slots.get(name)
@@ -305,9 +384,8 @@ class SlotsHandler(object):
                 logger.error("Failed to copy logical slots from the %s via postgresql connection: %r", leader.name, e)
 
         if isinstance(create_slots, dict) and create_slots and self._postgresql.stop():
-            pg_replslot_dir = os.path.join(self._postgresql.data_dir, 'pg_replslot')
             for name, value in create_slots.items():
-                slot_dir = os.path.join(pg_replslot_dir, name)
+                slot_dir = os.path.join(self._postgresql.slots_handler.pg_replslot_dir, name)
                 slot_tmp_dir = slot_dir + '.tmp'
                 if os.path.exists(slot_tmp_dir):
                     shutil.rmtree(slot_tmp_dir)
@@ -322,7 +400,7 @@ class SlotsHandler(object):
                 os.rename(slot_tmp_dir, slot_dir)
                 fsync_dir(slot_dir)
                 self._unready_logical_slots[name] = None
-            fsync_dir(pg_replslot_dir)
+            fsync_dir(self._postgresql.slots_handler.pg_replslot_dir)
             self._postgresql.start()
 
     def schedule(self, value=None):
@@ -331,6 +409,9 @@ class SlotsHandler(object):
         self._schedule_load_slots = self._force_readiness_check = value
 
     def on_promote(self):
+        if self._advance:
+            self._advance.on_promote()
+
         if self._unready_logical_slots:
             logger.warning('Logical replication slots that might be unsafe to use after promote: %s',
                            set(self._unready_logical_slots))

patroni/postgresql/sync.py

@@ -0,0 +1,258 @@
+import logging
+import re
+import time
+
+from copy import deepcopy
+
+from .validator import CaseInsensitiveDict
+from ..psycopg import quote_ident as _quote_ident
+
+logger = logging.getLogger(__name__)
+
+SYNC_STANDBY_NAME_RE = re.compile(r'^[A-Za-z_][A-Za-z_0-9\$]*$')
+SYNC_REP_PARSER_RE = re.compile(r"""
+           (?P<first> [fF][iI][rR][sS][tT] )
+         | (?P<any> [aA][nN][yY] )
+         | (?P<space> \s+ )
+         | (?P<ident> [A-Za-z_][A-Za-z_0-9\$]* )
+         | (?P<dquot> " (?: [^"]+ | "" )* " )
+         | (?P<star> [*] )
+         | (?P<num> \d+ )
+         | (?P<comma> , )
+         | (?P<parenstart> \( )
+         | (?P<parenend> \) )
+         | (?P<JUNK> . )
+        """, re.X)
+_EMPTY_SSN = {'type': 'off', 'num': 0, 'members': CaseInsensitiveDict({})}
+
+
+def quote_ident(value):
+    """Very simplified version of quote_ident"""
+    return value if SYNC_STANDBY_NAME_RE.match(value) else _quote_ident(value)
+
+
+def parse_sync_standby_names(value):
+    """Parse postgresql synchronous_standby_names to constituent parts.
+    Returns dict with the following keys:
+    * type: 'quorum'|'priority'
+    * num: int
+    * members: CaseInsensitiveDict, with names as keys
+    * has_star: bool - Present if true
+    If the configuration value can not be parsed, raises a ValueError.
+
+    >>> parse_sync_standby_names('')['type']
+    'off'
+
+    >>> parse_sync_standby_names('FiRsT')['type']
+    'priority'
+
+    >>> parse_sync_standby_names('FiRsT')['members']
+    {'FiRsT': True}
+
+    >>> parse_sync_standby_names('"1"')['members']
+    {'1': True}
+
+    >>> parse_sync_standby_names(' a , b ')['members']
+    {'a': True, 'b': True}
+
+    >>> parse_sync_standby_names(' a , b ')['num']
+    1
+
+    >>> parse_sync_standby_names('ANY 4("a",*,b)')['has_star']
+    True
+
+    >>> parse_sync_standby_names('ANY 4("a",*,b)')['num']
+    4
+
+    >>> parse_sync_standby_names('1')  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValueError: Unparseable synchronous_standby_names value
+
+    >>> parse_sync_standby_names('a,')  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValueError: Unparseable synchronous_standby_names value
+
+    >>> parse_sync_standby_names('ANY 4("a" b,"c c")')  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValueError: Unparseable synchronous_standby_names value
+
+    >>> parse_sync_standby_names('FIRST 4("a",)')  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValueError: Unparseable synchronous_standby_names value
+
+    >>> parse_sync_standby_names('2 (,)')  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValueError: Unparseable synchronous_standby_names value
+    """
+    tokens = [(m.lastgroup, m.group(0), m.start())
+              for m in SYNC_REP_PARSER_RE.finditer(value)
+              if m.lastgroup != 'space']
+    if not tokens:
+        return deepcopy(_EMPTY_SSN)
+
+    if [t[0] for t in tokens[0:3]] == ['any', 'num', 'parenstart'] and tokens[-1][0] == 'parenend':
+        result = {'type': 'quorum', 'num': int(tokens[1][1])}
+        synclist = tokens[3:-1]
+    elif [t[0] for t in tokens[0:3]] == ['first', 'num', 'parenstart'] and tokens[-1][0] == 'parenend':
+        result = {'type': 'priority', 'num': int(tokens[1][1])}
+        synclist = tokens[3:-1]
+    elif [t[0] for t in tokens[0:2]] == ['num', 'parenstart'] and tokens[-1][0] == 'parenend':
+        result = {'type': 'priority', 'num': int(tokens[0][1])}
+        synclist = tokens[2:-1]
+    else:
+        result = {'type': 'priority', 'num': 1}
+        synclist = tokens
+    result['members'] = CaseInsensitiveDict({})
+    for i, (a_type, a_value, a_pos) in enumerate(synclist):
+        if i % 2 == 1:  # odd elements are supposed to be commas
+            if len(synclist) == i + 1:  # except the last token
+                raise ValueError("Unparseable synchronous_standby_names value %r: Unexpected token %s %r at %d" %
+                                 (value, a_type, a_value, a_pos))
+            elif a_type != 'comma':
+                raise ValueError("Unparseable synchronous_standby_names value %r: ""Got token %s %r while"
+                                 " expecting comma at %d" % (value, a_type, a_value, a_pos))
+        elif a_type in {'ident', 'first', 'any'}:
+            result['members'][a_value] = True
+        elif a_type == 'star':
+            result['members'][a_value] = True
+            result['has_star'] = True
+        elif a_type == 'dquot':
+            result['members'][a_value[1:-1].replace('""', '"')] = True
+        else:
+            raise ValueError("Unparseable synchronous_standby_names value %r: Unexpected token %s %r at %d" %
+                             (value, a_type, a_value, a_pos))
+    return result
+
+
+class SyncHandler(object):
+    """Class responsible for working with the `synchronous_standby_names`.
+
+    Sync standbys are chosen based on their state in `pg_stat_replication`.
+    When `synchronous_standby_names` is changed we memorize the `_primary_flush_lsn`
+    and the `current_state()` method will count newly added names as "sync" only when
+    they reached memorized LSN and also reported as "sync" by `pg_stat_replication`"""
+
+    def __init__(self, postgresql):
+        self._postgresql = postgresql
+        self._synchronous_standby_names = ''  # last known value of synchronous_standby_names
+        self._ssn_data = deepcopy(_EMPTY_SSN)
+        self._primary_flush_lsn = 0
+        # "sync" replication connections, that were verified to reach self._primary_flush_lsn at some point
+        self._ready_replicas = CaseInsensitiveDict({})  # keys: member names, values: connection pids
+
+    def _handle_synchronous_standby_names_change(self):
+        """If synchronous_standby_names has changed we need to check that newly added replicas
+        have reached self._primary_flush_lsn. Only after that they could be counted as sync."""
+        synchronous_standby_names = self._postgresql.synchronous_standby_names()
+        if synchronous_standby_names == self._synchronous_standby_names:
+            return False
+
+        self._synchronous_standby_names = synchronous_standby_names
+        try:
+            self._ssn_data = parse_sync_standby_names(synchronous_standby_names)
+        except ValueError as e:
+            logger.warning('%s', e)
+            self._ssn_data = deepcopy(_EMPTY_SSN)
+
+        # Invalidate cache of "sync" connections
+        for app_name in list(self._ready_replicas.keys()):
+            if app_name not in self._ssn_data['members']:
+                del self._ready_replicas[app_name]
+
+        # Newly connected replicas will be counted as sync only when reached self._primary_flush_lsn
+        self._primary_flush_lsn = self._postgresql.last_operation()
+        self._postgresql.query('SELECT pg_catalog.txid_current()')  # Ensure some WAL traffic to move replication
+        self._postgresql.reset_cluster_info_state(None)  # Reset internal cache to query fresh values
+
+    def current_state(self, cluster, sync_node_count=1, sync_node_maxlag=-1):
+        """Finds best candidates to be the synchronous standbys.
+
+        Current synchronous standby is always preferred, unless it has disconnected or does not want to be a
+        synchronous standby any longer.
+        Parameter sync_node_maxlag(maximum_lag_on_syncnode) would help swapping unhealthy sync replica in case
+        if it stops responding (or hung). Please set the value high enough so it won't unncessarily swap sync
+        standbys during high loads. Any less or equal of 0 value keep the behavior backward compatible and
+        will not swap. Please note that it will not also swap sync standbys in case where all replicas are hung.
+
+        :returns: tuple of candidates list and synchronous standby list."""
+
+        self._handle_synchronous_standby_names_change()
+
+        # Pick candidates based on who has higher replay/remote_write/flush lsn.
+        sort_col = {
+            'remote_apply': 'replay',
+            'remote_write': 'write'
+        }.get(self._postgresql.synchronous_commit(), 'flush') + '_lsn'
+
+        pg_stat_replication = [(r['pid'], r['application_name'], r['sync_state'], r[sort_col])
+                               for r in self._postgresql.pg_stat_replication()
+                               if r[sort_col] is not None]
+
+        members = CaseInsensitiveDict({m.name: m for m in cluster.members})
+        replica_list = []
+        # pg_stat_replication.sync_state has 4 possible states - async, potential, quorum, sync.
+        # That is, alphabetically they are in the reversed order of priority.
+        # Since we are doing reversed sort on (sync_state, lsn) tuples, it helps to keep the result
+        # consistent in case if a synchronous standby member is slowed down OR async node receiving
+        # changes faster than the sync member (very rare but possible).
+        # Such cases would trigger sync standby member swapping, but only if lag on a sync node exceeding a threshold.
+        for pid, app_name, sync_state, replica_lsn in sorted(pg_stat_replication, key=lambda r: r[2:4], reverse=True):
+            member = members.get(app_name)
+            if member and member.is_running and not member.tags.get('nosync', False):
+                replica_list.append((pid, member.name, sync_state, replica_lsn, bool(member.nofailover)))
+
+        max_lsn = max(replica_list, key=lambda x: x[3])[3]\
+            if len(replica_list) > 1 else self._postgresql.last_operation()
+
+        if self._postgresql.major_version < 90600:
+            sync_node_count = 1
+
+        candidates = []
+        sync_nodes = []
+        # Prefer members without nofailover tag. We are relying on the fact that sorts are guaranteed to be stable.
+        for pid, app_name, sync_state, replica_lsn, _ in sorted(replica_list, key=lambda x: x[4]):
+            # if standby name is listed in the /sync key we can count it as synchronous, otherwice
+            # ig becomes really synchronous when sync_state = 'sync' and it is known that it managed to catch up
+            if app_name not in self._ready_replicas and app_name in self._ssn_data['members'] and\
+                    (cluster.sync and app_name in cluster.sync.members or
+                     sync_state == 'sync' and replica_lsn >= self._primary_flush_lsn):
+                self._ready_replicas[app_name] = pid
+
+            if sync_node_maxlag <= 0 or max_lsn - replica_lsn <= sync_node_maxlag:
+                candidates.append(app_name)
+                if sync_state == 'sync' and app_name in self._ready_replicas:
+                    sync_nodes.append(app_name)
+            if len(candidates) >= sync_node_count:
+                break
+
+        return candidates, sync_nodes
+
+    def set_synchronous_standby_names(self, value):
+        """Constructs and sets `synchronous_standby_names` value.
+
+        :param value: list[str] - the list of wanted sync members"""
+        if value and value != ['*']:
+            value = [quote_ident(x) for x in value]
+
+        if self._postgresql.major_version >= 90600 and len(value) > 1:
+            sync_param = '{0} ({1})'.format(len(value), ','.join(value))
+        else:
+            sync_param = next(iter(value), None)
+
+        if not (self._postgresql.config.set_synchronous_standby_names(sync_param) and
+                self._postgresql.state == 'running' and self._postgresql.is_leader()) or value == ['*']:
+            return
+
+        time.sleep(0.1)  # Usualy it takes 1ms to reload postgresql.conf, but we will give it 100ms
+
+        # Reset internal cache to query fresh values
+        self._postgresql.reset_cluster_info_state(None)
+
+        # timeline == 0 -- indicates that this is the replica, shoudn't ever happen
+        if self._postgresql.get_primary_timeline() > 0:
+            self._handle_synchronous_standby_names_change()

patroni/postgresql/validator.py

@@ -1,6 +1,5 @@
 import abc
 import logging
-import six
 
 from collections import namedtuple
 from urllib3.response import HTTPHeaderDict
@@ -34,8 +33,7 @@ class Bool(namedtuple('Bool', 'version_from,version_till')):
         logger.warning('Removing bool parameter=%s from the config due to the invalid value=%s', name, value)
 
 
-@six.add_metaclass(abc.ABCMeta)
-class Number(namedtuple('Number', 'version_from,version_till,min_val,max_val,unit')):
+class Number(abc.ABC, namedtuple('Number', 'version_from,version_till,min_val,max_val,unit')):
 
     @staticmethod
     @abc.abstractmethod
@@ -200,7 +198,6 @@ parameters = CaseInsensitiveDict({
     'enable_async_append': Bool(140000, None),
     'enable_bitmapscan': Bool(90300, None),
     'enable_gathermerge': Bool(100000, None),
-    'enable_group_by_reordering': Bool(150000, None),
     'enable_hashagg': Bool(90300, None),
     'enable_hashjoin': Bool(90300, None),
     'enable_incremental_sort': Bool(130000, None),

patroni/psycopg.py

@@ -6,7 +6,7 @@ try:
     from . import MIN_PSYCOPG2, parse_version
     if parse_version(__version__) < MIN_PSYCOPG2:
         raise ImportError
-    from psycopg2 import connect, Error, DatabaseError, OperationalError, ProgrammingError
+    from psycopg2 import connect as _connect, Error, DatabaseError, OperationalError, ProgrammingError
     from psycopg2.extensions import adapt
 
     try:
@@ -20,10 +20,10 @@ try:
             value.prepare(conn)
         return value.getquoted().decode('utf-8')
 except ImportError:
-    from psycopg import connect as _connect, sql, Error, DatabaseError, OperationalError, ProgrammingError
+    from psycopg import connect as __connect, sql, Error, DatabaseError, OperationalError, ProgrammingError
 
-    def connect(*args, **kwargs):
-        ret = _connect(*args, **kwargs)
+    def _connect(*args, **kwargs):
+        ret = __connect(*args, **kwargs)
         ret.server_version = ret.pgconn.server_version  # compatibility with psycopg2
         return ret
 
@@ -34,6 +34,16 @@ except ImportError:
         return sql.Literal(value).as_string(conn)
 
 
+def connect(*args, **kwargs):
+    if kwargs and 'replication' not in kwargs and kwargs.get('fallback_application_name') != 'Patroni ctl':
+        options = [kwargs['options']] if 'options' in kwargs else []
+        options.append('-c search_path=pg_catalog')
+        kwargs['options'] = ' '.join(options)
+    ret = _connect(*args, **kwargs)
+    ret.autocommit = True
+    return ret
+
+
 def quote_ident(value, conn=None):
     if _legacy or conn is None:
         return '"{0}"'.format(value.replace('"', '""'))

patroni/request.py

@@ -1,17 +1,16 @@
 import json
 import urllib3
-import six
 
-from six.moves.urllib_parse import urlparse, urlunparse
+from urllib.parse import urlparse, urlunparse
 
 from .utils import USER_AGENT
 
 
 class PatroniRequest(object):
 
-    def __init__(self, config, insecure=False):
-        cert_reqs = 'CERT_NONE' if insecure or config.get('ctl', {}).get('insecure', False) else 'CERT_REQUIRED'
-        self._pool = urllib3.PoolManager(num_pools=10, maxsize=10, cert_reqs=cert_reqs)
+    def __init__(self, config, insecure=None):
+        self._insecure = insecure
+        self._pool = urllib3.PoolManager(num_pools=10, maxsize=10)
         self.reload_config(config)
 
     @staticmethod
@@ -32,19 +31,26 @@ class PatroniRequest(object):
     def reload_config(self, config):
         self._pool.headers = urllib3.make_headers(basic_auth=self._get_cfg_value(config, 'auth'), user_agent=USER_AGENT)
 
+        insecure = self._insecure if isinstance(self._insecure, bool) else config.get('ctl', {}).get('insecure', False)
         if self._apply_ssl_file_param(config, 'cert'):
+            #  With client certificate the cert_reqs must be set to CERT_REQUIRED even if insecure option is used
+            self._pool.connection_pool_kw['cert_reqs'] = 'CERT_REQUIRED'
+            #  The assert_hostname = False helps to silence warnings
+            self._pool.connection_pool_kw['assert_hostname'] = False if insecure else None
+
             self._apply_ssl_file_param(config, 'key')
 
             password = self._get_cfg_value(config, 'keyfile_password')
             self._apply_pool_param('key_password', password)
         else:
+            self._pool.connection_pool_kw['cert_reqs'] = 'CERT_NONE' if insecure else 'CERT_REQUIRED'
             self._pool.connection_pool_kw.pop('key_file', None)
 
         cacert = config.get('ctl', {}).get('cacert') or config.get('restapi', {}).get('cafile')
         self._apply_pool_param('ca_certs', cacert)
 
     def request(self, method, url, body=None, **kwargs):
-        if body is not None and not isinstance(body, six.string_types):
+        if body is not None and not isinstance(body, str):
             body = json.dumps(body)
         return self._pool.request(method.upper(), url, body=body, **kwargs)
 

patroni/scripts/aws.py

@@ -6,9 +6,9 @@ import sys
 import boto3
 
 from ..utils import Retry, RetryFailedError
-from ..request import get as requests_get
 
 from botocore.exceptions import ClientError
+from botocore.utils import IMDSFetcher
 
 logger = logging.getLogger(__name__)
 
@@ -21,14 +21,16 @@ class AWSConnection(object):
         self._retry = Retry(deadline=300, max_delay=30, max_tries=-1, retry_exceptions=(ClientError,))
         try:
             # get the instance id
-            r = requests_get('http://169.254.169.254/latest/dynamic/instance-identity/document', timeout=2.1)
+            fetcher = IMDSFetcher(timeout=2.1)
+            token = fetcher._fetch_metadata_token()
+            r = fetcher._get_request("/latest/dynamic/instance-identity/document", None, token)
         except Exception:
             logger.error('cannot query AWS meta-data')
             return
 
-        if r.status < 400:
+        if r.status_code < 400:
             try:
-                content = json.loads(r.data.decode('utf-8'))
+                content = json.loads(r.text)
                 self.instance_id = content['instanceId']
                 self.region = content['region']
             except Exception:

patroni/scripts/wale_restore.py

@@ -11,7 +11,7 @@
 # arguments are:
 #   - cluster scope
 #   - cluster role
-#   - master connection string
+#   - leader connection string
 #   - number of retries
 #   - envdir for the WALE env
 # - WALE_BACKUP_THRESHOLD_MEGABYTES if WAL amount is above that - use pg_basebackup
@@ -104,11 +104,11 @@ WALEConfig = namedtuple(
 
 class WALERestore(object):
     def __init__(self, scope, datadir, connstring, env_dir, threshold_mb,
-                 threshold_pct, use_iam, no_master, retries):
+                 threshold_pct, use_iam, no_leader, retries):
         self.scope = scope
-        self.master_connection = connstring
+        self.leader_connection = connstring
         self.data_dir = datadir
-        self.no_master = no_master
+        self.no_leader = no_leader
 
         wale_cmd = [
             'envdir',
@@ -213,41 +213,44 @@ class WALERestore(object):
         diff_in_bytes = backup_size
         attempts_no = 0
         while True:
-            if self.master_connection:
+            if self.leader_connection:
+                con = None
                 try:
                     # get the difference in bytes between the current WAL location and the backup start offset
-                    with psycopg.connect(self.master_connection) as con:
-                        if con.server_version >= 100000:
-                            wal_name = 'wal'
-                            lsn_name = 'lsn'
-                        else:
-                            wal_name = 'xlog'
-                            lsn_name = 'location'
-                        con.autocommit = True
-                        with con.cursor() as cur:
-                            cur.execute(("SELECT CASE WHEN pg_catalog.pg_is_in_recovery()"
-                                         " THEN GREATEST(pg_catalog.pg_{0}_{1}_diff(COALESCE("
-                                         "pg_last_{0}_receive_{1}(), '0/0'), %s)::bigint, "
-                                         "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), %s)::bigint)"
-                                         " ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), %s)::bigint"
-                                         " END").format(wal_name, lsn_name),
-                                        (backup_start_lsn, backup_start_lsn, backup_start_lsn))
+                    con = psycopg.connect(self.leader_connection)
+                    if con.server_version >= 100000:
+                        wal_name = 'wal'
+                        lsn_name = 'lsn'
+                    else:
+                        wal_name = 'xlog'
+                        lsn_name = 'location'
+                    with con.cursor() as cur:
+                        cur.execute(("SELECT CASE WHEN pg_catalog.pg_is_in_recovery()"
+                                     " THEN GREATEST(pg_catalog.pg_{0}_{1}_diff(COALESCE("
+                                     "pg_last_{0}_receive_{1}(), '0/0'), %s)::bigint, "
+                                     "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), %s)::bigint)"
+                                     " ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), %s)::bigint"
+                                     " END").format(wal_name, lsn_name),
+                                    (backup_start_lsn, backup_start_lsn, backup_start_lsn))
 
-                            diff_in_bytes = int(cur.fetchone()[0])
+                        diff_in_bytes = int(cur.fetchone()[0])
                 except psycopg.Error:
-                    logger.exception('could not determine difference with the master location')
+                    logger.exception('could not determine difference with the leader location')
                     if attempts_no < self.retries:  # retry in case of a temporarily connection issue
                         attempts_no = attempts_no + 1
                         time.sleep(RETRY_SLEEP_INTERVAL)
                         continue
                     else:
-                        if not self.no_master:
+                        if not self.no_leader:
                             return False  # do no more retries on the outer level
-                        logger.info("continue with base backup from S3 since master is not available")
+                        logger.info("continue with base backup from S3 since leader is not available")
                         diff_in_bytes = 0
                         break
+                finally:
+                    if con:
+                        con.close()
             else:
-                # always try to use WAL-E if master connection string is not available
+                # always try to use WAL-E if leader connection string is not available
                 diff_in_bytes = 0
             break
 
@@ -343,22 +346,22 @@ def main():
     parser.add_argument('--threshold_megabytes', type=int, default=10240)
     parser.add_argument('--threshold_backup_size_percentage', type=int, default=30)
     parser.add_argument('--use_iam', type=int, default=0)
-    parser.add_argument('--no_master', type=int, default=0)
+    parser.add_argument('--no_leader', '--no_master', type=int, default=0)
     args = parser.parse_args()
 
     exit_code = None
     assert args.retries >= 0
 
-    # Retry cloning in a loop. We do separate retries for the master
+    # Retry cloning in a loop. We do separate retries for the leader
     # connection attempt inside should_use_s3_to_create_replica,
     # because we need to differentiate between the last attempt and
     # the rest and make a decision when the last attempt fails on
-    # whether to use WAL-E or not depending on the no_master flag.
+    # whether to use WAL-E or not depending on the no_leader flag.
     for _ in range(0, args.retries + 1):
         restore = WALERestore(scope=args.scope, datadir=args.datadir, connstring=args.connstring,
                               env_dir=args.envdir, threshold_mb=args.threshold_megabytes,
                               threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam,
-                              no_master=args.no_master, retries=args.retries)
+                              no_leader=args.no_leader, retries=args.retries)
         exit_code = restore.run()
         if not exit_code == ExitCode.RETRY_LATER:  # only WAL-E failures lead to the retry
             logger.debug('exit_code is %r, not retrying', exit_code)

patroni/utils.py

@@ -514,21 +514,3 @@ def enable_keepalive(sock, timeout, idle, cnt=3):
 
     for opt in keepalive_socket_options(timeout, idle, cnt):
         sock.setsockopt(*opt)
-
-
-def find_executable(executable, path=None):
-    _, ext = os.path.splitext(executable)
-
-    if (sys.platform == 'win32') and (ext != '.exe'):
-        executable = executable + '.exe'
-
-    if os.path.isfile(executable):
-        return executable
-
-    if path is None:
-        path = os.environ.get('PATH', os.defpath)
-
-    for p in path.split(os.pathsep):
-        f = os.path.join(p, executable)
-        if os.path.isfile(f):
-            return f