Release 3.0.0 (#2545)

* bump version
* update release notes
* removed 2.7, 3.4, 3.5, and 3.6 from supported versions in setup.py
* switched GH actions back to ubuntu-latest, removed tests with 2.7 and 3.6, and added 3.11
* some little fixes in Citus documentation and behave tests

.github/ISSUE_TEMPLATE/bug_report.md

@@ -20,9 +20,9 @@ A clear and concise description of what you expected to happen.
 If applicable, add screenshots to help explain your problem.
 
 **Environment**
-- Patroni version:
-- PostgreSQL version:
-- DCS (and its version):
+-   Patroni version:
+-   PostgreSQL version:
+-   DCS (and its version):
 
 **Patroni configuration file**
 ```

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

.github/workflows/install_deps.py

@@ -5,7 +5,6 @@ import subprocess
 import stat
 import sys
 import tarfile
-import time
 import zipfile
 
 
@@ -20,7 +19,7 @@ def install_requirements(what):
     requirements = ['mock>=2.0.0', 'flake8', 'pytest', 'pytest-cov'] if what == 'all' else ['behave']
     requirements += ['coverage']
     # try to split tests between psycopg2 and psycopg3
-    requirements += ['psycopg[binary]'] if sys.version_info >= (3, 6, 0) and\
+    requirements += ['psycopg[binary]'] if sys.version_info > (3, 7, 0) and\
         (sys.platform != 'darwin' or what == 'etcd3') else ['psycopg2-binary']
     for r in read('requirements.txt').split('\n'):
         r = r.strip()
@@ -30,6 +29,7 @@ def install_requirements(what):
                 requirements.append(r)
 
     subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'pip'])
+    subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'wheel'])
     r = subprocess.call([sys.executable, '-m', 'pip', 'install'] + requirements)
     s = subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'setuptools'])
     return s | r
@@ -45,10 +45,8 @@ def install_packages(what):
     packages['exhibitor'] = packages['zookeeper']
     packages = packages.get(what, [])
     ver = versions.get(what)
-    subprocess.call(['sudo', 'sed', '-i', 's/pgdg main.*$/pgdg main {0}/'.format(ver),
-                     '/etc/apt/sources.list.d/pgdg.list'])
     subprocess.call(['sudo', 'apt-get', 'update', '-y'])
-    return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev', 'wget'] + packages)
+    return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev'] + packages)
 
 
 def get_file(url, name):
@@ -98,7 +96,7 @@ def unpack(archive, name):
 
 
 def install_etcd():
-    version = os.environ.get('ETCDVERSION', '3.3.13')
+    version = os.environ.get('ETCDVERSION', '3.4.23')
     platform = {'linux2': 'linux', 'win32': 'windows', 'cygwin': 'windows'}.get(sys.platform, sys.platform)
     dirname = 'etcd-v{0}-{1}-amd64'.format(version, platform)
     ext = 'tar.gz' if platform == 'linux' else 'zip'
@@ -110,59 +108,17 @@ def install_etcd():
 
 
 def install_postgres():
-    version = os.environ.get('PGVERSION', '14.1-1')
+    version = os.environ.get('PGVERSION', '15.1-1')
     platform = {'darwin': 'osx', 'win32': 'windows-x64', 'cygwin': 'windows-x64'}[sys.platform]
+    if platform == 'osx':
+        return subprocess.call(['brew', 'install', 'expect', 'postgresql@{0}'.format(version.split('.')[0])])
     name = 'postgresql-{0}-{1}-binaries.zip'.format(version, platform)
     get_file('http://get.enterprisedb.com/postgresql/' + name, name)
     unzip_all(name)
     bin_dir = os.path.join('pgsql', 'bin')
     for f in os.listdir(bin_dir):
         chmod_755(os.path.join(bin_dir, f))
-    subprocess.call(['pgsql/bin/postgres', '-V'])
-    return 0
-
-
-def setup_kubernetes():
-    get_file('https://storage.googleapis.com/minikube/k8sReleases/v1.7.0/localkube-linux-amd64', 'localkube')
-    chmod_755('localkube')
-
-    devnull = open(os.devnull, 'w')
-    subprocess.Popen(['sudo', 'nohup', './localkube', '--logtostderr=true', '--enable-dns=false'],
-                     stdout=devnull, stderr=devnull)
-    for _ in range(0, 120):
-        if subprocess.call(['wget', '-qO', '-', 'http://127.0.0.1:8080/'], stdout=devnull, stderr=devnull) == 0:
-            break
-        time.sleep(1)
-    else:
-        print('localkube did not start')
-        return 1
-
-    subprocess.call('sudo chmod 644 /var/lib/localkube/certs/*', shell=True)
-    print('Set up .kube/config')
-    kube = os.path.join(os.path.expanduser('~'), '.kube')
-    os.makedirs(kube)
-    with open(os.path.join(kube, 'config'), 'w') as f:
-        f.write("""apiVersion: v1
-clusters:
-- cluster:
-    certificate-authority: /var/lib/localkube/certs/ca.crt
-    server: https://127.0.0.1:8443
-  name: local
-contexts:
-- context:
-    cluster: local
-    user: myself
-  name: local
-current-context: local
-kind: Config
-preferences: {}
-users:
-- name: myself
-  user:
-    client-certificate: /var/lib/localkube/certs/apiserver.crt
-    client-key: /var/lib/localkube/certs/apiserver.key
-""")
-    return 0
+    return subprocess.call(['pgsql/bin/postgres', '-V'])
 
 
 def main():
@@ -171,8 +127,6 @@ def main():
     if what != 'all':
         if sys.platform.startswith('linux'):
             r = install_packages(what)
-            if r == 0 and what == 'kubernetes':
-                r = setup_kubernetes()
         else:
             r = install_postgres()
 

.github/workflows/mapping.py

@@ -1 +1 @@
-versions = {'etcd': '9.6', 'etcd3': '14', 'consul': '13', 'exhibitor': '12', 'raft': '11', 'kubernetes': '14'}
+versions = {'etcd': '9.6', 'etcd3': '14', 'consul': '13', 'exhibitor': '12', 'raft': '11', 'kubernetes': '15'}

.github/workflows/release.yaml

@@ -0,0 +1,41 @@
+name: Publish Patroni distributions to PyPI and TestPyPI
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+  release:
+    types:
+      - published
+jobs:
+  build-n-publish:
+    name: Build and publish Patroni distributions to PyPI and TestPyPI
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+
+    - name: Run tests and flake8
+      run: python .github/workflows/run_tests.py
+
+    - name: Build a binary wheel and a source tarball
+      run: python setup.py sdist bdist_wheel
+
+    - name: Publish distribution to Test PyPI
+      if: github.event_name == 'push'
+      uses: pypa/[email protected]
+      with:
+        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        repository_url: https://test.pypi.org/legacy/
+
+    - name: Publish distribution to PyPI
+      if: github.event_name == 'release'
+      uses: pypa/[email protected]
+      with:
+        password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/run_tests.py

@@ -28,22 +28,20 @@ def main():
         version = versions.get(what)
         path = '/usr/lib/postgresql/{0}/bin:.'.format(version)
         unbuffer = ['timeout', '900', 'unbuffer']
-        args = ['--tags=-skip'] if what == 'etcd' else []
     else:
-        path = os.path.abspath(os.path.join('pgsql', 'bin'))
         if sys.platform == 'darwin':
-            path += ':.'
-        args = unbuffer = []
+            version = os.environ.get('PGVERSION', '15.1-1')
+            path = '/usr/local/opt/postgresql@{0}/bin:.'.format(version.split('.')[0])
+            unbuffer = ['unbuffer']
+        else:
+            path = os.path.abspath(os.path.join('pgsql', 'bin'))
+            unbuffer = []
     env['PATH'] = path + os.pathsep + env['PATH']
     env['DCS'] = what
+    if what == 'kubernetes':
+        env['PATRONI_KUBERNETES_CONTEXT'] = 'k3d-k3s-default'
 
-    ret = subprocess.call(unbuffer + [sys.executable, '-m', 'behave'] + args, env=env)
-
-    if ret != 0:
-        if subprocess.call('grep . features/output/*_failed/*postgres?.*', shell=True) != 0:
-            subprocess.call('grep . features/output/*/*postgres?.*', shell=True)
-        return 1
-    return 0
+    return subprocess.call(unbuffer + [sys.executable, '-m', 'behave'], env=env)
 
 
 if __name__ == '__main__':

.github/workflows/tests.yaml

@@ -5,8 +5,10 @@ on:
   push:
     branches:
     - master
-    tags:
-    - v.*
+
+env:
+  CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
+  SECRETS_AVAILABLE: ${{ secrets.CODACY_PROJECT_TOKEN != '' }}
 
 jobs:
   unit:
@@ -17,30 +19,10 @@ jobs:
         os: [ubuntu, windows, macos]
 
     steps:
-    - uses: actions/checkout@v1
-    - name: Set up Python 2.7
-      uses: actions/setup-python@v2
-      with:
-        python-version: 2.7
-      if: matrix.os != 'windows'
-    - name: Install dependencies
-      run: python .github/workflows/install_deps.py
-      if: matrix.os != 'windows'
-    - name: Run tests and flake8
-      run: python .github/workflows/run_tests.py
-      if: matrix.os != 'windows'
-
-    - name: Set up Python 3.6
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.6
-    - name: Install dependencies
-      run: python .github/workflows/install_deps.py
-    - name: Run tests and flake8
-      run: python .github/workflows/run_tests.py
+    - uses: actions/checkout@v3
 
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.7
     - name: Install dependencies
@@ -49,7 +31,7 @@ jobs:
       run: python .github/workflows/run_tests.py
 
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.8
     - name: Install dependencies
@@ -58,7 +40,7 @@ jobs:
       run: python .github/workflows/run_tests.py
 
     - name: Set up Python 3.9
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - name: Install dependencies
@@ -67,7 +49,7 @@ jobs:
       run: python .github/workflows/run_tests.py
 
     - name: Set up Python 3.10
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: '3.10'
     - name: Install dependencies
@@ -75,6 +57,15 @@ jobs:
     - name: Run tests and flake8
       run: python .github/workflows/run_tests.py
 
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.11
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+    - name: Run tests and flake8
+      run: python .github/workflows/run_tests.py
+
     - name: Combine coverage
       run: python .github/workflows/run_tests.py combine
 
@@ -92,60 +83,75 @@ jobs:
     runs-on: ${{ matrix.os }}-latest
     env:
       DCS: ${{ matrix.dcs }}
-      ETCDVERSION: 3.3.13
-      PGVERSION: 12.1-1  # for windows and macos
+      ETCDVERSION: 3.4.23
+      PGVERSION: 15.1-1  # for windows and macos
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu]
-        python-version: [2.7, 3.6, 3.9]
+        python-version: [3.7, '3.10']
         dcs: [etcd, etcd3, consul, exhibitor, kubernetes, raft]
-        exclude:
-        - dcs: kubernetes
-          python-version: 2.7
         include:
         - os: macos
-          python-version: 3.7
+          python-version: 3.8
           dcs: raft
         - os: macos
-          python-version: 3.8
+          python-version: 3.9
           dcs: etcd
         - os: macos
-          python-version: '3.10'
+          python-version: 3.11
           dcs: etcd3
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
+    - uses: nolar/setup-k3d-k3s@v1
+      if: matrix.dcs == 'kubernetes'
     - name: Add postgresql apt repo
-      run: sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+      run: |
+        sudo apt-get update -y
+        sudo apt-get install -y wget ca-certificates gnupg
+        sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+        sudo sh -c 'wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg'
       if: matrix.os == 'ubuntu'
     - name: Install dependencies
       run: python .github/workflows/install_deps.py
     - name: Run behave tests
       run: python .github/workflows/run_tests.py
-    - uses: actions/setup-python@v2
+    - name: Upload logs if behave failed
+      uses: actions/upload-artifact@v3
+      if: failure()
       with:
-        python-version: '3.10'
-    - name: Install coveralls
-      run: python -m pip install coveralls
-    - name: Upload Coverage
-      env:
-        COVERALLS_FLAG_NAME: behave-${{ matrix.os }}-${{ matrix.dcs }}-${{ matrix.python-version }}
-        COVERALLS_PARALLEL: 'true'
-        GITHUB_TOKEN: ${{ secrets.github_token }}
-      run: python -m coveralls --service=github
+        name: behave-${{ matrix.os }}-${{ matrix.dcs }}-${{ matrix.python-version }}-logs
+        path: |
+          features/output/*_failed/*postgres?.*
+          features/output/*.log
+        if-no-files-found: error
+        retention-days: 5
+    - name: Generate coverage xml report
+      run: python -m coverage xml -o cobertura.xml
+    - name: Upload coverage to Codacy
+      run: bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r cobertura.xml -l Python --partial
+      if: ${{ env.SECRETS_AVAILABLE == 'true' }}
 
   coveralls-finish:
     name: Finalize coveralls.io
-    needs: [unit, behave]
+    needs: unit
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
     - run: python -m pip install coveralls
     - run: python -m coveralls --service=github --finish
       env:
         GITHUB_TOKEN: ${{ secrets.github_token }}
+
+  codacy-final:
+    name: Finalize Codacy
+    needs: behave
+    runs-on: ubuntu-latest
+    steps:
+    - run: bash <(curl -Ls https://coverage.codacy.com/get.sh) final
+      if: ${{ env.SECRETS_AVAILABLE == 'true' }}

CODEOWNERS

@@ -0,0 +1,2 @@
+# global owners
+*        @CyberDem0n @hughcapet

Dockerfile

@@ -1,6 +1,6 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
-ARG PG_MAJOR=10
+ARG PG_MAJOR=15
 ARG COMPRESS=false
 ARG PGHOME=/home/postgres
 ARG PGDATA=$PGHOME/data
@@ -43,18 +43,18 @@ RUN set -ex \
     && echo 'syntax on\nfiletype plugin indent on\nset mouse-=a\nautocmd FileType yaml setlocal ts=2 sts=2 sw=2 expandtab' > /etc/vim/vimrc.local \
 \
     # Prepare postgres/patroni/haproxy environment
-    && mkdir -p $PGHOME/.config/patroni /patroni /run/haproxy \
-    && ln -s ../../postgres0.yml $PGHOME/.config/patroni/patronictl.yaml \
+    && mkdir -p "$PGHOME/.config/patroni" /patroni /run/haproxy \
+    && ln -s ../../postgres0.yml "$PGHOME/.config/patroni/patronictl.yaml" \
     && ln -s /patronictl.py /usr/local/bin/patronictl \
     && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \
     && chown -R postgres:postgres /var/log \
 \
     # Download etcd
-    && curl -sL https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
+    && curl -sL "https://github.com/coreos/etcd/releases/download/v$ETCDVERSION/etcd-v$ETCDVERSION-linux-$(dpkg --print-architecture).tar.gz" \
             | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl \
 \
     # Download confd
-    && curl -sL https://github.com/kelseyhightower/confd/releases/download/v${CONFDVERSION}/confd-${CONFDVERSION}-linux-amd64 \
+    && curl -sL "https://github.com/kelseyhightower/confd/releases/download/v$CONFDVERSION/confd-$CONFDVERSION-linux-$(dpkg --print-architecture)" \
             > /usr/local/bin/confd && chmod +x /usr/local/bin/confd \
 \
     # Clean up all useless packages and some files
@@ -90,7 +90,7 @@ RUN set -ex \
     && find /usr/bin -xtype l -delete \
     && find /var/log -type f -exec truncate --size 0 {} \; \
     && find /usr/lib/python3/dist-packages -name '*test*' | xargs rm -fr \
-    && find /lib/x86_64-linux-gnu/security -type f ! -name pam_env.so ! -name pam_permit.so ! -name pam_unix.so -delete
+    && find /lib/$(uname -m)-linux-gnu/security -type f ! -name pam_env.so ! -name pam_permit.so ! -name pam_unix.so -delete
 
 # perform compression if it is necessary
 ARG COMPRESS
@@ -99,8 +99,10 @@ RUN if [ "$COMPRESS" = "true" ]; then \
         # Allow certain sudo commands from postgres
         && echo 'postgres ALL=(ALL) NOPASSWD: /bin/tar xpJf /a.tar.xz -C /, /bin/rm /a.tar.xz, /bin/ln -snf dash /bin/sh' >> /etc/sudoers \
         && ln -snf busybox /bin/sh \
-        && files="/bin/sh /usr/bin/sudo /usr/lib/sudo/sudoers.so /lib/x86_64-linux-gnu/security/pam_*.so" \
-        && libs="$(ldd $files | awk '{print $3;}' | grep '^/' | sort -u) /lib/x86_64-linux-gnu/ld-linux-x86-64.so.* /lib/x86_64-linux-gnu/libnsl.so.* /lib/x86_64-linux-gnu/libnss_compat.so.*" \
+        && arch=$(uname -m) \
+        && darch=$(uname -m | sed 's/_/-/') \
+        && files="/bin/sh /usr/bin/sudo /usr/lib/sudo/sudoers.so /lib/$arch-linux-gnu/security/pam_*.so" \
+        && libs="$(ldd $files | awk '{print $3;}' | grep '^/' | sort -u) /lib/ld-linux-$darch.so.* /lib/$arch-linux-gnu/ld-linux-$darch.so.* /lib/$arch-linux-gnu/libnsl.so.* /lib/$arch-linux-gnu/libnss_compat.so.* /lib/$arch-linux-gnu/libnss_files.so.*" \
         && (echo /var/run $files $libs | tr ' ' '\n' && realpath $files $libs) | sort -u | sed 's/^\///' > /exclude \
         && find /etc/alternatives -xtype l -delete \
         && save_dirs="usr lib var bin sbin etc/ssl etc/init.d etc/alternatives etc/apt" \
@@ -117,7 +119,7 @@ RUN if [ "$COMPRESS" = "true" ]; then \
 FROM scratch
 COPY --from=builder / /
 
-LABEL maintainer="Alexander Kukushkin <alexander.kukushkin@zalando.de>"
+LABEL maintainer="Alexander Kukushkin <akukushkin@microsoft.com>"
 
 ARG PG_MAJOR
 ARG COMPRESS
@@ -151,7 +153,7 @@ RUN sed -i 's/env python/&3/' /patroni*.py \
     && sed -i 's/^      parameters:/      pg_hba:\n      - local all all trust\n      - host replication all all md5\n      - host all all all md5\n&\n        max_connections: 100/'  postgres?.yml \
     && if [ "$COMPRESS" = "true" ]; then chmod u+s /usr/bin/sudo; fi \
     && chmod +s /bin/ping \
-    && chown -R postgres:postgres $PGHOME /run /etc/haproxy
+    && chown -R postgres:postgres "$PGHOME" /run /etc/haproxy
 
 USER postgres
 

Dockerfile.citus

@@ -0,0 +1,173 @@
+## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
+## It has all the necessary components to play/debug with a single node appliance, running etcd
+ARG PG_MAJOR=15
+ARG COMPRESS=false
+ARG PGHOME=/home/postgres
+ARG PGDATA=$PGHOME/data
+ARG LC_ALL=C.UTF-8
+ARG LANG=C.UTF-8
+
+FROM postgres:$PG_MAJOR as builder
+
+ARG PGHOME
+ARG PGDATA
+ARG LC_ALL
+ARG LANG
+
+ENV ETCDVERSION=3.3.13 CONFDVERSION=0.16.0
+
+RUN set -ex \
+    && export DEBIAN_FRONTEND=noninteractive \
+    && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && apt-get update -y \
+    # postgres:10 is based on debian, which has the patroni package. We will install all required dependencies
+    && apt-cache depends patroni | sed -n -e 's/.*Depends: \(python3-.\+\)$/\1/p' \
+            | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
+            | xargs apt-get install -y vim curl less jq locales haproxy sudo \
+                            python3-etcd python3-kazoo python3-pip busybox \
+                            net-tools iputils-ping --fix-missing \
+    && curl https://install.citusdata.com/community/deb.sh | bash \
+    && apt-get -y install postgresql-$PG_MAJOR-citus-11.1 \
+    && pip3 install dumb-init \
+\
+    # Cleanup all locales but en_US.UTF-8
+    && find /usr/share/i18n/charmaps/ -type f ! -name UTF-8.gz -delete \
+    && find /usr/share/i18n/locales/ -type f ! -name en_US ! -name en_GB ! -name i18n* ! -name iso14651_t1 ! -name iso14651_t1_common ! -name 'translit_*' -delete \
+    && echo 'en_US.UTF-8 UTF-8' > /usr/share/i18n/SUPPORTED \
+\
+    # Make sure we have a en_US.UTF-8 locale available
+    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
+\
+    # haproxy dummy config
+    && echo 'global\n        stats socket /run/haproxy/admin.sock mode 660 level admin' > /etc/haproxy/haproxy.cfg \
+\
+    # vim config
+    && echo 'syntax on\nfiletype plugin indent on\nset mouse-=a\nautocmd FileType yaml setlocal ts=2 sts=2 sw=2 expandtab' > /etc/vim/vimrc.local \
+\
+    # Prepare postgres/patroni/haproxy environment
+    && mkdir -p $PGHOME/.config/patroni /patroni /run/haproxy \
+    && ln -s ../../postgres0.yml $PGHOME/.config/patroni/patronictl.yaml \
+    && ln -s /patronictl.py /usr/local/bin/patronictl \
+    && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \
+    && chown -R postgres:postgres /var/log \
+\
+    # Download etcd
+    && curl -sL https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-$(dpkg --print-architecture).tar.gz \
+            | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl \
+\
+    # Download confd
+    && curl -sL https://github.com/kelseyhightower/confd/releases/download/v${CONFDVERSION}/confd-${CONFDVERSION}-linux-$(dpkg --print-architecture) \
+            > /usr/local/bin/confd && chmod +x /usr/local/bin/confd \
+    # Prepare client cert for HAProxy
+    && cat /etc/ssl/private/ssl-cert-snakeoil.key  /etc/ssl/certs/ssl-cert-snakeoil.pem > /etc/ssl/private/ssl-cert-snakeoil.crt \
+\
+    # Clean up all useless packages and some files
+    && apt-get purge -y --allow-remove-essential python3-pip gzip bzip2 util-linux e2fsprogs \
+                libmagic1 bsdmainutils login ncurses-bin libmagic-mgc e2fslibs bsdutils \
+                exim4-config gnupg-agent dirmngr libpython2.7-stdlib libpython2.7-minimal \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* \
+        /root/.cache \
+        /var/cache/debconf/* \
+        /etc/rc?.d \
+        /etc/systemd \
+        /docker-entrypoint* \
+        /sbin/pam* \
+        /sbin/swap* \
+        /sbin/unix* \
+        /usr/local/bin/gosu \
+        /usr/sbin/[acgipr]* \
+        /usr/sbin/*user* \
+        /usr/share/doc* \
+        /usr/share/man \
+        /usr/share/info \
+        /usr/share/i18n/locales/translit_hangul \
+        /usr/share/locale/?? \
+        /usr/share/locale/??_?? \
+        /usr/share/postgresql/*/man \
+        /usr/share/postgresql-common/pg_wrapper \
+        /usr/share/vim/vim80/doc \
+        /usr/share/vim/vim80/lang \
+        /usr/share/vim/vim80/tutor \
+#        /var/lib/dpkg/info/* \
+    && find /usr/bin -xtype l -delete \
+    && find /var/log -type f -exec truncate --size 0 {} \; \
+    && find /usr/lib/python3/dist-packages -name '*test*' | xargs rm -fr \
+    && find /lib/$(uname -m)-linux-gnu/security -type f ! -name pam_env.so ! -name pam_permit.so ! -name pam_unix.so -delete
+
+# perform compression if it is necessary
+ARG COMPRESS
+RUN if [ "$COMPRESS" = "true" ]; then \
+        set -ex \
+        # Allow certain sudo commands from postgres
+        && echo 'postgres ALL=(ALL) NOPASSWD: /bin/tar xpJf /a.tar.xz -C /, /bin/rm /a.tar.xz, /bin/ln -snf dash /bin/sh' >> /etc/sudoers \
+        && ln -snf busybox /bin/sh \
+        && arch=$(uname -m) \
+        && darch=$(uname -m | sed 's/_/-/') \
+        && files="/bin/sh /usr/bin/sudo /usr/lib/sudo/sudoers.so /lib/$arch-linux-gnu/security/pam_*.so" \
+        && libs="$(ldd $files | awk '{print $3;}' | grep '^/' | sort -u) /lib/ld-linux-$darch.so.* /lib/$arch-linux-gnu/ld-linux-$darch.so.* /lib/$arch-linux-gnu/libnsl.so.* /lib/$arch-linux-gnu/libnss_compat.so.* /lib/$arch-linux-gnu/libnss_files.so.*" \
+        && (echo /var/run $files $libs | tr ' ' '\n' && realpath $files $libs) | sort -u | sed 's/^\///' > /exclude \
+        && find /etc/alternatives -xtype l -delete \
+        && save_dirs="usr lib var bin sbin etc/ssl etc/init.d etc/alternatives etc/apt" \
+        && XZ_OPT=-e9v tar -X /exclude -cpJf a.tar.xz $save_dirs \
+        # we call "cat /exclude" to avoid including files from the $save_dirs that are also among
+        # the exceptions listed in the /exclude, as "uniq -u" eliminates all non-unique lines.
+        # By calling "cat /exclude" a second time we guarantee that there will be at least two lines
+        # for each exception and therefore they will be excluded from the output passed to 'rm'.
+        && /bin/busybox sh -c "(find $save_dirs -not -type d && cat /exclude /exclude && echo exclude) | sort | uniq -u | xargs /bin/busybox rm" \
+        && /bin/busybox --install -s \
+        && /bin/busybox sh -c "find $save_dirs -type d -depth -exec rmdir -p {} \; 2> /dev/null"; \
+    else \
+        /bin/busybox --install -s; \
+    fi
+
+FROM scratch
+COPY --from=builder / /
+
+LABEL maintainer="Alexander Kukushkin <[email protected]>"
+
+ARG PG_MAJOR
+ARG COMPRESS
+ARG PGHOME
+ARG PGDATA
+ARG LC_ALL
+ARG LANG
+
+ARG PGBIN=/usr/lib/postgresql/$PG_MAJOR/bin
+
+ENV LC_ALL=$LC_ALL LANG=$LANG EDITOR=/usr/bin/editor
+ENV PGDATA=$PGDATA PATH=$PATH:$PGBIN
+
+COPY patroni /patroni/
+COPY extras/confd/conf.d/haproxy.toml /etc/confd/conf.d/
+COPY extras/confd/templates/haproxy-citus.tmpl /etc/confd/templates/haproxy.tmpl
+COPY patroni*.py docker/entrypoint.sh /
+COPY postgres?.yml $PGHOME/
+
+WORKDIR $PGHOME
+
+RUN sed -i 's/env python/&3/' /patroni*.py \
+    # "fix" patroni configs
+    && sed -i 's/^\(  connect_address:\|  - host\)/#&/' postgres?.yml \
+    && sed -i 's/^  listen: 127.0.0.1/  listen: 0.0.0.0/' postgres?.yml \
+    && sed -i "s|^\(  data_dir: \).*|\1$PGDATA|" postgres?.yml \
+    && sed -i "s|^#\(  bin_dir: \).*|\1$PGBIN|" postgres?.yml \
+    && sed -i 's/^  - encoding: UTF8/  - locale: en_US.UTF-8\n&/' postgres?.yml \
+    && sed -i 's/^scope:/log:\n  loggers:\n    patroni.postgresql.citus: DEBUG\n#&/' postgres?.yml \
+    && sed -i 's/^\(name\|etcd\|  host\|  authentication\|  pg_hba\|  parameters\):/#&/' postgres?.yml \
+    && sed -i 's/^    \(replication\|superuser\|rewind\|unix_socket_directories\|\(\(  \)\{0,1\}\(username\|password\)\)\):/#&/' postgres?.yml \
+    && sed -i 's/^postgresql:/&\n  basebackup:\n    checkpoint: fast/' postgres?.yml \
+    && sed -i 's|^      parameters:|      pg_hba:\n      - local all all trust\n      - hostssl replication all all md5 clientcert=verify-ca\n      - hostssl all all all md5 clientcert=verify-ca\n&\n        max_connections: 100\n        shared_buffers: 16MB\n        ssl: "on"\n        ssl_ca_file: /etc/ssl/certs/ssl-cert-snakeoil.pem\n        ssl_cert_file: /etc/ssl/certs/ssl-cert-snakeoil.pem\n        ssl_key_file: /etc/ssl/private/ssl-cert-snakeoil.key\n        citus.node_conninfo: "sslrootcert=/etc/ssl/certs/ssl-cert-snakeoil.pem sslkey=/etc/ssl/private/ssl-cert-snakeoil.key sslcert=/etc/ssl/certs/ssl-cert-snakeoil.pem sslmode=verify-ca"|'  postgres?.yml \
+    && sed -i 's/^#\(ctl\|  certfile\|  keyfile\)/\1/' postgres?.yml \
+    && sed -i 's|^#  cafile: .*$|  verify_client: required\n  cafile: /etc/ssl/certs/ssl-cert-snakeoil.pem|' postgres?.yml \
+    && sed -i 's|^#  cacert: .*$|  cacert: /etc/ssl/certs/ssl-cert-snakeoil.pem|' postgres?.yml \
+    && sed -i 's/^#  insecure: .*/  insecure: on/' postgres?.yml \
+    # client cert for HAProxy to access Patroni REST API
+    && if [ "$COMPRESS" = "true" ]; then chmod u+s /usr/bin/sudo; fi \
+    && chmod +s /bin/ping \
+    && chown -R postgres:postgres $PGHOME /run /etc/haproxy
+
+USER postgres
+
+ENTRYPOINT ["/bin/sh", "/entrypoint.sh"]

MAINTAINERS

@@ -1,3 +1,2 @@
-Alexander Kukushkin <alexander.kukushkin@zalando.de>
-Feike Steenbergen <feike.steenbergen@zalando.de>
-Oleksii Kliukin <[email protected]>
+Alexander Kukushkin <akukushkin@microsoft.com>
+Polina Bungina <polina.bungina@zalando.de>

README.rst

@@ -12,7 +12,9 @@ Patroni is a template for you to create your own customized, high-availability s
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
 
-Currently supported PostgreSQL versions: 9.3 to 14.
+Currently supported PostgreSQL versions: 9.3 to 15.
+
+**Note to Citus users**: Starting from 3.0 Patroni nicely integrates with `Citus <https://www.citusdata.com>`__. Please check `Citus support <https://github.com/zalando/patroni/blob/master/docs/citus.rst>`__ page for more information.
 
 **Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the `Kubernetes <https://github.com/zalando/patroni/blob/master/docs/kubernetes.rst>`__ chapter of the Patroni documentation.
 
@@ -47,7 +49,7 @@ We report new releases information `here <https://github.com/zalando/patroni/rel
 Community
 =========
 
-There are two places to connect with the Patroni community: `on github <https://github.com/zalando/patroni>`__, via Issues and PRs, and on channel #patroni in the `PostgreSQL Slack <https://postgres-slack.herokuapp.com/>`__.  If you're using Patroni, or just interested, please join us.
+There are two places to connect with the Patroni community: `on github <https://github.com/zalando/patroni>`__, via Issues and PRs, and on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://postgresteam.slack.com/>`__.  If you're using Patroni, or just interested, please join us.
 
 ===================================
 Technical Requirements/Installation

TODO.md

@@ -1,12 +0,0 @@
-Failover
-========
-- When determining who should become master, include the minor version of PostgreSQL in the decision.
-
-Configuration
-==============
-- Provide a way to change pg_hba.conf of a running cluster on the Patroni level, without changing individual nodes.
-- Provide hooks to store and retrieve cluster-wide passwords without exposing them in a plain-text form to unauthorized users.
-
-Documentation
-==============
-- Document how to run cascading replication and possibly initialize the cluster without an access to the master node.

docker-compose-citus.yml

@@ -0,0 +1,139 @@
+# docker compose file for running a Citus cluster
+# with 3-node etcd v3 cluster as the DCS and one haproxy node.
+# The Citus cluster has a coordinator (3 nodes)
+# and two worker clusters (2 nodes).
+#
+# Before starting it up you need to build the docker image:
+# $ docker build -f Dockerfile.citus -t patroni-citus .
+# The cluster could be started as:
+# $ docker-compose -f docker-compose-citus.yml up -d
+# You can read more about it in the:
+# https://github.com/zalando/patroni/blob/master/docker/README.md#citus-cluster
+version: "2"
+
+networks:
+    demo:
+
+services:
+    etcd1: &etcd
+        image: patroni-citus
+        networks: [ demo ]
+        environment:
+            ETCDCTL_API: 3
+            ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380
+            ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379
+            ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380
+            ETCD_INITIAL_CLUSTER_STATE: new
+            ETCD_INITIAL_CLUSTER_TOKEN: tutorial
+        container_name: demo-etcd1
+        hostname: etcd1
+        command: etcd -name etcd1 -initial-advertise-peer-urls http://etcd1:2380
+
+    etcd2:
+        <<: *etcd
+        container_name: demo-etcd2
+        hostname: etcd2
+        command: etcd -name etcd2 -initial-advertise-peer-urls http://etcd2:2380
+
+    etcd3:
+        <<: *etcd
+        container_name: demo-etcd3
+        hostname: etcd3
+        command: etcd -name etcd3 -initial-advertise-peer-urls http://etcd3:2380
+
+    haproxy:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: haproxy
+        container_name: demo-haproxy
+        ports:
+            - "5000:5000"  # Access to the coorinator primary
+            - "5001:5001"  # Load-balancing across workers primaries
+        command: haproxy
+        environment: &haproxy_env
+            ETCDCTL_API: 3
+            ETCDCTL_ENDPOINTS: http://etcd1:2379,http://etcd2:2379,http://etcd3:2379
+            PATRONI_ETCD3_HOSTS: "'etcd1:2379','etcd2:2379','etcd3:2379'"
+            PATRONI_SCOPE: demo
+            PATRONI_CITUS_GROUP: 0
+            PATRONI_CITUS_DATABASE: citus
+            PGSSLMODE: verify-ca
+            PGSSLKEY: /etc/ssl/private/ssl-cert-snakeoil.key
+            PGSSLCERT: /etc/ssl/certs/ssl-cert-snakeoil.pem
+            PGSSLROOTCERT: /etc/ssl/certs/ssl-cert-snakeoil.pem
+
+    coord1:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: coord1
+        container_name: demo-coord1
+        environment: &coord_env
+            <<: *haproxy_env
+            PATRONI_NAME: coord1
+            PATRONI_CITUS_GROUP: 0
+
+    coord2:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: coord2
+        container_name: demo-coord2
+        environment:
+            <<: *coord_env
+            PATRONI_NAME: coord2
+
+    coord3:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: coord3
+        container_name: demo-coord3
+        environment:
+            <<: *coord_env
+            PATRONI_NAME: coord3
+
+
+    work1-1:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: work1-1
+        container_name: demo-work1-1
+        environment: &work1_env
+            <<: *haproxy_env
+            PATRONI_NAME: work1-1
+            PATRONI_CITUS_GROUP: 1
+
+    work1-2:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: work1-2
+        container_name: demo-work1-2
+        environment:
+            <<: *work1_env
+            PATRONI_NAME: work1-2
+
+
+    work2-1:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: work2-1
+        container_name: demo-work2-1
+        environment: &work2_env
+            <<: *haproxy_env
+            PATRONI_NAME: work2-1
+            PATRONI_CITUS_GROUP: 2
+
+    work2-2:
+        image: patroni-citus
+        networks: [ demo ]
+        env_file: docker/patroni.env
+        hostname: work2-2
+        container_name: demo-work2-2
+        environment:
+            <<: *work2_env
+            PATRONI_NAME: work2-2

docker-compose.yml

@@ -1,5 +1,12 @@
 # docker compose file for running a 3-node PostgreSQL cluster
 # with 3-node etcd cluster as the DCS and one haproxy node
+#
+# requires a patroni image build from the Dockerfile:
+# $ docker build -t patroni .
+# The cluster could be started as:
+# $ docker-compose up -d
+# You can read more about it in the:
+# https://github.com/zalando/patroni/blob/master/docker/README.md
 version: "2"
 
 networks:

docker/README.md

@@ -1,10 +1,10 @@
-# Patroni Dockerfile
-You can run Patroni in a docker container using this Dockerfile
+# Dockerfile and Dockerfile.citus
+You can run Patroni in a docker container using these Dockerfiles
 
-This Dockerfile is meant in aiding development of Patroni and quick testing of features. It is not a production-worthy
-Dockerfile
+They are meant in aiding development of Patroni and quick testing of features and not a production-worthy!
 
     docker build -t patroni .
+    docker build -f Dockerfile.citus -t patroni-citus .
 
 # Examples
 
@@ -12,7 +12,10 @@ Dockerfile
 
     docker run -d patroni
 
-## Three-node Patroni cluster with three-node etcd cluster and one haproxy container using docker-compose
+## Three-node Patroni cluster
+
+In addition to three Patroni containers the stack starts three containers with etcd (forming a three-node cluster), and one container with haproxy.
+The haproxy listens on ports 5000 (connects to the primary) and 5001 (does load-balancing between healthy standbys).
 
 Example session:
 
@@ -92,7 +95,8 @@ Example session:
     b2e169fcb8a34028: name=etcd1 peerURLs=http://etcd1:2380 clientURLs=http://etcd1:2379 isLeader=false
     postgres@patroni1:~$ exit
 
-    $ psql -h localhost -p 5000 -U postgres -W
+    $ docker exec -ti demo-haproxy bash
+    postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -W
     Password: postgres
     psql (11.2 (Ubuntu 11.2-1.pgdg18.04+1), server 10.7 (Debian 10.7-1.pgdg90+1))
     Type "help" for help.
@@ -105,7 +109,7 @@ Example session:
 
     localhost/postgres=# \q
 
-    $ psql -h localhost -p 5001 -U postgres -W
+    $postgres@haproxy:~ psql -h localhost -p 5001 -U postgres -W
     Password: postgres
     psql (11.2 (Ubuntu 11.2-1.pgdg18.04+1), server 10.7 (Debian 10.7-1.pgdg90+1))
     Type "help" for help.
@@ -115,3 +119,188 @@ Example session:
     ───────────────────
      t
     (1 row)
+
+## Citus cluster
+
+The stack starts three containers with etcd (forming a three-node etcd cluster), seven containers with Patroni+PostgreSQL+Citus (three coordinator nodes, and two worker clusters with two nodes each), and one container with haproxy.
+The haproxy listens on ports 5000 (connects to the coordinator primary) and 5001 (does load-balancing between worker primary nodes).
+
+Example session:
+
+    $ docker-compose -f docker-compose-citus.yml up -d
+    Creating demo-work2-1 ... done
+    Creating demo-work1-1 ... done
+    Creating demo-etcd2   ... done
+    Creating demo-etcd1   ... done
+    Creating demo-coord3  ... done
+    Creating demo-etcd3   ... done
+    Creating demo-coord1  ... done
+    Creating demo-haproxy ... done
+    Creating demo-work2-2 ... done
+    Creating demo-coord2  ... done
+    Creating demo-work1-2 ... done
+
+    $ docker ps
+    CONTAINER ID   IMAGE                  COMMAND                  CREATED         STATUS         PORTS                              NAMES
+    852d8885a612   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-coord3
+    cdd692f947ab   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-work1-2
+    9f4e340b36da   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-etcd3
+    d69c129a960a   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-etcd1
+    c5849689b8cd   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-coord1
+    c9d72bd6217d   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-work2-1
+    24b1b43efa05   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-coord2
+    cb0cc2b4ca0a   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-work2-2
+    9796c6b8aad5   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 5 seconds                                      demo-work1-1
+    8baccd74dcae   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-etcd2
+    353ec62a0187   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds   0.0.0.0:5000-5001->5000-5001/tcp   demo-haproxy
+
+    $ docker logs demo-coord1
+    2023-01-05 15:09:31,295 INFO: Selected new etcd server http://172.27.0.4:2379
+    2023-01-05 15:09:31,388 INFO: Lock owner: None; I am coord1
+    2023-01-05 15:09:31,501 INFO: trying to bootstrap a new cluster
+    ...
+    2023-01-05 15:09:45,096 INFO: postmaster pid=39
+    localhost:5432 - no response
+    2023-01-05 15:09:45.137 UTC [39] LOG:  starting PostgreSQL 15.1 (Debian 15.1-1.pgdg110+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
+    2023-01-05 15:09:45.137 UTC [39] LOG:  listening on IPv4 address "0.0.0.0", port 5432
+    2023-01-05 15:09:45.152 UTC [39] LOG:  listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
+    2023-01-05 15:09:45.177 UTC [43] LOG:  database system was shut down at 2023-01-05 15:09:32 UTC
+    2023-01-05 15:09:45.193 UTC [39] LOG:  database system is ready to accept connections
+    localhost:5432 - accepting connections
+    localhost:5432 - accepting connections
+    2023-01-05 15:09:46,139 INFO: establishing a new patroni connection to the postgres cluster
+    2023-01-05 15:09:46,208 INFO: running post_bootstrap
+    2023-01-05 15:09:47.209 UTC [55] LOG:  starting maintenance daemon on database 16386 user 10
+    2023-01-05 15:09:47.209 UTC [55] CONTEXT:  Citus maintenance daemon for database 16386 user 10
+    2023-01-05 15:09:47,215 WARNING: Could not activate Linux watchdog device: "Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'"
+    2023-01-05 15:09:47.446 UTC [41] LOG:  checkpoint starting: immediate force wait
+    2023-01-05 15:09:47,466 INFO: initialized a new cluster
+    2023-01-05 15:09:47,594 DEBUG: query(SELECT nodeid, groupid, nodename, nodeport, noderole FROM pg_catalog.pg_dist_node WHERE noderole = 'primary', ())
+    2023-01-05 15:09:47,594 INFO: establishing a new patroni connection to the postgres cluster
+    2023-01-05 15:09:47,467 INFO: Lock owner: coord1; I am coord1
+    2023-01-05 15:09:47,613 DEBUG: query(SELECT pg_catalog.citus_set_coordinator_host(%s, %s, 'primary', 'default'), ('172.27.0.6', 5432))
+    2023-01-05 15:09:47,924 INFO: no action. I am (coord1), the leader with the lock
+    2023-01-05 15:09:51.282 UTC [41] LOG:  checkpoint complete: wrote 1086 buffers (53.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.029 s, sync=3.746 s, total=3.837 s; sync files=280, longest=0.028 s, average=0.014 s; distance=8965 kB, estimate=8965 kB
+    2023-01-05 15:09:51.283 UTC [41] LOG:  checkpoint starting: immediate force wait
+    2023-01-05 15:09:51.495 UTC [41] LOG:  checkpoint complete: wrote 18 buffers (0.9%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.044 s, sync=0.091 s, total=0.212 s; sync files=15, longest=0.015 s, average=0.007 s; distance=67 kB, estimate=8076 kB
+    2023-01-05 15:09:57,467 INFO: Lock owner: coord1; I am coord1
+    2023-01-05 15:09:57,569 INFO: Assigning synchronous standby status to ['coord3']
+    server signaled
+    2023-01-05 15:09:57.574 UTC [39] LOG:  received SIGHUP, reloading configuration files
+    2023-01-05 15:09:57.580 UTC [39] LOG:  parameter "synchronous_standby_names" changed to "coord3"
+    2023-01-05 15:09:59,637 INFO: Synchronous standby status assigned to ['coord3']
+    2023-01-05 15:09:59,638 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.27.0.2', 5432, 1))
+    2023-01-05 15:09:59.690 UTC [67] LOG:  standby "coord3" is now a synchronous standby with priority 1
+    2023-01-05 15:09:59.690 UTC [67] STATEMENT:  START_REPLICATION SLOT "coord3" 0/3000000 TIMELINE 1
+    2023-01-05 15:09:59,694 INFO: no action. I am (coord1), the leader with the lock
+    2023-01-05 15:09:59,704 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.27.0.8', 5432, 2))
+    2023-01-05 15:10:07,625 INFO: no action. I am (coord1), the leader with the lock
+    2023-01-05 15:10:17,579 INFO: no action. I am (coord1), the leader with the lock
+
+    $ docker exec -ti demo-haproxy bash
+    postgres@haproxy:~$ etcdctl member list
+    1bab629f01fa9065, started, etcd3, http://etcd3:2380, http://172.27.0.10:2379
+    8ecb6af518d241cc, started, etcd2, http://etcd2:2380, http://172.27.0.4:2379
+    b2e169fcb8a34028, started, etcd1, http://etcd1:2380, http://172.27.0.7:2379
+
+    postgres@haproxy:~$ etcdctl get --keys-only --prefix /service/demo
+    /service/demo/0/config
+    /service/demo/0/initialize
+    /service/demo/0/leader
+    /service/demo/0/members/coord1
+    /service/demo/0/members/coord2
+    /service/demo/0/members/coord3
+    /service/demo/0/status
+    /service/demo/0/sync
+    /service/demo/1/config
+    /service/demo/1/initialize
+    /service/demo/1/leader
+    /service/demo/1/members/work1-1
+    /service/demo/1/members/work1-2
+    /service/demo/1/status
+    /service/demo/1/sync
+    /service/demo/2/config
+    /service/demo/2/initialize
+    /service/demo/2/leader
+    /service/demo/2/members/work2-1
+    /service/demo/2/members/work2-2
+    /service/demo/2/status
+    /service/demo/2/sync
+
+    postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus
+    Password for user postgres: postgres
+    psql (15.1 (Debian 15.1-1.pgdg110+1))
+    SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off)
+    Type "help" for help.
+
+    citus=# select pg_is_in_recovery();
+     pg_is_in_recovery
+    -------------------
+     f
+    (1 row)
+
+    citus=# table pg_dist_node;
+     nodeid | groupid |  nodename  | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards
+    --------+---------+------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------
+          1 |       0 | 172.27.0.6 |     5432 | default  | t           | t        | primary  | default     | t              | f
+          2 |       1 | 172.27.0.2 |     5432 | default  | t           | t        | primary  | default     | t              | t
+          3 |       2 | 172.27.0.8 |     5432 | default  | t           | t        | primary  | default     | t              | t
+    (3 rows)
+
+    citus=# \q
+
+    postgres@haproxy:~$ patronictl list
+    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+    |     0 | coord1  | 172.27.0.6  | Leader       | running |  1 |           |
+    |     0 | coord2  | 172.27.0.5  | Replica      | running |  1 |         0 |
+    |     0 | coord3  | 172.27.0.9  | Sync Standby | running |  1 |         0 |
+    |     1 | work1-1 | 172.27.0.2  | Leader       | running |  1 |           |
+    |     1 | work1-2 | 172.27.0.12 | Sync Standby | running |  1 |         0 |
+    |     2 | work2-1 | 172.27.0.11 | Sync Standby | running |  1 |         0 |
+    |     2 | work2-2 | 172.27.0.8  | Leader       | running |  1 |           |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+
+    postgres@haproxy:~$ patronictl switchover --group 2 --force
+    Current cluster topology
+    + Citus cluster: demo (group: 2, 7185185529556963355) +-----------+
+    | Member  | Host        | Role         | State   | TL | Lag in MB |
+    +---------+-------------+--------------+---------+----+-----------+
+    | work2-1 | 172.27.0.11 | Sync Standby | running |  1 |         0 |
+    | work2-2 | 172.27.0.8  | Leader       | running |  1 |           |
+    +---------+-------------+--------------+---------+----+-----------+
+    2023-01-05 15:29:29.54204 Successfully switched over to "work2-1"
+    + Citus cluster: demo (group: 2, 7185185529556963355) -------+
+    | Member  | Host        | Role    | State   | TL | Lag in MB |
+    +---------+-------------+---------+---------+----+-----------+
+    | work2-1 | 172.27.0.11 | Leader  | running |  1 |           |
+    | work2-2 | 172.27.0.8  | Replica | stopped |    |   unknown |
+    +---------+-------------+---------+---------+----+-----------+
+
+    postgres@haproxy:~$ patronictl list
+    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+    |     0 | coord1  | 172.27.0.6  | Leader       | running |  1 |           |
+    |     0 | coord2  | 172.27.0.5  | Replica      | running |  1 |         0 |
+    |     0 | coord3  | 172.27.0.9  | Sync Standby | running |  1 |         0 |
+    |     1 | work1-1 | 172.27.0.2  | Leader       | running |  1 |           |
+    |     1 | work1-2 | 172.27.0.12 | Sync Standby | running |  1 |         0 |
+    |     2 | work2-1 | 172.27.0.11 | Leader       | running |  2 |           |
+    |     2 | work2-2 | 172.27.0.8  | Sync Standby | running |  2 |         0 |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+
+    postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus
+    Password for user postgres: postgres
+    psql (15.1 (Debian 15.1-1.pgdg110+1))
+    SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off)
+    Type "help" for help.
+
+    citus=# table pg_dist_node;
+     nodeid | groupid |  nodename   | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards
+    --------+---------+-------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------
+          1 |       0 | 172.27.0.6  |     5432 | default  | t           | t        | primary  | default     | t              | f
+          3 |       2 | 172.27.0.11 |     5432 | default  | t           | t        | primary  | default     | t              | t
+          2 |       1 | 172.27.0.2  |     5432 | default  | t           | t        | primary  | default     | t              | t
+    (3 rows)

docker/entrypoint.sh

@@ -7,29 +7,36 @@ if [ -f /a.tar.xz ]; then
     sudo ln -snf dash /bin/sh
 fi
 
-readonly PATRONI_SCOPE=${PATRONI_SCOPE:-batman}
-PATRONI_NAMESPACE=${PATRONI_NAMESPACE:-/service}
-readonly PATRONI_NAMESPACE=${PATRONI_NAMESPACE%/}
-readonly DOCKER_IP=$(hostname --ip-address)
+readonly PATRONI_SCOPE="${PATRONI_SCOPE:-batman}"
+PATRONI_NAMESPACE="${PATRONI_NAMESPACE:-/service}"
+readonly PATRONI_NAMESPACE="${PATRONI_NAMESPACE%/}"
+DOCKER_IP=$(hostname --ip-address)
+readonly DOCKER_IP
 
 case "$1" in
     haproxy)
         haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D
-        CONFD="confd -prefix=$PATRONI_NAMESPACE/$PATRONI_SCOPE -interval=10 -backend"
-        if [ ! -z "$PATRONI_ZOOKEEPER_HOSTS" ]; then
-            while ! /usr/share/zookeeper/bin/zkCli.sh -server $PATRONI_ZOOKEEPER_HOSTS ls /; do
+        set -- confd "-prefix=$PATRONI_NAMESPACE/$PATRONI_SCOPE" -interval=10 -backend
+        if [ -n "$PATRONI_ZOOKEEPER_HOSTS" ]; then
+            while ! /usr/share/zookeeper/bin/zkCli.sh -server "$PATRONI_ZOOKEEPER_HOSTS" ls /; do
                 sleep 1
             done
-            exec dumb-init $CONFD zookeeper -node $PATRONI_ZOOKEEPER_HOSTS
+            set -- "$@" zookeeper -node "$PATRONI_ZOOKEEPER_HOSTS"
         else
-            while ! etcdctl cluster-health 2> /dev/null; do
+            while ! etcdctl member list 2> /dev/null; do
                 sleep 1
             done
-            exec dumb-init $CONFD etcdv3 -node $(echo $ETCDCTL_ENDPOINTS | sed 's/,/ -node /g')
+            set -- "$@" etcdv3
+            while IFS='' read -r line; do
+                set -- "$@" -node "$line"
+            done <<-EOT
+$(echo "$ETCDCTL_ENDPOINTS" | sed 's/,/\n/g')
+EOT
         fi
+        exec dumb-init "$@"
         ;;
     etcd)
-        exec "$@" -advertise-client-urls http://$DOCKER_IP:2379
+        exec "$@" -advertise-client-urls "http://$DOCKER_IP:2379"
         ;;
     zookeeper)
         exec /usr/share/zookeeper/bin/zkServer.sh start-foreground
@@ -56,5 +63,13 @@ export PATRONI_REPLICATION_USERNAME="${PATRONI_REPLICATION_USERNAME:-replicator}
 export PATRONI_REPLICATION_PASSWORD="${PATRONI_REPLICATION_PASSWORD:-replicate}"
 export PATRONI_SUPERUSER_USERNAME="${PATRONI_SUPERUSER_USERNAME:-postgres}"
 export PATRONI_SUPERUSER_PASSWORD="${PATRONI_SUPERUSER_PASSWORD:-postgres}"
+export PATRONI_REPLICATION_SSLMODE="${PATRONI_REPLICATION_SSLMODE:-$PGSSLMODE}"
+export PATRONI_REPLICATION_SSLKEY="${PATRONI_REPLICATION_SSLKEY:-$PGSSLKEY}"
+export PATRONI_REPLICATION_SSLCERT="${PATRONI_REPLICATION_SSLCERT:-$PGSSLCERT}"
+export PATRONI_REPLICATION_SSLROOTCERT="${PATRONI_REPLICATION_SSLROOTCERT:-$PGSSLROOTCERT}"
+export PATRONI_SUPERUSER_SSLMODE="${PATRONI_SUPERUSER_SSLMODE:-$PGSSLMODE}"
+export PATRONI_SUPERUSER_SSLKEY="${PATRONI_SUPERUSER_SSLKEY:-$PGSSLKEY}"
+export PATRONI_SUPERUSER_SSLCERT="${PATRONI_SUPERUSER_SSLCERT:-$PGSSLCERT}"
+export PATRONI_SUPERUSER_SSLROOTCERT="${PATRONI_SUPERUSER_SSLROOTCERT:-$PGSSLROOTCERT}"
 
 exec python3 /patroni.py postgres0.yml

docs/CONTRIBUTING.rst

@@ -8,7 +8,7 @@ Wanna contribute to Patroni? Yay - here is how!
 Chatting
 --------
 
-Just want to chat with other Patroni users?  Looking for interactive troubleshooting help? Join us on channel #patroni in the `PostgreSQL Slack <https://postgres-slack.herokuapp.com/>`__.
+Just want to chat with other Patroni users?  Looking for interactive troubleshooting help? Join us on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://postgresteam.slack.com/>`__.
 
 Running tests
 -------------

docs/ENVIRONMENT.rst

@@ -33,6 +33,13 @@ It is possible to create new database users right after the successful initializ
 
 Example: defining ``PATRONI_admin_PASSWORD=strongpasswd`` and ``PATRONI_admin_OPTIONS='createrole,createdb'`` will cause creation of the user **admin** with the password **strongpasswd** that is allowed to create other users and databases.
 
+Citus
+-----
+Enables integration Patroni with `Citus <https://docs.citusdata.com>`__. If configured, Patroni will take care of registering Citus worker nodes on the coordinator. You can find more information about Citus support :ref:`here <citus>`.
+
+-  **PATRONI\_CITUS\_GROUP**: the Citus group id, integer. Use ``0`` for coordinator and ``1``, ``2``, etc... for workers
+-  **PATRONI\_CITUS\_DATABASE**: the database where ``citus`` extension should be created. Must be the same on the coordinator and all workers. Currently only one database is supported.
+
 Consul
 ------
 -  **PATRONI\_CONSUL\_HOST**: the host:port for the Consul local agent.
@@ -47,7 +54,8 @@ Consul
 -  **PATRONI\_CONSUL\_DC**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
 -  **PATRONI\_CONSUL\_CONSISTENCY**: (optional) Select consul consistency mode. Possible values are ``default``, ``consistent``, or ``stale`` (more details in `consul API reference <https://www.consul.io/api/features/consistency.html/>`__)
 -  **PATRONI\_CONSUL\_CHECKS**: (optional) list of Consul health checks used for the session. By default an empty list is used.
--  **PATRONI\_CONSUL\_REGISTER\_SERVICE**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, replica or standby-leader depending on the node's role. Defaults to **false**
+-  **PATRONI\_CONSUL\_REGISTER\_SERVICE**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, primary, replica, or standby-leader depending on the node's role. Defaults to **false**
+-  **PATRONI\_CONSUL\_SERVICE\_TAGS**: (optional) additional static tags to add to the Consul service apart from the role (``master``/``primary``/``replica``/``standby-leader``). By default an empty list is used.
 -  **PATRONI\_CONSUL\_SERVICE\_CHECK\_INTERVAL**: (optional) how often to perform health check against registered url
 -  **PATRONI\_CONSUL\_SERVICE\_CHECK\_TLS\_SERVER\_NAME**: (optional) overide SNI host when connecting via TLS, see also `consul agent check API reference <https://www.consul.io/api-docs/agent/check#tlsservername>`__.
 
@@ -110,8 +118,8 @@ Kubernetes
 -  **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='[{"name": "postgresql", "port": 5432}]'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
 -  **PATRONI\_KUBERNETES\_CACERT**: (optional) Specifies the file with the CA_BUNDLE file with certificates of trusted CAs to use while verifying Kubernetes API SSL certs. If not provided, patroni will use the value provided by the ServiceAccount secret.
 
-Raft
-----
+Raft (deprecated)
+-----------------
 
 -  **PATRONI\_RAFT\_SELF\_ADDR**: ``ip:port`` to listen on for Raft connections. The ``self_addr`` must be accessible from other nodes of the cluster. If not set, the node will not participate in consensus.
 -  **PATRONI\_RAFT\_BIND\_ADDR**: (optional) ``ip:port`` to listen on for Raft connections. If not specified the ``self_addr`` will be used.
@@ -123,11 +131,12 @@ PostgreSQL
 ----------
 -  **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
 -  **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications.
+-  **PATRONI\_POSTGRESQL\_PROXY\_ADDRESS**: IP address + port through which a connection pool (e.g. pgbouncer) running next to Postgres is accessible. The value is written to the member key in DCS as ``proxy_url`` and could be used/useful for service discovery.
 -  **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
 -  **PATRONI\_POSTGRESQL\_CONFIG\_DIR**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni.
 -  **PATRONI\_POSTGRESQL\_BIN_DIR**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
 -  **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
--  **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
+-  **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access the replication source via streaming replication
 -  **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization.
 -  **PATRONI\_REPLICATION\_SSLMODE**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
 -  **PATRONI\_REPLICATION\_SSLKEY**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.

docs/README.rst

@@ -109,7 +109,7 @@ Planning the Number of PostgreSQL Nodes
 ---------------------------------------
 
 Patroni/PostgreSQL nodes are decoupled from DCS nodes (except when Patroni implements RAFT on its own) and therefore
-there is no requirement on the minimal number of nodes. Running a cluster consisting of one master and one standby is 
+there is no requirement on the minimal number of nodes. Running a cluster consisting of one primary and one standby is
 perfectly fine. You can add more standby nodes later.
 
 Running and Configuring
@@ -177,7 +177,7 @@ Testing an HA solution is a time consuming process, with many variables. This is
 That said, here are some pieces of your infrastructure you should be sure to test:
 
 * Network (the network in front of your system as well as the NICs [physical or virtual] themselves)
-* Disk IO 
+* Disk IO
 * file limits (nofile in Linux)
 * RAM. Even if you have oomkiller turned off as suggested, the unavailability of RAM could cause issues.
 * CPU

docs/SETTINGS.rst

@@ -17,21 +17,22 @@ Dynamic configuration is stored in the DCS (Distributed Configuration Store) and
 -  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
 -  **maximum\_lag\_on\_syncnode**: the maximum bytes a synchronous follower may lag before it is considered as an unhealthy candidate and swapped by healthy asynchronous follower. Patroni utilize the max replica lsn if there is more than one follower, otherwise it will use leader's current wal lsn. Default is -1, Patroni will not take action to swap synchronous unhealthy follower when the value is set to 0 or below. Please set the value high enough so Patroni won't swap synchrounous follower fequently during high transaction volume.
 -  **max\_timelines\_history**: maximum number of timeline history items kept in DCS.  Default value: 0. When set to 0, it keeps the full history in DCS.
--  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered (in seconds). Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
-- **master\_stop\_timeout**: The number of seconds Patroni is allowed to wait when stopping Postgres and effective only when synchronous_mode is enabled. When set to > 0 and the synchronous_mode is enabled, Patroni sends SIGKILL to the postmaster if the stop operation is running for more than the value set by master_stop_timeout. Set the value according to your durability/availability tradeoff. If the parameter is not set or set <= 0, master_stop_timeout does not apply.
+-  **primary\_start\_timeout**: the amount of time a primary is allowed to recover from failures before failover is triggered (in seconds). Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Worst case failover time for primary failure is: loop\_wait + primary\_start\_timeout + loop\_wait, unless primary\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
+- **primary\_stop\_timeout**: The number of seconds Patroni is allowed to wait when stopping Postgres and effective only when synchronous_mode is enabled. When set to > 0 and the synchronous_mode is enabled, Patroni sends SIGKILL to the postmaster if the stop operation is running for more than the value set by primary\_stop\_timeout. Set the value according to your durability/availability tradeoff. If the parameter is not set or set <= 0, primary\_stop\_timeout does not apply.
 -  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
--  **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the master. See :ref:`replication modes documentation <replication_modes>` for details.
+-  **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the primary. See :ref:`replication modes documentation <replication_modes>` for details.
+-  **failsafe\_mode**: Enables :ref:`DCS Failsafe Mode <dcs_failsafe_mode>`. Defaults to `false`.
 -  **postgresql**:
     -  **use\_pg\_rewind**: whether or not to use pg_rewind. Defaults to `false`.
     -  **use\_slots**: whether or not to use replication slots. Defaults to `true` on PostgreSQL 9.4+.
     -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. There is no recovery.conf anymore in PostgreSQL 12, but you may continue using this section, because Patroni handles it transparently.
     -  **parameters**: list of configuration settings for Postgres.
 -  **standby\_cluster**: if this section is defined, we want to bootstrap a standby cluster.
-    -  **host**: an address of remote master
-    -  **port**: a port of remote master
-    -  **primary\_slot\_name**: which slot on the remote master to use for replication. This parameter is optional, the default value is derived from the instance name (see function `slot_name_from_member_name`).
-    -  **create\_replica\_methods**: an ordered list of methods that can be used to bootstrap standby leader from the remote master, can be different from the list defined in :ref:`postgresql_settings`
-    -  **restore\_command**: command to restore WAL records from the remote master to standby leader, can be different from the list defined in :ref:`postgresql_settings`
+    -  **host**: an address of remote node
+    -  **port**: a port of remote node
+    -  **primary\_slot\_name**: which slot on the remote node to use for replication. This parameter is optional, the default value is derived from the instance name (see function `slot_name_from_member_name`).
+    -  **create\_replica\_methods**: an ordered list of methods that can be used to bootstrap standby leader from the remote primary, can be different from the list defined in :ref:`postgresql_settings`
+    -  **restore\_command**: command to restore WAL records from the remote primary to nodes in a standby cluster, can be different from the list defined in :ref:`postgresql_settings`
     -  **archive\_cleanup\_command**: cleanup command for standby leader
     -  **recovery\_min\_apply\_delay**: how long to wait before actually apply WAL records on a standby leader
 -  **slots**: define permanent replication slots. These slots will be preserved during switchover/failover. The logical slots are copied from the primary to a standby with restart, and after that their position advanced every **loop_wait** seconds (if necessary). Copying logical slot files performed via ``libpq`` connection and using either rewind or superuser credentials (see **postgresql.authentication** section). There is always a chance that the logical slot position on the replica is a bit behind the former primary, therefore application should be prepared that some messages could be received the second time after the failover. The easiest way of doing so - tracking ``confirmed_flush_lsn``. Enabling permanent logical replication slots requires **postgresql.use_slots** to be set and will also automatically enable the ``hot_standby_feedback``. Since the failover of logical replication slots is unsafe on PostgreSQL 9.6 and older and PostgreSQL version 10 is missing some important functions, the feature only works with PostgreSQL 11+.
@@ -111,6 +112,15 @@ Bootstrap configuration
                 -  **- createdb**
     -  **post\_bootstrap** or **post\_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
 
+.. _citus_settings:
+
+Citus
+-----
+Enables integration Patroni with `Citus <https://docs.citusdata.com>`__. If configured, Patroni will take care of registering Citus worker nodes on the coordinator. You can find more information about Citus support :ref:`here <citus>`.
+
+-  **group**: the Citus group id, integer. Use ``0`` for coordinator and ``1``, ``2``, etc... for workers
+-  **database**: the database where ``citus`` extension should be created. Must be the same on the coordinator and all workers. Currently only one database is supported.
+
 .. _consul_settings:
 
 Consul
@@ -129,9 +139,9 @@ Most of the parameters are optional, but you have to specify one of the **host**
 -  **dc**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
 -  **consistency**: (optional) Select consul consistency mode. Possible values are ``default``, ``consistent``, or ``stale`` (more details in `consul API reference <https://www.consul.io/api/features/consistency.html/>`__)
 -  **checks**: (optional) list of Consul health checks used for the session. By default an empty list is used.
--  **register\_service**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, replica or standby-leader depending on the node's role. Defaults to **false**.
--  **service\_tags**: (optional) additional static tags to add to the Consul service apart from the role (``master``/``replica``/``standby-leader``).  By default an empty list is used.
--  **service\_check\_interval**: (optional) how often to perform health check against registered url.
+-  **register\_service**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, primary, replica, or standby-leader depending on the node's role. Defaults to **false**.
+-  **service\_tags**: (optional) additional static tags to add to the Consul service apart from the role (``master``/``primary``/``replica``/``standby-leader``). By default an empty list is used.
+-  **service\_check\_interval**: (optional) how often to perform health check against registered url. Defaults to '5s'.
 -  **service\_check\_tls\_server\_name**: (optional) overide SNI host when connecting via TLS, see also `consul agent check API reference <https://www.consul.io/api-docs/agent/check#tlsservername>`__.
 
 The ``token`` needs to have the following ACL permissions:
@@ -183,7 +193,7 @@ ZooKeeper
 -  **key**: (optional) File with the client key.
 -  **key_password**: (optional) The client key password.
 -  **verify**: (optional) Whether to verify certificate or not. Defaults to ``true``.
--  **set_acls**: (optional) If set, configure Kazoo to apply a default ACL to each ZNode that it creates. ACLs will assume 'x509' schema and should be specified as a dictionary with the principal as the key and one or more permissions as a list in the value.  Permissions may be one of ``CREATE``, ``READ``, ``WRITE``, ``DELETE`` or ``ADMIN``.  For example, ``set_acls: {CN=principal1: [CREATE, READ], CN=principal2: [ALL]}``. 
+-  **set_acls**: (optional) If set, configure Kazoo to apply a default ACL to each ZNode that it creates. ACLs will assume 'x509' schema and should be specified as a dictionary with the principal as the key and one or more permissions as a list in the value.  Permissions may be one of ``CREATE``, ``READ``, ``WRITE``, ``DELETE`` or ``ADMIN``.  For example, ``set_acls: {CN=principal1: [CREATE, READ], CN=principal2: [ALL]}``.
 
 .. note::
     It is required to install ``kazoo>=2.6.0`` to support SSL.
@@ -212,8 +222,8 @@ Kubernetes
 
 .. _raft_settings:
 
-Raft
-----
+Raft (deprecated)
+-----------------
 -  **self\_addr**: ``ip:port`` to listen on for Raft connections. The ``self_addr`` must be accessible from other nodes of the cluster. If not set, the node will not participate in consensus.
 -  **bind\_addr**: (optional) ``ip:port`` to listen on for Raft connections. If not specified the ``self_addr`` will be used.
 -  **partner\_addrs**: list of other Patroni nodes in the cluster in format: ['ip1:port', 'ip2:port', 'etc...']
@@ -224,7 +234,7 @@ Raft
 
   - Q: How to list all the nodes providing consensus?
 
-    A: ``syncobj_admin -conn host:port`` -status where the host:port is the address of one of the cluster nodes
+    A: ``syncobj_admin -conn host:port -status`` where the host:port is the address of one of the cluster nodes
 
   - Q: Node that was a part of consensus and has gone and I can't reuse the same IP for other node. How to remove this node from the consensus?
 
@@ -262,7 +272,7 @@ PostgreSQL
             -  **gssencmode**: (optional) maps to the `gssencmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-GSSENCMODE>`__ connection parameter, which determines whether or with what priority a secure GSS TCP/IP connection will be negotiated with the server
             -  **channel_binding**: (optional) maps to the `channel_binding <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-CHANNEL-BINDING>`__ connection parameter, which controls the client's use of channel binding.
         -  **replication**:
-            -  **username**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
+            -  **username**: replication username; the user will be created during initialization. Replicas will use this user to access the replication source via streaming replication
             -  **password**: replication password; the user will be created during initialization.
             -  **sslmode**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
             -  **sslkey**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
@@ -292,6 +302,7 @@ PostgreSQL
             -  **on\_start**: run this script when the postgres starts.
             -  **on\_stop**: run this script when the postgres stops.
     -  **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications.
+    -  **proxy\_address**: IP address + port through which a connection pool (e.g. pgbouncer) running next to Postgres is accessible. The value is written to the member key in DCS as ``proxy_url`` and could be used/useful for service discovery.
     -  **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica.
        "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its
        own config item. See :ref:`custom replica creation methods documentation <custom_replica_creation>` for further explanation.
@@ -314,14 +325,16 @@ PostgreSQL
     -  **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds.
     -  **use\_pg\_rewind**: try to use pg\_rewind on the former leader when it joins cluster as a replica.
     -  **remove\_data\_directory\_on\_rewind\_failure**: If this option is enabled, Patroni will remove the PostgreSQL data directory and recreate the replica. Otherwise it will try to follow the new leader. Default value is **false**.
-    -  **remove\_data\_directory\_on\_diverged\_timelines**: Patroni will remove the PostgreSQL data directory and recreate the replica if it notices that timelines are diverging and the former master can not start streaming from the new master. This option is useful when ``pg_rewind`` can not be used. While performing timelines divergence check on PostgreSQL v10 and older Patroni will try to connect with replication credential to the "postgres" database. Hence, such access should be allowed in the  pg_hba.conf. Default value is **false**.
+    -  **remove\_data\_directory\_on\_diverged\_timelines**: Patroni will remove the PostgreSQL data directory and recreate the replica if it notices that timelines are diverging and the former primary can not start streaming from the new primary. This option is useful when ``pg_rewind`` can not be used. While performing timelines divergence check on PostgreSQL v10 and older Patroni will try to connect with replication credential to the "postgres" database. Hence, such access should be allowed in the  pg_hba.conf. Default value is **false**.
     -  **replica\_method**: for each create_replica_methods other than basebackup, you would add a configuration section of the same name. At a minimum, this should include "command" with a full path to the actual script to be executed. Other configuration parameters will be passed along to the script in the form "parameter=value".
     -  **pre\_promote**: a fencing script that executes during a failover after acquiring the leader lock but before promoting the replica. If the script exits with a non-zero code, Patroni does not promote the replica and removes the leader key from DCS.
 
+.. _restapi_settings:
+
 REST API
 --------
 -  **restapi**:
-        -  **connect\_address**: IP address (or hostname) and port, to access the Patroni's :ref:`REST API <rest_api>`. All the members of the cluster must be able to connect to this address, so unless the Patroni setup is intended for a demo inside the localhost, this address must be a non "localhost" or loopback address (ie: "localhost" or "127.0.0.1"). It can serve as an endpoint for HTTP health checks (read below about the "listen" REST API parameter), and also for user queries (either directly or via the REST API), as well as for the health checks done by the cluster members during leader elections (for example, to determine whether the master is still running, or if there is a node which has a WAL position that is ahead of the one doing the query; etc.) The connect_address is put in the member key in DCS, making it possible to translate the member name into the address to connect to its REST API.
+        -  **connect\_address**: IP address (or hostname) and port, to access the Patroni's :ref:`REST API <rest_api>`. All the members of the cluster must be able to connect to this address, so unless the Patroni setup is intended for a demo inside the localhost, this address must be a non "localhost" or loopback address (ie: "localhost" or "127.0.0.1"). It can serve as an endpoint for HTTP health checks (read below about the "listen" REST API parameter), and also for user queries (either directly or via the REST API), as well as for the health checks done by the cluster members during leader elections (for example, to determine whether the leader is still running, or if there is a node which has a WAL position that is ahead of the one doing the query; etc.) The connect_address is put in the member key in DCS, making it possible to translate the member name into the address to connect to its REST API.
 
         -  **listen**: IP address (or hostname) and port that Patroni will listen to for the REST API - to provide also the same health checks and cluster messaging between the participating nodes, as described above. to provide health-check information for HAProxy (or any other load balancer capable of doing a HTTP "OPTION" or "GET" checks).
 

docs/citus.rst

@@ -0,0 +1,352 @@
+.. _citus:
+
+Citus support
+=============
+
+Patroni makes it extremely simple to deploy `Multi-Node Citus`__ clusters.
+
+__ https://docs.citusdata.com/en/stable/installation/multi_node.html
+
+TL;DR
+-----
+
+There are only a few simple rules you need to follow:
+
+1. Citus extension must be available on all nodes. Absolute minimum supported
+   Citus version is 10.0, but, to take all benefits from transparent
+   switchovers and restarts of workers we recommend using at least Citus 11.2.
+2. Cluster name (``scope``) must be the same for all Citus nodes!
+3. Superuser credentials must be the same on coordinator and all worker
+   nodes, and ``pg_hba.conf`` should allow superuser access between all nodes.
+4. :ref:`REST API <restapi_settings>` access should be allowed from worker
+   nodes to the coordinator. E.g., credentials should be the same and if
+   configured, client certificates from worker nodes must be accepted by the
+   coordinator.
+5. Add the following section to the ``patroni.yaml``:
+
+.. code:: YAML
+
+        citus:
+          group: X  # 0 for coordinator and 1, 2, 3, etc for workers
+          database: citus  # must be the same on all nodes
+
+
+After that you just need to start Patroni and it will handle the rest:
+
+1. ``citus`` extension will be automatically added to ``shared_preload_libraries``.
+2. If ``max_prepared_transactions`` isn't explicitly set in the global
+   :ref:`dynamic configuration <dynamic_configuration>` Patroni will
+   automatically set it to ``2*max_connections``.
+3. The ``citus.database`` will be automatically created followed by ``CREATE EXTENSION citus``.
+4. Current superuser :ref:`credentials <postgresql_settings>` will be added to the ``pg_dist_authinfo``
+   table to allow cross-node communication. Don't forget to update them if
+   later you decide to change superuser username/password/sslcert/sslkey!
+5. The coordinator primary node will automatically discover worker primary
+   nodes and add them to the ``pg_dist_node`` table using the
+   ``citus_add_node()`` function.
+6. Patroni will also maintain ``pg_dist_node`` in case failover/switchover
+   on the coordinator or worker clusters occurs.
+
+patronictl
+----------
+
+Coordinator and worker clusters are physically different PostgreSQL/Patroni
+clusters that are just logically groupped together using Citus. Therefore in
+most cases it is not possible to manage them as a single entity.
+
+It results in two major differences in ``patronictl`` behaviour when
+``patroni.yaml`` has the ``citus`` section comparing with the usual:
+
+1. The ``list`` and the ``topology`` by default output all members of the Citus
+   formation (coordinators and workers). The new column ``Group`` indicates
+   which Citus group they belong to.
+2. For all ``patronictl`` commands the new option is introduced, named
+   ``--group``. For some commands the default value for the group might be
+   taken from the ``patroni.yaml``. For example, ``patronictl pause`` will
+   enable the maintenance mode by default for the ``group`` that is set in the
+   ``citus`` section, but for example for ``patronictl  switchover`` or
+   ``patronictl remove`` the group must be explicitly specified.
+
+An example of ``patronictl list`` output for the Citus cluster::
+
+    postgres@coord1:~$ patronictl list demo
+    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+    |     0 | coord1  | 172.27.0.10 | Replica      | running |  1 |         0 |
+    |     0 | coord2  | 172.27.0.6  | Sync Standby | running |  1 |         0 |
+    |     0 | coord3  | 172.27.0.4  | Leader       | running |  1 |           |
+    |     1 | work1-1 | 172.27.0.8  | Sync Standby | running |  1 |         0 |
+    |     1 | work1-2 | 172.27.0.2  | Leader       | running |  1 |           |
+    |     2 | work2-1 | 172.27.0.5  | Sync Standby | running |  1 |         0 |
+    |     2 | work2-2 | 172.27.0.7  | Leader       | running |  1 |           |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+
+If we add the ``--group`` option, the output will change to::
+
+    postgres@coord1:~$ patronictl list demo --group 0
+    + Citus cluster: demo (group: 0, 7179854923829112860) -----------+
+    | Member | Host        | Role         | State   | TL | Lag in MB |
+    +--------+-------------+--------------+---------+----+-----------+
+    | coord1 | 172.27.0.10 | Replica      | running |  1 |         0 |
+    | coord2 | 172.27.0.6  | Sync Standby | running |  1 |         0 |
+    | coord3 | 172.27.0.4  | Leader       | running |  1 |           |
+    +--------+-------------+--------------+---------+----+-----------+
+
+    postgres@coord1:~$ patronictl list demo --group 1
+    + Citus cluster: demo (group: 1, 7179854923881963547) -----------+
+    | Member  | Host       | Role         | State   | TL | Lag in MB |
+    +---------+------------+--------------+---------+----+-----------+
+    | work1-1 | 172.27.0.8 | Sync Standby | running |  1 |         0 |
+    | work1-2 | 172.27.0.2 | Leader       | running |  1 |           |
+    +---------+------------+--------------+---------+----+-----------+
+
+Citus worker switchover
+-----------------------
+
+When a switchover is orchestrated for a Citus worker node, Citus offers the
+opportunity to make the switchover close to transparent for an application.
+Because the application connects to the coordinator, which in turn connects to
+the worker nodes, then it is possible with Citus to `pause` the SQL traffic on
+the coordinator for the shards hosted on a worker node. The switchover then
+happens while the traffic is kept on the coordinator, and resumes as soon as a
+new primary worker node is ready to accept read-write queries.
+
+An example of ``patronictl switchover`` on the worker cluster::
+
+    postgres@coord1:~$ patronictl switchover demo
+    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+    |     0 | coord1  | 172.27.0.10 | Replica      | running |  1 |         0 |
+    |     0 | coord2  | 172.27.0.6  | Sync Standby | running |  1 |         0 |
+    |     0 | coord3  | 172.27.0.4  | Leader       | running |  1 |           |
+    |     1 | work1-1 | 172.27.0.8  | Leader       | running |  1 |           |
+    |     1 | work1-2 | 172.27.0.2  | Sync Standby | running |  1 |         0 |
+    |     2 | work2-1 | 172.27.0.5  | Sync Standby | running |  1 |         0 |
+    |     2 | work2-2 | 172.27.0.7  | Leader       | running |  1 |           |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+    Citus group: 2
+    Primary [work2-2]:
+    Candidate ['work2-1'] []:
+    When should the switchover take place (e.g. 2022-12-22T08:02 )  [now]:
+    Current cluster topology
+    + Citus cluster: demo (group: 2, 7179854924063375386) -----------+
+    | Member  | Host       | Role         | State   | TL | Lag in MB |
+    +---------+------------+--------------+---------+----+-----------+
+    | work2-1 | 172.27.0.5 | Sync Standby | running |  1 |         0 |
+    | work2-2 | 172.27.0.7 | Leader       | running |  1 |           |
+    +---------+------------+--------------+---------+----+-----------+
+    Are you sure you want to switchover cluster demo, demoting current primary work2-2? [y/N]: y
+    2022-12-22 07:02:40.33003 Successfully switched over to "work2-1"
+    + Citus cluster: demo (group: 2, 7179854924063375386) ------+
+    | Member  | Host       | Role    | State   | TL | Lag in MB |
+    +---------+------------+---------+---------+----+-----------+
+    | work2-1 | 172.27.0.5 | Leader  | running |  1 |           |
+    | work2-2 | 172.27.0.7 | Replica | stopped |    |   unknown |
+    +---------+------------+---------+---------+----+-----------+
+
+    postgres@coord1:~$ patronictl list demo
+    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+    |     0 | coord1  | 172.27.0.10 | Replica      | running |  1 |         0 |
+    |     0 | coord2  | 172.27.0.6  | Sync Standby | running |  1 |         0 |
+    |     0 | coord3  | 172.27.0.4  | Leader       | running |  1 |           |
+    |     1 | work1-1 | 172.27.0.8  | Leader       | running |  1 |           |
+    |     1 | work1-2 | 172.27.0.2  | Sync Standby | running |  1 |         0 |
+    |     2 | work2-1 | 172.27.0.5  | Leader       | running |  2 |           |
+    |     2 | work2-2 | 172.27.0.7  | Sync Standby | running |  2 |         0 |
+    +-------+---------+-------------+--------------+---------+----+-----------+
+
+And this is how it looks on the coordinator side::
+
+    # The worker primary notifies the coordinator that it is going to execute "pg_ctl stop".
+    2022-12-22 07:02:38,636 DEBUG: query("BEGIN")
+    2022-12-22 07:02:38,636 DEBUG: query("SELECT pg_catalog.citus_update_node(3, '172.27.0.7-demoted', 5432, true, 10000)")
+    # From this moment all application traffic on the coordinator to the worker group 2 is paused.
+
+    # The future worker primary notifies the coordinator that it acquired the leader lock in DCS and about to run "pg_ctl promote".
+    2022-12-22 07:02:40,085 DEBUG: query("SELECT pg_catalog.citus_update_node(3, '172.27.0.5', 5432)")
+
+    # The new worker primary just finished promote and notifies coordinator that it is ready to accept read-write traffic.
+    2022-12-22 07:02:41,485 DEBUG: query("COMMIT")
+    # From this moment the application traffic on the coordinator to the worker group 2 is unblocked.
+
+Peek into DCS
+-------------
+
+The Citus cluster (coordinator and workers) are stored in DCS as a fleet of
+Patroni clusters logically grouped together::
+
+    /service/batman/              # scope=batman
+    /service/batman/0/            # citus.group=0, coordinator
+    /service/batman/0/initialize
+    /service/batman/0/leader
+    /service/batman/0/members/
+    /service/batman/0/members/m1
+    /service/batman/0/members/m2
+    /service/batman/1/            # citus.group=1, worker
+    /service/batman/1/initialize
+    /service/batman/1/leader
+    /service/batman/1/members/
+    /service/batman/1/members/m3
+    /service/batman/1/members/m4
+    ...
+
+Such an approach was chosen because for most DCS it becomes possible to fetch
+the entire Citus cluster with a single recursive read request. Only Citus
+coordinator nodes are reading the whole tree, because they have to discover
+worker nodes. Worker nodes are reading only the subtree for their own group and
+in some cases they could read the subtree of the coordinator group.
+
+Citus on Kubernetes
+-------------------
+
+Since Kubernetes doesn't support hierarchical structures we had to include the
+citus group to all K8s objects Patroni creates::
+
+    batman-0-leader  # the leader config map for the coordinator
+    batman-0-config  # the config map holding initialize, config, and history "keys"
+    ...
+    batman-1-leader  # the leader config map for worker group 1
+    batman-1-config
+    ...
+
+I.e., the naming pattern is: ``${scope}-${citus.group}-${type}``.
+
+All Kubernetes objects are discovered by Patroni using the `label selector`__,
+therefore all Pods with Patroni&Citus and Endpoints/ConfigMaps must have
+similar labels, and Patroni must be configured to use them using Kubernetes
+:ref:`settings <kubernetes_settings>` or :ref:`environment variables
+<kubernetes_environment>`.
+
+__ https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors
+
+A couple of examples of Patroni configuration using Pods environment variables:
+
+1. for the coordinator cluster
+
+.. code:: YAML
+
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          labels:
+            application: patroni
+            citus-group: "0"
+            citus-type: coordinator
+            cluster-name: citusdemo
+          name: citusdemo-0-0
+          namespace: default
+        spec:
+          containers:
+          - env:
+            - name: PATRONI_SCOPE
+              value: citusdemo
+            - name: PATRONI_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
+            - name: PATRONI_KUBERNETES_POD_IP
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: status.podIP
+            - name: PATRONI_KUBERNETES_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: PATRONI_KUBERNETES_LABELS
+              value: '{application: patroni}'
+            - name: PATRONI_CITUS_DATABASE
+              value: citus
+            - name: PATRONI_CITUS_GROUP
+              value: "0"
+
+2. for the worker cluster from the group 2
+
+.. code:: YAML
+
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          labels:
+            application: patroni
+            citus-group: "2"
+            citus-type: worker
+            cluster-name: citusdemo
+          name: citusdemo-2-0
+          namespace: default
+        spec:
+          containers:
+          - env:
+            - name: PATRONI_SCOPE
+              value: citusdemo
+            - name: PATRONI_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
+            - name: PATRONI_KUBERNETES_POD_IP
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: status.podIP
+            - name: PATRONI_KUBERNETES_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: PATRONI_KUBERNETES_LABELS
+              value: '{application: patroni}'
+            - name: PATRONI_CITUS_DATABASE
+              value: citus
+            - name: PATRONI_CITUS_GROUP
+              value: "2"
+
+As you may noticed, both examples have ``citus-group`` label set. This label
+allows Patroni to identify object as belonging to a certain Citus group. In
+addition to that, there is also ``PATRONI_CITUS_GROUP`` environment variable,
+which has the same value as the ``citus-group`` label. When Patroni creates
+new Kubernetes objects ConfigMaps or Endpoints, it automatically puts the
+``citus-group: ${env.PATRONI_CITUS_GROUP}`` label on them:
+
+.. code:: YAML
+
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: citusdemo-0-leader  # Is generated as ${env.PATRONI_SCOPE}-${env.PATRONI_CITUS_GROUP}-leader
+          labels:
+            application: patroni    # Is set from the ${env.PATRONI_KUBERNETES_LABELS}
+            cluster-name: citusdemo # Is automatically set from the ${env.PATRONI_SCOPE}
+            citus-group: '0'        # Is automatically set from the ${env.PATRONI_CITUS_GROUP}
+
+You can find a complete example of Patroni deployment on Kubernetes with Citus
+support in the `kubernetes`__ folder of the Patroni repository.
+
+__ https://github.com/zalando/patroni/tree/master/kubernetes
+
+There are two important files for you:
+
+1. Dockerfile.citus
+2. citus_k8s.yaml
+
+Citus upgrades and PostgreSQL major upgrades
+--------------------------------------------
+
+First, please read about upgrading Citus version in the `documentation`__.
+There is one minor change in the process. When executing upgrade, you have to
+use ``patronictl restart`` instead of ``systemctl restart`` to restart
+PostgreSQL.
+
+__ https://docs.citusdata.com/en/latest/admin_guide/upgrading_citus.html
+
+The PostgreSQL major upgrade with Citus is a bit more complex. You will have to
+combine techniques used in the Citus documentation about major upgrades and
+Patroni documentation about :ref:`PostgreSQL major upgrade<major_upgrade>`.
+Please keep in mind that Citus cluster consists of many Patroni clusters
+(coordinator and workers) and they all have to be upgraded independently.

docs/dcs_failsafe_mode.rst

@@ -0,0 +1,63 @@
+.. _dcs_failsafe_mode:
+
+DCS Failsafe Mode
+=================
+
+The problem
+-----------
+
+Patroni is heavily relying on Distributed Configuration Store (DCS) to solve the task of leader elections and detect network partitioning. That is, the node is allowed to run Postgres as the primary only if it can update the leader lock in DCS. In case the update of the leader lock fails, Postgres is immediately demoted and started as read-only. Depending on which DCS is used, the chances of hitting the "problem" differ. For example, with Etcd which is only used for Patroni, chances are close to zero, while with K8s API (backed by Etcd) it could be observed more frequently.
+
+
+Reasons for the current implementation
+---------------------------------------
+
+The leader lock update failure could be caused by two main reasons:
+
+1. Network partitioning
+2. DCS being down
+
+In general, it is impossible to distinguish between these two from a single node, and therefore Patroni assumes the worst case - network partitioning. In the case of a partitioned network, other nodes of the Patroni cluster may successfully grab the leader lock and promote Postgres to primary. In order to avoid a split-brain, the old primary is demoted before the leader lock expires.
+
+
+DCS Failsafe Mode
+-----------------
+
+We introduce a new special option, the ``failsafe_mode``. It could be enabled only via global configuration stored in the DCS ``/config`` key. If the failsafe mode is enabled and the leader lock update in DCS failed due to reasons different from the version/value/index mismatch, Postgres may continue to run as a primary if it can access all known members of the cluster via Patroni REST API.
+
+
+Low-level implementation details
+--------------------------------
+
+- We introduce a new, permanent key in DCS, named ``/failsafe``.
+- The ``/failsafe`` key contains all known members of the given Patroni cluster at a given time.
+- The current leader maintains the ``/failsafe`` key.
+- The member is allowed to participate in the leader race and become the new leader only if it is present in the ``/failsafe`` key.
+- If the cluster consists of a single node the ``/failsafe`` key will contain a single member.
+- In the case of DCS "outage" the existing primary connects to all members presented in the ``/failsafe`` key via the ``POST /failsafe`` REST API and may continue to run as the primary if all replicas acknowledge it.
+- If one of the members doesn't respond, the primary is demoted.
+- Replicas are using incoming ``POST /failsafe`` REST API requests as an indicator that the primary is still alive. This information is cached for ``ttl`` seconds.
+
+
+F.A.Q.
+------
+
+- Why MUST the current primary see ALL other members? Can’t we rely on quorum here?
+
+  This is a great question! The problem is that the view on the quorum might be different from the perspective of DCS and Patroni. While DCS nodes must be evenly distributed across availability zones, there is no such rule for Patroni, and more importantly, there is no mechanism for introducing and enforcing such a rule. If the majority of Patroni nodes ends up in the losing part of the partitioned network (including primary) while minority nodes are in the winning part, the primary must be demoted. Only checking ALL other members allows detecting such a situation.
+
+- What if node/pod gets terminated while DCS is down?
+
+  If DCS isn’t accessible, the check “are ALL other cluster members accessible?” is executed every cycle of the heartbeat loop (every ``loop_wait`` seconds). If pod/node is terminated, the check will fail and Postgres will be demoted to a read-only and will not recover until DCS is restored.
+
+- What if all members of the Patroni cluster are lost while DCS is down?
+
+  Patroni could be configured to create the new replica from the backup even when the cluster doesn't have a leader. But, if the new member isn't present in the ``/failsafe`` key, it will not be able to grab the leader lock and promote.
+  
+- What will happen if the primary lost access to DCS while replicas didn't?
+
+  The primary will execute the failsafe code and contact all known replicas. These replicas will use this information as an indicator that the primary is alive and will not start the leader race even if the leader lock in DCS has expired.
+
+- How to enable the Failsafe Mode?
+
+  Before enabling the ``failsafe_mode`` please make sure that Patroni version on all members is up-to-date. After that, you can use either the ``PATCH /config`` :ref:`REST API <rest_api>` or ``patronictl edit-config -s failsafe_mode=true``

docs/dynamic_configuration.rst

@@ -22,7 +22,7 @@ Patroni configuration is stored in the DCS (Distributed Configuration Store). Th
 
 The local configuration can be either a single YAML file or a directory. When it is a directory, all YAML files in that directory are loaded one by one in sorted order. In case a key is defined in multiple files, the occurrence in the last file takes precedence.
 
-Some of the PostgreSQL parameters must hold the same values on the master and the replicas. For those, values set either in the local patroni configuration files or via the environment variables take no effect. To alter or set their values one must change the shared configuration in the DCS. Below is the actual list of such parameters together with the default values:
+Some of the PostgreSQL parameters must hold the same values on the primary and the replicas. For those, values set either in the local patroni configuration files or via the environment variables take no effect. To alter or set their values one must change the shared configuration in the DCS. Below is the actual list of such parameters together with the default values:
 
 - max_connections: 100
 - max_locks_per_transaction: 64
@@ -32,7 +32,7 @@ Some of the PostgreSQL parameters must hold the same values on the master and th
 - wal_log_hints: on
 - track_commit_timestamp: off
 
-For the parameters below, PostgreSQL does not require equal values among the master and all the replicas. However, considering the possibility of a replica to become the master at any time, it doesn't really make sense to set them differently; therefore, Patroni restricts setting their values to the Dynamic configuration
+For the parameters below, PostgreSQL does not require equal values among the primary and all the replicas. However, considering the possibility of a replica to become the primary at any time, it doesn't really make sense to set them differently; therefore, Patroni restricts setting their values to the Dynamic configuration
 
 - max_wal_senders: 5
 - max_replication_slots: 5
@@ -86,4 +86,4 @@ Also, the following Patroni configuration options can be changed only dynamicall
 Upon changing these options, Patroni will read the relevant section of the configuration stored in DCS and change its
 run-time values.
 
-Patroni nodes are dumping the state of the DCS options to disk upon for every change of the configuration into the file ``patroni.dynamic.json`` located in the Postgres data directory. Only the master is allowed to restore these options from the on-disk dump if these are completely absent from the DCS or if they are invalid.
+Patroni nodes are dumping the state of the DCS options to disk upon for every change of the configuration into the file ``patroni.dynamic.json`` located in the Postgres data directory. Only the leader is allowed to restore these options from the on-disk dump if these are completely absent from the DCS or if they are invalid.

docs/existing_data.rst

@@ -23,17 +23,19 @@ A Patroni cluster can be started with a data directory from a single-node Postgr
 3. Start Patroni (e.g. ``patroni /etc/patroni/patroni.yml``). It automatically detects that PostgreSQL daemon is already running but its configuration might be out-of-date.
 4. Ask Patroni to restart the node with ``patronictl restart cluster-name node-name``. This step is only required if PostgreSQL configuration is out-of-date.
 
+.. _major_upgrade:
+
 Major Upgrade of PostgreSQL Version
 ===================================
 
 The only possible way to do a major upgrade currently is:
 
 1. Stop Patroni
-2. Upgrade PostgreSQL binaries and perform `pg_upgrade <https://www.postgresql.org/docs/current/pgupgrade.html>`_ on the master node
+2. Upgrade PostgreSQL binaries and perform `pg_upgrade <https://www.postgresql.org/docs/current/pgupgrade.html>`_ on the primary node
 3. Update patroni.yml
 4. Remove the initialize key from DCS or wipe complete cluster state from DCS. The second one could be achieved by running ``patronictl remove <cluster-name>``. It is necessary because pg_upgrade runs initdb which actually creates a new database with a new PostgreSQL system identifier.
 5. If you wiped the cluster state in the previous step, you may wish to copy patroni.dynamic.json from old data dir to the new one.  It will help you to retain some PostgreSQL parameters you had set before.
-6. Start Patroni on the master node.
+6. Start Patroni on the primary node.
 7. Upgrade PostgreSQL binaries, update patroni.yml and wipe the data_dir on standby nodes.
 8. Start Patroni on the standby nodes and wait for the replication to complete.
 

docs/index.rst

@@ -10,7 +10,9 @@ Patroni is a template for you to create your own customized, high-availability s
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
-Currently supported PostgreSQL versions: 9.3 to 14.
+Currently supported PostgreSQL versions: 9.3 to 15.
+
+**Note to Citus users**: Starting from 3.0 Patroni nicely integrates with `Citus <https://www.citusdata.com>`__. Please check :ref:`Citus support <citus>` page for more information.
 
 **Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the :ref:`Kubernetes <kubernetes>` chapter of the Patroni documentation.
 
@@ -20,7 +22,9 @@ Currently supported PostgreSQL versions: 9.3 to 14.
    :caption: Contents:
 
    README
+   citus
    dynamic_configuration
+   dcs_failsafe_mode
    rest_api
    existing_data
    ENVIRONMENT

docs/kubernetes.rst

@@ -23,10 +23,7 @@ Use ConfigMaps
 In this mode, Patroni will create ConfigMaps instead of Endpoints and store keys inside meta-data of those ConfigMaps.
 Changing the leader takes at least two updates, one to the leader ConfigMap and another to the respective Endpoint.
 
-There are two ways to direct the traffic to the Postgres master:
-
-- use the `callback script <https://github.com/zalando/patroni/blob/master/kubernetes/callback.py>`_ provided by Patroni
-- configure the Kubernetes Postgres service to use the label selector with the `role_label` (configured in patroni configuration).
+To direct the traffic to the Postgres leader you need to configure the Kubernetes Postgres service to use the label selector with the `role_label` (configured in patroni configuration).
 
 Note that in some cases, for instance, when running on OpenShift, there is no alternative to using ConfigMaps.
 
@@ -39,7 +36,7 @@ Examples
 --------
 
 - The `kubernetes <https://github.com/zalando/patroni/tree/master/kubernetes>`__ folder of the Patroni repository contains
-  examples of the Docker image, the Kubernetes manifest and the callback script in order to test Patroni Kubernetes setup.
+  examples of the Docker image, and the Kubernetes manifest to test Patroni Kubernetes setup.
   Note that in the current state it will not be able to use PersistentVolumes because of permission issues.
 
 - You can find the full-featured Docker image that can use Persistent Volumes in the

docs/pause.rst

@@ -6,7 +6,7 @@ Pause/Resume mode for the cluster
 The goal
 --------
 
-Under certain circumstances Patroni needs to temporary step down from managing the cluster, while still retaining the cluster state in DCS. Possible use cases are uncommon activities on the cluster, such as major version upgrades or corruption recovery. During those activities nodes are often started and stopped for the reason unknown to Patroni, some nodes can be even temporary promoted, violating the assumption of running only one master. Therefore, Patroni needs to be able to "detach" from the running cluster, implementing an equivalent of the maintenance mode in Pacemaker.
+Under certain circumstances Patroni needs to temporarily step down from managing the cluster, while still retaining the cluster state in DCS. Possible use cases are uncommon activities on the cluster, such as major version upgrades or corruption recovery. During those activities nodes are often started and stopped for reasons unknown to Patroni, some nodes can be even temporarily promoted, violating the assumption of running only one primary. Therefore, Patroni needs to be able to "detach" from the running cluster, implementing an equivalent of the maintenance mode in Pacemaker.
 
 
 
@@ -17,16 +17,18 @@ When Patroni runs in a paused mode, it does not change the state of PostgreSQL,
 
 - For each node, the member key in DCS is updated with the current information about the cluster. This causes Patroni to run read-only queries on a member node if the member is running.
 
-- For the Postgres master with the leader lock Patroni updates the lock. If the node with the leader lock stops being the master (i.e. is demoted manually), Patroni will release the lock instead of promoting the node back.
+- For the Postgres primary with the leader lock Patroni updates the lock. If the node with the leader lock stops being the primary (i.e. is demoted manually), Patroni will release the lock instead of promoting the node back.
 
-- Manual unscheduled restart, reinitialize and manual failover are allowed. Manual failover is only allowed if the node to failover to is specified. In the paused mode, manual failover does not require a running master node.
+- Manual unscheduled restart, reinitialize and manual failover are allowed. Manual failover is only allowed if the node to failover to is specified. In the paused mode, manual failover does not require a running primary node.
 
-- If 'parallel' masters are detected by Patroni, it emits a warning, but does not demote the masters without the leader lock.
+- If 'parallel' primaries are detected by Patroni, it emits a warning, but does not demote the primary without the leader lock.
 
-- If there is no leader lock in the cluster, the running master acquires the lock. If there is more than one master node, then the first master to acquire the lock wins. If there are no masters altogether, Patroni does not try to promote any replicas. There is an exception in this rule: if there is no leader lock because the old master has demoted itself due to the manual promotion, then only the candidate node mentioned in the promotion request may take the leader lock. When the new leader lock is granted (i.e. after promoting a replica manually), Patroni makes sure the replicas that were streaming from the previous leader will switch to the new one.
+- If there is no leader lock in the cluster, the running primary acquires the lock. If there is more than one primary node, then the first primary to acquire the lock wins. If there are no primary altogether, Patroni does not try to promote any replicas. There is an exception in this rule: if there is no leader lock because the old primary has demoted itself due to the manual promotion, then only the candidate node mentioned in the promotion request may take the leader lock. When the new leader lock is granted (i.e. after promoting a replica manually), Patroni makes sure the replicas that were streaming from the previous leader will switch to the new one.
 
 - When Postgres is stopped, Patroni does not try to start it. When Patroni is stopped, it does not try to stop the Postgres instance it is managing.
 
+- Patroni will not try to remove replication slots that don't represent the other cluster member or are not listed in the configuration of the permanent slots.
+
 User guide
 ----------
 

docs/releases.rst

@@ -3,6 +3,219 @@
 Release notes
 =============
 
+Version 3.0.0
+-------------
+
+This version adds integration with `Citus <https://www.citusdata.com>`__ and makes it possible to survive temporary DCS outages without demoting primary.
+
+.. warning::
+   - Version 3.0.0 is the last release supporting Python 2.7. Upcoming release will drop support of Python versions older than 3.7.
+
+   - The RAFT support is deprecated. We will do our best to maintain it, but take neither guarantee nor responsibility for possible issues.
+
+   - This version is the first step in getting rid of the "master", in favor of "primary". Upgrading to the next major release will work reliably only if you run at least 3.0.0.
+
+
+**New features**
+
+- DCS failsafe mode (Alexander Kukushkin, Polina Bungina)
+
+  If the feature is enabled it will allow Patroni cluster to survive temporary DCS outages. You can find more details in the :ref:`documentation <dcs_failsafe_mode>`.
+
+- Citus support (Alexander, Polina, Jelte Fennema)
+
+  Patroni enables easy deployment and management of `Citus <https://www.citusdata.com>`__ clusters with HA. Please check :ref:`here <citus>` page for more information.
+
+
+**Improvements**
+
+- Suppress recurring errors when dropping unknown but active replication slots (Michael Banck)
+
+  Patroni will still write these logs, but only in DEBUG.
+
+- Run only one monitoring query per HA loop (Alexander)
+
+  It wasn't the case if synchronous replication is enabled.
+
+- Keep only latest failed data directory (William Albertus Dembo)
+
+  If bootstrap failed Patroni used to rename $PGDATA folder with timestamp suffix. From now on the suffix will be ``.failed`` and if such folder exists it is removed before renaming.
+
+- Improved check of synchronous replication connections (Alexander)
+
+  When the new host is added to the ``synchronous_standby_names`` it will be set as synchronous in DCS only when it managed to catch up with the primary in addition to ``pg_stat_replication.sync_state = 'sync'``.
+
+
+**Removed functionality**
+
+- Remove ``patronictl scaffold`` (Alexander)
+
+  The only reason for having it was a hacky way of running standby clusters.
+
+
+Version 2.1.7
+-------------
+
+**Bugfixes**
+
+- Fixed little incompatibilities with legacy python modules (Alexander Kukushkin)
+
+  They prevented from building/running Patroni on Debian buster/Ubuntu bionic.
+
+
+Version 2.1.6
+-------------
+
+**Improvements**
+
+- Fix annoying exceptions on ssl socket shutdown (Alexander Kukushkin)
+
+  The HAProxy is closing connections as soon as it got the HTTP Status code leaving no time for Patroni to properly shutdown SSL connection.
+
+- Adjust example Dockerfile for arm64 (Polina Bungina)
+
+  Remove explicit ``amd64`` and ``x86_64``, don't remove ``libnss_files.so.*``.
+
+
+**Security improvements**
+
+- Enforce ``search_path=pg_catalog`` for non-replication connections (Alexander)
+
+  Since Patroni is heavily relying on superuser connections, we want to protect it from the possible attacks carried out using user-defined functions and/or operators in ``public`` schema with the same name and signature as the corresponding objects in ``pg_catalog``. For that, ``search_path=pg_catalog`` is enforced for all connections created by Patroni (except replication connections).
+
+- Prevent passwords from being recorded in ``pg_stat_statements`` (Feike Steenbergen)
+
+  It is achieved by setting ``pg_stat_statements.track_utility=off`` when creating users.
+
+
+**Bugfixes**
+
+- Declare ``proxy_address`` as optional (Denis Laxalde)
+
+  As it is effectively a non-required option.
+
+- Improve behaviour of the insecure option (Alexander)
+
+  Ctl's ``insecure`` option didn't work properly when client certificates were used for REST API requests.
+
+- Take watchdog configuration from ``bootstrap.dcs`` when the new cluster is bootstrapped (Matt Baker)
+
+  Patroni used to initially configure watchdog with defaults when bootstrapping a new cluster rather than taking configuration used to bootstrap the DCS.
+
+- Fix the way file extensions are treated while finding executables in WIN32 (Martín Marqués)
+
+  Only add ``.exe`` to a file name if it has no extension yet.
+
+- Fix Consul TTL setup (Alexander)
+
+  We used ``ttl/2.0`` when setting the value on the HTTPClient, but forgot to multiply the current value by 2 in the class' property. It was resulting in Consul TTL off by twice.
+
+
+**Removed functionality**
+
+- Remove ``patronictl configure`` (Polina)
+
+  There is no more need for a separate ``patronictl`` config creation.
+
+
+Version 2.1.5
+-------------
+
+This version enhances compatibility with PostgreSQL 15 and declares Etcd v3 support as production ready. The Patroni on Raft remains in Beta.
+
+**New features**
+
+- Improve ``patroni --validate-config`` (Denis Laxalde)
+
+  Exit with code 1 if config is invalid and print errors to stderr.
+
+- Don't drop replication slots in pause (Alexander Kukushkin)
+
+  Patroni is automatically creating/removing physical replication slots when members are joining/leaving the cluster. In pause slots will no longer be removed.
+
+- Support the ``HEAD`` request method for monitoring endpoints (Robert Cutajar)
+
+  If used instead of ``GET`` Patroni will return only the HTTP Status Code.
+
+- Support behave tests on Windows (Alexander)
+
+  Emulate graceful Patroni shutdown (``SIGTERM``) on Windows by introduce the new REST API endpoint ``POST /sigterm``.
+
+- Introduce ``postgresql.proxy_address`` (Alexander)
+
+  It will be written to the member key in DCS as the ``proxy_url`` and could be used/useful for service discovery.
+
+
+**Stability improvements**
+
+- Call ``pg_replication_slot_advance()`` from a thread (Alexander)
+
+  On busy clusters with many logical replication slots the ``pg_replication_slot_advance()`` call was affecting the main HA loop and could result in the member key expiration.
+
+- Archive possibly missing WALs before calling ``pg_rewind`` on the old primary (Polina Bungina)
+
+  If the primary crashed and was down during considerable time, some WAL files could be missing from archive and from the new primary. There is a chance that ``pg_rewind`` could remove these WAL files from the old primary making it impossible to start it as a standby. By archiving ``ready`` WAL files we not only mitigate this problem but in general improving continues archiving experience.
+
+- Ignore ``403`` errors when trying to create Kubernetes Service (Nick Hudson, Polina)
+
+  Patroni was spamming logs by unsuccessful attempts to create the service, which in fact could already exist.
+
+- Improve liveness probe (Alexander)
+
+  The liveness problem will start failing if the heartbeat loop is running longer than `ttl` on the primary or `2*ttl` on the replica. That will allow us to use it as an alternative for :ref:`watchdog <watchdog>` on Kubernetes.
+
+- Make sure only sync node tries to grab the lock when switchover (Alexander, Polina)
+
+  Previously there was a slim chance that up-to-date async member could become the leader if the manual switchover was performed without specifying the target.
+
+- Avoid cloning while bootstrap is running (Ants Aasma)
+
+  Do not allow a create replica method that does not require a leader to be triggered while the cluster bootstrap is running.
+
+- Compatibility with kazoo-2.9.0 (Alexander)
+
+  Depending on python version the ``SequentialThreadingHandler.select()`` method may raise ``TypeError`` and ``IOError`` exceptions if ``select()`` is called on the closed socket.
+
+- Explicitly shut down SSL connection before socket shutdown (Alexander)
+
+  Not doing it resulted in ``unexpected eof while reading`` errors with OpenSSL 3.0.
+
+- Compatibility with `prettytable>=2.2.0` (Alexander)
+
+  Due to the internal API changes the cluster name header was shown on the incorrect line.
+
+
+**Bugfixes**
+
+- Handle expired token for Etcd lease_grant (monsterxx03)
+
+  In case of error get the new token and retry request.
+
+- Fix bug in the ``GET /read-only-sync`` endpoint (Alexander)
+
+  It was introduced in previous release and effectively never worked.
+
+- Handle the case when data dir storage disappeared (Alexander)
+
+  Patroni is periodically checking that the PGDATA is there and not empty, but in case of issues with storage the ``os.listdir()`` is raising the ``OSError`` exception, breaking the heart-beat loop.
+
+- Apply ``master_stop_timeout`` when waiting for user backends to close (Alexander)
+
+  Something that looks like user backend could be in fact a background worker (e.g., Citus Maintenance Daemon) that is failing to stop.
+
+- Accept ``*:<port>`` for ``postgresql.listen`` (Denis)
+
+  The ``patroni --validate-config`` was complaining about it being invalid.
+
+- Timeouts fixes in Raft (Alexander)
+
+  When Patroni or patronictl are starting they try to get Raft cluster topology from known members. These calls were made without proper timeouts.
+
+- Forcefully update consul service if token was changed (John A. Lotoski)
+
+  Not doing so results in errors "rpc error making call: rpc error making call: ACL not found".
+
+
 Version 2.1.4
 -------------
 
@@ -1108,7 +1321,7 @@ Version 1.6.1
 
 - Kill all children along with the callback process before starting the new one (Alexander Kukushkin)
 
-  Not doing so makes it hard to implement callbacks in bash and eventually can lead to the situation when two callbacks are running at the same time. 
+  Not doing so makes it hard to implement callbacks in bash and eventually can lead to the situation when two callbacks are running at the same time.
 
 - Fix 'start failed' issue (Alexander Kukushkin)
 

docs/replica_bootstrap.rst

@@ -63,7 +63,7 @@ Building replicas
 -----------------
 
 Patroni uses tried and proven ``pg_basebackup`` in order to create new replicas. One downside of it is that it requires
-a running master node. Another one is the lack of 'on-the-fly' compression for the backup data and no built-in cleanup
+a running leader node. Another one is the lack of 'on-the-fly' compression for the backup data and no built-in cleanup
 for outdated backup files. Some people prefer other backup solutions, such as ``WAL-E``, ``pgBackRest``, ``Barman`` and
 others, or simply roll their own scripts. In order to accommodate all those use-cases Patroni supports running custom
 scripts to clone a new replica. Those are configured in the ``postgresql`` configuration block:
@@ -77,7 +77,7 @@ scripts to clone a new replica. Those are configured in the ``postgresql`` confi
             command: <command name>
             keep_data: True
             no_params: True
-            no_master: 1
+            no_leader: 1
 
 example: wal_e
 
@@ -89,7 +89,7 @@ example: wal_e
             - basebackup
         wal_e:
             command: patroni_wale_restore
-            no_master: 1
+            no_leader: 1
             envdir: {{WALE_ENV_DIR}}
             use_iam: 1
         basebackup:
@@ -123,11 +123,11 @@ to execute and any custom parameters that should be passed to that command. All
 --role
     Always 'replica'
 --connstring
-    Connection string to connect to the cluster member to clone from (master or other replica). The user in the
+    Connection string to connect to the cluster member to clone from (primary or other replica). The user in the
     connection string can execute SQL and replication protocol commands.
 
-A special ``no_master`` parameter, if defined, allows Patroni to call the replica creation method even if there is no
-running master or replicas. In that case, an empty string will be passed in a connection string. This is useful for
+A special ``no_leader`` parameter, if defined, allows Patroni to call the replica creation method even if there is no
+running leader or replicas. In that case, an empty string will be passed in a connection string. This is useful for
 restoring the formerly running cluster from the binary backup.
 
 A special ``keep_data`` parameter, if defined, will instruct Patroni to  not clean PGDATA folder before calling restore.
@@ -137,7 +137,7 @@ A special ``no_params`` parameter, if defined, restricts passing parameters to c
 A ``basebackup`` method is a special case: it will be used if
 ``create_replica_methods`` is empty, although it is possible
 to list it explicitly among the ``create_replica_methods`` methods. This method initializes a new replica with the
-``pg_basebackup``, the base backup is taken from the master unless there are replicas with ``clonefrom`` tag, in which case one
+``pg_basebackup``, the base backup is taken from the leader unless there are replicas with ``clonefrom`` tag, in which case one
 of such replicas will be used as the origin for pg_basebackup. It works without any configuration; however, it is
 possible to specify a ``basebackup`` configuration section. Same rules as with the other method configuration apply,
 namely, only long (with --) options should be specified there. Not all parameters make sense, if you override a connection
@@ -176,10 +176,10 @@ Standby cluster
 ---------------
 
 Another available option is to run a "standby cluster", that contains only of
-standby nodes replicating from some remote master. This type of clusters has:
+standby nodes replicating from some remote node. This type of clusters has:
 
 * "standby leader", that behaves pretty much like a regular cluster leader,
-  except it replicates from a remote master.
+  except it replicates from a remote node.
 
 * cascade replicas, that are replicating from standby leader.
 
@@ -187,6 +187,13 @@ Standby leader holds and updates a leader lock in DCS. If the leader lock
 expires, cascade replicas will perform an election to choose another leader
 from the standbys.
 
+There is no further relationship between the standby cluster and the primary
+cluster it replicates from, in particular, they must not share the same DCS
+scope if they use the same DCS. They do not know anything else from each other
+apart from replication information. Also, the standby cluster is not being
+displayed in ``patronictl list`` or ``patronictl topology`` output on the
+primary cluster.
+
 For the sake of flexibility, you can specify methods of creating a replica and
 recovery WAL records when a cluster is in the "standby mode" by providing
 `create_replica_methods` key in `standby_cluster` section. It is distinct from
@@ -212,4 +219,9 @@ in a patroni configuration:
 Note, that these options will be applied only once during cluster bootstrap,
 and the only way to change them afterwards is through DCS.
 
+Patroni expects to find `postgresql.conf` or `postgresql.conf.backup` in PGDATA
+of the remote primary and will not start if it does not find it after a
+basebackup. If the remote primary keeps its `postgresql.conf` elsewhere, it is
+your responsibility to copy it to PGDATA.
+
 If you use replication slots on the standby cluster, you must also create the corresponding replication slot on the primary cluster.  It will not be done automatically by the standby cluster implementation.  You can use Patroni's permanent replication slots feature on the primary cluster to maintain a replication slot with the same name as ``primary_slot_name``, or its default value if ``primary_slot_name`` is not provided.

docs/replication_modes.rst

@@ -13,7 +13,7 @@ In asynchronous mode the cluster is allowed to lose some committed transactions
 
 The amount of transactions that can be lost is controlled via ``maximum_lag_on_failover`` parameter. Because the primary transaction log position is not sampled in real time, in reality the amount of lost data on failover is worst case bounded by  ``maximum_lag_on_failover`` bytes of transaction log plus the amount that is written in the last ``ttl`` seconds (``loop_wait``/2 seconds in the average case). However typical steady state replication delay is well under a second.
 
-By default, when running leader elections, Patroni does not take into account the current timeline of replicas, what in some cases could be undesirable behavior. You can prevent the node not having the same timeline as a former master become the new leader by changing the value of ``check_timeline`` parameter to ``true``.
+By default, when running leader elections, Patroni does not take into account the current timeline of replicas, what in some cases could be undesirable behavior. You can prevent the node not having the same timeline as a former primary become the new leader by changing the value of ``check_timeline`` parameter to ``true``.
 
 PostgreSQL synchronous replication
 ----------------------------------

docs/rest_api.rst

@@ -7,12 +7,11 @@ Patroni has a rich REST API, which is used by Patroni itself during the leader r
 
 Health check endpoints
 ----------------------
-For all health check ``GET`` requests Patroni returns a JSON document with the status of the node, along with the HTTP status code. If you don't want or don't need the JSON document, you might consider using the ``OPTIONS`` method instead of ``GET``.
+For all health check ``GET`` requests Patroni returns a JSON document with the status of the node, along with the HTTP status code. If you don't want or don't need the JSON document, you might consider using the ``HEAD`` or ``OPTIONS`` method instead of ``GET``.
 
 - The following requests to Patroni REST API will return HTTP status code **200** only when the Patroni node is running as the primary with leader lock:
 
   - ``GET /``
-  - ``GET /master``
   - ``GET /primary``
   - ``GET /read-write``
 
@@ -33,7 +32,6 @@ For all health check ``GET`` requests Patroni returns a JSON document with the s
 
  In the following requests, since we are checking for the leader or standby-leader status, Patroni doesn't apply any of the user defined tags and they will be ignored.
   - ``GET /?tag_key1=value1&tag_key2=value2``
-  - ``GET /master?tag_key1=value1&tag_key2=value2``
   - ``GET /leader?tag_key1=value1&tag_key2=value2``
   - ``GET /primary?tag_key1=value1&tag_key2=value2``
   - ``GET /read-write?tag_key1=value1&tag_key2=value2``
@@ -58,7 +56,7 @@ For all health check ``GET`` requests Patroni returns a JSON document with the s
 
 - ``GET /health``: returns HTTP status code **200** only when PostgreSQL is up and running.
 
-- ``GET /liveness``: always returns HTTP status code **200** what only indicates that Patroni is running. Could be used for ``livenessProbe``.
+- ``GET /liveness``: returns HTTP status code **200** if Patroni heartbeat loop is properly running and **503** if the last run was more than ``ttl`` seconds ago on the primary or ``2*ttl`` on the replica. Could be used for ``livenessProbe``.
 
 - ``GET /readiness``: returns HTTP status code **200** when the Patroni node is running as the leader or when PostgreSQL is up and running. The endpoint could be used for ``readinessProbe`` when it is not possible to use Kubernetes endpoints for leader elections (OpenShift).
 
@@ -368,7 +366,7 @@ Restart endpoint
   - **restart_pending**: boolean, if set to ``true`` Patroni will restart PostgreSQL only when restart is pending in order to apply some changes in the PostgreSQL config.
   - **role**: perform restart only if the current role of the node matches with the role from the POST request.
   - **postgres_version**: perform restart only if the current version of postgres is smaller than specified in the POST request.
-  - **timeout**: how long we should wait before PostgreSQL starts accepting connections. Overrides ``master_start_timeout``.
+  - **timeout**: how long we should wait before PostgreSQL starts accepting connections. Overrides ``primary_start_timeout``.
   - **schedule**: timestamp with time zone, schedule the restart somewhere in the future.
 
 - ``DELETE /restart``: delete the scheduled restart

docs/security.rst

@@ -9,11 +9,11 @@ A Patroni cluster has two interfaces to be protected from unauthorized access: t
 Protecting DCS
 ==============
 
-Patroni and patronictl both store and retrieve data to/from the DCS. 
+Patroni and patronictl both store and retrieve data to/from the DCS.
 
 Despite DCS doesn't contain any sensitive information, it allows changing some of Patroni/Postgres configuration. Therefore the very first thing that should be protected is DCS itself.
 
-The details of protection depend on the type of DCS used. The authentication and encryption parameters (tokens/basic-auth/client certificates) for the supported types of DCS are covered in :ref:`SETTINGS <bootstrap_settings>` 
+The details of protection depend on the type of DCS used. The authentication and encryption parameters (tokens/basic-auth/client certificates) for the supported types of DCS are covered in :ref:`SETTINGS <bootstrap_settings>`
 
 The general recommendation is to enable TLS for all DCS communication.
 
@@ -22,7 +22,7 @@ Protecting the REST API
 
 Protecting the REST API is a more complicated task.
 
-The Patroni REST API is used by Patroni itself during the leader race, by the ``patronictl`` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring. 
+The Patroni REST API is used by Patroni itself during the leader race, by the ``patronictl`` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring.
 
 From the point of view of security, REST API contains safe (``GET`` requests, only retrieve information) and unsafe (``PUT``, ``POST``, ``PATCH`` and ``DELETE`` requests, change the state of nodes) endpoints.
 

docs/watchdog.rst

@@ -3,7 +3,7 @@
 Watchdog support
 ================
 
-Having multiple PostgreSQL servers running as master can result in transactions lost due to diverging timelines. This situation is also called a split-brain problem. To avoid split-brain Patroni needs to ensure PostgreSQL will not accept any transaction commits after leader key expires in the DCS. Under normal circumstances Patroni will try to achieve this by stopping PostgreSQL when leader lock update fails for any reason. However, this may fail to happen due to various reasons:
+Having multiple PostgreSQL servers running as primary can result in transactions lost due to diverging timelines. This situation is also called a split-brain problem. To avoid split-brain Patroni needs to ensure PostgreSQL will not accept any transaction commits after leader key expires in the DCS. Under normal circumstances Patroni will try to achieve this by stopping PostgreSQL when leader lock update fails for any reason. However, this may fail to happen due to various reasons:
 
 - Patroni has crashed due to a bug, out-of-memory condition or by being accidentally killed by a system administrator.
 
@@ -13,7 +13,7 @@ Having multiple PostgreSQL servers running as master can result in transactions
 
 To guarantee correct behavior under these conditions Patroni supports watchdog devices. Watchdog devices are software or hardware mechanisms that will reset the whole system when they do not get a keepalive heartbeat within a specified timeframe. This adds an additional layer of fail safe in case usual Patroni split-brain protection mechanisms fail.
 
-Patroni will try to activate the watchdog before promoting PostgreSQL to master. If watchdog activation fails and watchdog mode is ``required`` then the node will refuse to become master. When deciding to participate in leader election Patroni will also check that watchdog configuration will allow it to become leader at all. After demoting PostgreSQL (for example due to a manual failover) Patroni will disable the watchdog again. Watchdog will also be disabled while Patroni is in paused state.
+Patroni will try to activate the watchdog before promoting PostgreSQL to primary. If watchdog activation fails and watchdog mode is ``required`` then the node will refuse to become leader. When deciding to participate in leader election Patroni will also check that watchdog configuration will allow it to become leader at all. After demoting PostgreSQL (for example due to a manual failover) Patroni will disable the watchdog again. Watchdog will also be disabled while Patroni is in paused state.
 
 By default Patroni will set up the watchdog to expire 5 seconds before TTL expires. With the default setup of ``loop_wait=10`` and ``ttl=30`` this gives HA loop at least 15 seconds (``ttl`` - ``safety_margin`` - ``loop_wait``) to complete before the system gets forcefully reset. By default accessing DCS is configured to time out after 10 seconds. This means that when DCS is unavailable, for example due to network issues, Patroni and PostgreSQL will have at least 5 seconds (``ttl`` - ``safety_margin`` - ``loop_wait`` - ``retry_timeout``) to come to a state where all client connections are terminated.
 

extras/confd/conf.d/haproxy.toml

@@ -9,5 +9,5 @@ check_cmd = "/usr/sbin/haproxy -c -f {{ .src }}"
 reload_cmd = "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)"
 
 keys = [
-    "/members/",
+    "/",
 ]

extras/confd/templates/haproxy-citus.tmpl

@@ -0,0 +1,32 @@
+global
+	maxconn 100
+
+defaults
+	log	global
+	mode	tcp
+	retries 2
+	timeout client 30m
+	timeout connect 4s
+	timeout server 30m
+	timeout check 5s
+
+listen stats
+	mode http
+	bind *:7000
+	stats enable
+	stats uri /
+
+listen coordinator
+	bind *:5000
+	option httpchk HEAD /primary
+	http-check expect status 200
+	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
+{{range gets "/0/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check check-ssl port {{index (split (index (split $data.api_url "/") 2) ":") 1}} verify required ca-file /etc/ssl/certs/ssl-cert-snakeoil.pem crt /etc/ssl/private/ssl-cert-snakeoil.crt
+{{end}}
+listen workers
+	bind *:5001
+	option httpchk HEAD /primary
+	http-check expect status 200
+	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
+{{range gets "/*/members/*"}}{{$group := index (split .Key "/") 1}}{{if ne $group "0"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check check-ssl port {{index (split (index (split $data.api_url "/") 2) ":") 1}} verify required ca-file /etc/ssl/certs/ssl-cert-snakeoil.pem crt /etc/ssl/private/ssl-cert-snakeoil.crt
+{{end}}{{end}}

extras/confd/templates/haproxy.tmpl

@@ -16,16 +16,16 @@ listen stats
 	stats enable
 	stats uri /
 
-listen master
+listen primary
 	bind *:5000
-	option httpchk OPTIONS /master
+	option httpchk HEAD /primary
 	http-check expect status 200
 	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
 {{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
 {{end}}
 listen replicas
 	bind *:5001
-	option httpchk OPTIONS /replica
+	option httpchk HEAD /replica
 	http-check expect status 200
 	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
 {{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}

extras/startup-scripts/README.md

@@ -1,6 +1,6 @@
 # startup scripts for Patroni
 
-This directory contains sample startup scripts for various OSes 
+This directory contains sample startup scripts for various OSes
 and management tools for Patroni.
 
 Scripts supplied:
@@ -10,7 +10,7 @@ Scripts supplied:
 Upstart job for Ubuntu 12.04 or 14.04.  Requires Upstart > 1.4. Intended for systems where Patroni has been installed on a base system, rather than in Docker.
 
 ### patroni.service
-Systemd service file, to be copied to /etc/systemd/system/patroni.service, tested on Centos 7.1 with Patroni installed from pip. 
+Systemd service file, to be copied to /etc/systemd/system/patroni.service, tested on Centos 7.1 with Patroni installed from pip.
 
 ### patroni
 Init.d service file for Debian-like distributions. Copy it to /etc/init.d/, make executable:

extras/startup-scripts/patroni.service

@@ -14,7 +14,7 @@ Group=postgres
 # Read in configuration file if it exists, otherwise proceed
 EnvironmentFile=-/etc/patroni_env.conf
 
-# the default is the user's home directory, and if you want to change it, you must provide an absolute path.
+# The default is the user's home directory, and if you want to change it, you must provide an absolute path.
 # WorkingDirectory=/home/sameuser
 
 # Where to send early-startup messages from the server
@@ -32,14 +32,14 @@ ExecStart=/bin/patroni /etc/patroni.yml
 # Send HUP to reload from patroni.yml
 ExecReload=/bin/kill -s HUP $MAINPID
 
-# only kill the patroni process, not it's children, so it will gracefully stop postgres
+# Only kill the patroni process, not it's children, so it will gracefully stop postgres
 KillMode=process
 
 # Give a reasonable amount of time for the server to start up/shut down
 TimeoutSec=30
 
-# Do not restart the service if it crashes, we want to manually inspect database on failure
-Restart=no
+# Restart the service if it crashed
+Restart=on-failure
 
 [Install]
 WantedBy=multi-user.target

features/basic_replication.feature

@@ -5,7 +5,7 @@ Feature: basic replication
     Given I start postgres0
     Then postgres0 is a leader after 10 seconds
     And there is a non empty initialize key in DCS after 15 seconds
-    When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 2, "synchronous_mode": true}
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "synchronous_mode": true}
     Then I receive a response code 200
     When I start postgres1
     And I configure and start postgres2 with a tag replicatefrom postgres0
@@ -21,12 +21,9 @@ Feature: basic replication
     And I shut down postgres1
     Then "sync" key in DCS has sync_standby=postgres2 after 10 seconds
     When I start postgres1
-    And "members/postgres1" key in DCS has state=running after 10 seconds
-    And I sleep for 2 seconds
-    When I issue a GET request to http://127.0.0.1:8010/sync
-    Then I receive a response code 200
-    When I issue a GET request to http://127.0.0.1:8009/async
-    Then I receive a response code 200
+    Then "members/postgres1" key in DCS has state=running after 10 seconds
+    And Status code on GET http://127.0.0.1:8010/sync is 200 after 3 seconds
+    And Status code on GET http://127.0.0.1:8009/async is 200 after 3 seconds
 
   Scenario: check stuck sync replica
     Given I issue a PATCH request to http://127.0.0.1:8008/config with {"pause": true, "maximum_lag_on_syncnode": 15000000, "postgresql": {"parameters": {"synchronous_commit": "remote_apply"}}}
@@ -38,11 +35,8 @@ Feature: basic replication
     And I load data on postgres0
     Then "sync" key in DCS has sync_standby=postgres1 after 15 seconds
     And I resume wal replay on postgres2
-    And I sleep for 2 seconds
-    And I issue a GET request to http://127.0.0.1:8009/sync
-    Then I receive a response code 200
-    When I issue a GET request to http://127.0.0.1:8010/async
-    Then I receive a response code 200
+    And Status code on GET http://127.0.0.1:8009/sync is 200 after 3 seconds
+    And Status code on GET http://127.0.0.1:8010/async is 200 after 3 seconds
     When I issue a PATCH request to http://127.0.0.1:8008/config with {"pause": null, "maximum_lag_on_syncnode": -1, "postgresql": {"parameters": {"synchronous_commit": "on"}}}
     Then I receive a response code 200
     And I drop table on postgres0
@@ -50,23 +44,17 @@ Feature: basic replication
   Scenario: check multi sync replication
     Given I issue a PATCH request to http://127.0.0.1:8008/config with {"synchronous_node_count": 2}
     Then I receive a response code 200
-    And I sleep for 10 seconds
-    Then "sync" key in DCS has sync_standby=postgres1,postgres2 after 5 seconds
-    When I issue a GET request to http://127.0.0.1:8010/sync
-    Then I receive a response code 200
-    When I issue a GET request to http://127.0.0.1:8009/sync
-    Then I receive a response code 200
+    Then "sync" key in DCS has sync_standby=postgres1,postgres2 after 10 seconds
+    And Status code on GET http://127.0.0.1:8010/sync is 200 after 3 seconds
+    And Status code on GET http://127.0.0.1:8009/sync is 200 after 3 seconds
     When I issue a PATCH request to http://127.0.0.1:8008/config with {"synchronous_node_count": 1}
     Then I receive a response code 200
     And I shut down postgres1
-    And I sleep for 10 seconds
     Then "sync" key in DCS has sync_standby=postgres2 after 10 seconds
     When I start postgres1
-    And "members/postgres1" key in DCS has state=running after 10 seconds
-    When I issue a GET request to http://127.0.0.1:8010/sync
-    Then I receive a response code 200
-    When I issue a GET request to http://127.0.0.1:8009/async
-    Then I receive a response code 200
+    Then "members/postgres1" key in DCS has state=running after 10 seconds
+    And Status code on GET http://127.0.0.1:8010/sync is 200 after 3 seconds
+    And Status code on GET http://127.0.0.1:8009/async is 200 after 3 seconds
 
   Scenario: check the basic failover in synchronous mode
     Given I run patronictl.py pause batman
@@ -88,7 +76,7 @@ Feature: basic replication
     Then postgres1 is a leader after 10 seconds
     And postgres1 role is the primary after 10 seconds
 
-  Scenario: check rejoin of the former master with pg_rewind
+  Scenario: check rejoin of the former primary with pg_rewind
     Given I add the table splitbrain to postgres0
     And I start postgres0
     Then postgres0 role is the secondary after 20 seconds

features/citus.feature

@@ -0,0 +1,72 @@
+Feature: citus
+  We should check that coordinator discovers and registers workers and clients don't have errors when worker cluster switches over
+
+  Scenario: check that worker cluster is registered in the coordinator
+    Given I start postgres0 in citus group 0
+    And I start postgres2 in citus group 1
+    Then postgres0 is a leader in a group 0 after 10 seconds
+    And postgres2 is a leader in a group 1 after 10 seconds
+    When I start postgres1 in citus group 0
+    And I start postgres3 in citus group 1
+    Then replication works from postgres0 to postgres1 after 15 seconds
+    Then replication works from postgres2 to postgres3 after 15 seconds
+    And postgres0 is registered in the coordinator postgres0 as the worker in group 0
+    And postgres2 is registered in the coordinator postgres0 as the worker in group 1
+
+  Scenario: coordinator failover updates pg_dist_node
+    Given I run patronictl.py failover batman --group 0 --candidate postgres1 --force
+    Then postgres1 role is the primary after 10 seconds
+    And replication works from postgres1 to postgres0 after 15 seconds
+    And "sync" key in a group 0 in DCS has sync_standby=postgres0 after 15 seconds
+    And postgres1 is registered in the coordinator postgres1 as the worker in group 0
+    When I run patronictl.py failover batman --group 0 --candidate postgres0 --force
+    Then postgres0 role is the primary after 10 seconds
+    And replication works from postgres0 to postgres1 after 15 seconds
+    And "sync" key in a group 0 in DCS has sync_standby=postgres1 after 15 seconds
+    And postgres0 is registered in the coordinator postgres0 as the worker in group 0
+
+  Scenario: worker switchover doesn't break client queries on the coordinator
+    Given I create a distributed table on postgres0
+    And I start a thread inserting data on postgres0
+    When I run patronictl.py switchover batman --group 1 --force
+    Then I receive a response returncode 0
+    And postgres3 role is the primary after 10 seconds
+    And replication works from postgres3 to postgres2 after 15 seconds
+    And "sync" key in a group 1 in DCS has sync_standby=postgres2 after 15 seconds
+    And postgres3 is registered in the coordinator postgres0 as the worker in group 1
+    And a thread is still alive
+    When I run patronictl.py switchover batman --group 1 --force
+    Then I receive a response returncode 0
+    And postgres2 role is the primary after 10 seconds
+    And replication works from postgres2 to postgres3 after 15 seconds
+    And "sync" key in a group 1 in DCS has sync_standby=postgres3 after 15 seconds
+    And postgres2 is registered in the coordinator postgres0 as the worker in group 1
+    And a thread is still alive
+    When I stop a thread
+    Then a distributed table on postgres0 has expected rows
+
+  Scenario: worker primary restart doesn't break client queries on the coordinator
+    Given I cleanup a distributed table on postgres0
+    And I start a thread inserting data on postgres0
+    When I run patronictl.py restart batman postgres2 --group 1 --force
+    Then I receive a response returncode 0
+    And postgres2 role is the primary after 10 seconds
+    And replication works from postgres2 to postgres3 after 15 seconds
+    And postgres2 is registered in the coordinator postgres0 as the worker in group 1
+    And a thread is still alive
+    When I stop a thread
+    Then a distributed table on postgres0 has expected rows
+
+  Scenario: check that in-flight transaction is rolled back after timeout when other workers need to change pg_dist_node
+    Given I start postgres4 in citus group 2
+    Then postgres4 is a leader in a group 2 after 10 seconds
+    And "members/postgres4" key in a group 2 in DCS has role=master after 3 seconds
+    When I run patronictl.py edit-config batman --group 2 -s ttl=20 --force
+    Then I receive a response returncode 0
+    And I receive a response output "+ttl: 20"
+    When I sleep for 2 seconds
+    Then postgres4 is registered in the coordinator postgres0 as the worker in group 2
+    When I shut down postgres4
+    Then There is a transaction in progress on postgres0 changing pg_dist_node
+    When I run patronictl.py restart batman postgres2 --group 1 --force
+    Then a transaction finishes in 20 seconds

features/dcs_failsafe_mode.feature

@@ -0,0 +1,85 @@
+Feature: dcs failsafe mode
+  We should check the basic dcs failsafe mode functioning
+
+  Scenario: check failsafe mode can be successfully enabled
+    Given I start postgres0
+    And postgres0 is a leader after 10 seconds
+    And I sleep for 3 seconds
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"loop_wait": 2, "ttl": 20, "retry_timeout": 5, "failsafe_mode": true}
+    Then I receive a response code 200
+    And Response on GET http://127.0.0.1:8008/failsafe contains postgres0 after 10 seconds
+    When I issue a GET request to http://127.0.0.1:8008/failsafe
+    Then I receive a response code 200
+    And I receive a response postgres0 http://127.0.0.1:8008/patroni
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"wal_level": "logical"}}}
+    Then I receive a response code 200
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"slots": {"dcs_slot_0": {"type": "logical", "database": "postgres", "plugin": "test_decoding"}}}
+    Then I receive a response code 200
+
+  @dcs-failsafe
+  Scenario: check one-node cluster is functioning while DCS is down
+    Given DCS is down
+    Then Response on GET http://127.0.0.1:8008/primary contains failsafe_mode_is_active after 12 seconds
+    And postgres0 role is the primary after 10 seconds
+
+  @dcs-failsafe
+  Scenario: check new replica isn't promoted when leader is down and DCS is up
+    Given DCS is up
+    When I do a backup of postgres0
+    And I shut down postgres0
+    When I start postgres1 in a cluster batman from backup with no_leader
+    And I sleep for 2 seconds
+    Then postgres1 role is the replica after 12 seconds
+
+  Scenario: check leader and replica are both in /failsafe key after leader is back
+    Given I start postgres0
+    And I start postgres1
+    Then "members/postgres0" key in DCS has state=running after 10 seconds
+    And "members/postgres1" key in DCS has state=running after 2 seconds
+    And Response on GET http://127.0.0.1:8009/failsafe contains postgres1 after 10 seconds
+    When I issue a GET request to http://127.0.0.1:8009/failsafe
+    Then I receive a response code 200
+    And I receive a response postgres0 http://127.0.0.1:8008/patroni
+    And I receive a response postgres1 http://127.0.0.1:8009/patroni
+
+  @dcs-failsafe
+  @slot-advance
+  Scenario: check leader and replica are functioning while DCS is down
+    Given logical slot dcs_slot_0 is in sync between postgres0 and postgres1 after 10 seconds
+    And DCS is down
+    Then Response on GET http://127.0.0.1:8008/primary contains failsafe_mode_is_active after 12 seconds
+    Then postgres0 role is the primary after 10 seconds
+    And postgres1 role is the replica after 2 seconds
+    And replication works from postgres0 to postgres1 after 10 seconds
+    And I get all changes from logical slot dcs_slot_0 on postgres0
+    And logical slot dcs_slot_0 is in sync between postgres0 and postgres1 after 20 seconds
+
+  @dcs-failsafe
+  Scenario: check primary is demoted when one replica is shut down and DCS is down
+    Given DCS is down
+    And I kill postgres1
+    And I kill postmaster on postgres1
+    And I sleep for 2 seconds
+    Then postgres0 role is the replica after 12 seconds
+
+  @dcs-failsafe
+  Scenario: check known replica is promoted when leader is down and DCS is up
+    Given I shut down postgres0
+    And DCS is up
+    When I start postgres1
+    Then "members/postgres1" key in DCS has state=running after 10 seconds
+    And postgres1 role is the primary after 25 seconds
+
+  @dcs-failsafe
+  Scenario: check three-node cluster is functioning while DCS is down
+    Given I start postgres0
+    And I start postgres2
+    Then "members/postgres2" key in DCS has state=running after 10 seconds
+    And "members/postgres0" key in DCS has state=running after 20 seconds
+    And Response on GET http://127.0.0.1:8008/failsafe contains postgres2 after 10 seconds
+    And replication works from postgres1 to postgres0 after 10 seconds
+    Given DCS is down
+    Then Response on GET http://127.0.0.1:8008/primary contains failsafe_mode_is_active after 12 seconds
+    Then postgres1 role is the primary after 10 seconds
+    And postgres0 role is the replica after 2 seconds
+    And postgres2 role is the replica after 2 seconds

features/environment.py

@@ -1,7 +1,10 @@
 import abc
 import datetime
+import glob
 import os
 import json
+import psutil
+import re
 import shutil
 import signal
 import six
@@ -14,6 +17,7 @@ import yaml
 
 import patroni.psycopg as psycopg
 
+from patroni.request import PatroniRequest
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 
 
@@ -99,6 +103,7 @@ class PatroniController(AbstractController):
             self.watchdog = None
 
         self._scope = (custom_config or {}).get('scope', 'batman')
+        self._citus_group = (custom_config or {}).get('citus', {}).get('group')
         self._config = self._make_patroni_test_config(name, custom_config)
         self._closables = []
 
@@ -138,12 +143,24 @@ class PatroniController(AbstractController):
     def _start(self):
         if self.watchdog:
             self.watchdog.start()
+        env = os.environ.copy()
         if isinstance(self._context.dcs_ctl, KubernetesController):
-            self._context.dcs_ctl.create_pod(self._name[8:], self._scope)
-            os.environ['PATRONI_KUBERNETES_POD_IP'] = '10.0.0.' + self._name[-1]
-        return subprocess.Popen([sys.executable, '-m', 'coverage', 'run',
-                                '--source=patroni', '-p', 'patroni.py', self._config],
-                                stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
+            self._context.dcs_ctl.create_pod(self._name[8:], self._scope, self._citus_group)
+            env['PATRONI_KUBERNETES_POD_IP'] = '10.0.0.' + self._name[-1]
+        if os.name == 'nt':
+            env['BEHAVE_DEBUG'] = 'true'
+        patroni = subprocess.Popen([sys.executable, '-m', 'coverage', 'run',
+                                   '--source=patroni', '-p', 'patroni.py', self._config], env=env,
+                                   stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
+        if os.name == 'nt':
+            patroni.terminate = self.terminate
+        return patroni
+
+    def terminate(self):
+        try:
+            self._context.request_executor.request('POST', self._restapi_url + '/sigterm')
+        except Exception:
+            pass
 
     def stop(self, kill=False, timeout=15, postgres=False):
         if postgres:
@@ -164,29 +181,59 @@ class PatroniController(AbstractController):
         patroni_config_name = self.PATRONI_CONFIG.format(name)
         patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
 
-        with open(patroni_config_name) as f:
+        with open('postgres0.yml') as f:
             config = yaml.safe_load(f)
             config.pop('etcd', None)
 
         raft_port = os.environ.get('RAFT_PORT')
-        if raft_port:
+        # If patroni_raft_controller is suspended two Patroni members is enough to get a quorum,
+        # therefore we don't want Patroni to join as a voting member when testing dcs_failsafe_mode.
+        if raft_port and not self._output_dir.endswith('dcs_failsafe_mode'):
             os.environ['RAFT_PORT'] = str(int(raft_port) + 1)
             config['raft'] = {'data_dir': self._output_dir, 'self_addr': 'localhost:' + os.environ['RAFT_PORT']}
 
-        host = config['postgresql']['listen'].split(':')[0]
+        host = config['restapi']['listen'].rsplit(':', 1)[0]
+        config['restapi']['listen'] = config['restapi']['connect_address'] = '{0}:{1}'.format(host, 8008+int(name[-1]))
 
+        host = config['postgresql']['listen'].rsplit(':', 1)[0]
         config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT)
 
         config['name'] = name
-        config['postgresql']['data_dir'] = self._data_dir
+        config['postgresql']['data_dir'] = self._data_dir.replace('\\', '/')
         config['postgresql']['basebackup'] = [{'checkpoint': 'fast'}]
         config['postgresql']['use_unix_socket'] = os.name != 'nt'  # windows doesn't yet support unix-domain sockets
-        config['postgresql']['use_unix_socket_repl'] = os.name != 'nt'  # windows doesn't yet support unix-domain sockets
-        config['postgresql']['pgpass'] = os.path.join(tempfile.gettempdir(), 'pgpass_' + name)
+        config['postgresql']['use_unix_socket_repl'] = os.name != 'nt'
+        config['postgresql']['pgpass'] = os.path.join(tempfile.gettempdir(), 'pgpass_' + name).replace('\\', '/')
         config['postgresql']['parameters'].update({
-            'logging_collector': 'on', 'log_destination': 'csvlog', 'log_directory': self._output_dir,
+            'logging_collector': 'on', 'log_destination': 'csvlog',
+            'log_directory': self._output_dir.replace('\\', '/'),
             'log_filename': name + '.log', 'log_statement': 'all', 'log_min_messages': 'debug1',
-            'unix_socket_directories': tempfile.gettempdir()})
+            'shared_buffers': '1MB', 'unix_socket_directories': tempfile.gettempdir().replace('\\', '/')})
+        config['postgresql']['pg_hba'] = [
+            'local all all trust',
+            'local replication all trust',
+            'host replication replicator all md5',
+            'host all all all md5'
+        ]
+
+        if self._context.postgres_supports_ssl and self._context.certfile:
+            config['postgresql']['parameters'].update({
+                'ssl': 'on',
+                'ssl_ca_file': self._context.certfile.replace('\\', '/'),
+                'ssl_cert_file': self._context.certfile.replace('\\', '/'),
+                'ssl_key_file': self._context.keyfile.replace('\\', '/')
+            })
+            for user in config['postgresql'].get('authentication').keys():
+                config['postgresql'].get('authentication', {}).get(user, {}).update({
+                    'sslmode': 'verify-ca',
+                    'sslrootcert': self._context.certfile,
+                    'sslcert': self._context.certfile,
+                    'sslkey': self._context.keyfile
+                })
+            for i, line in enumerate(list(config['postgresql']['pg_hba'])):
+                if line.endswith('md5'):
+                    # we want to verify client cert first and than password
+                    config['postgresql']['pg_hba'][i] = 'hostssl' + line[4:] + ' clientcert=verify-ca'
 
         if 'bootstrap' in config:
             config['bootstrap']['post_bootstrap'] = 'psql -w -c "SELECT 1"'
@@ -197,26 +244,43 @@ class PatroniController(AbstractController):
             self.recursive_update(config, custom_config)
 
         self.recursive_update(config, {
-            'bootstrap': {'dcs': {'postgresql': {'parameters': {'wal_keep_segments': 100}}}}})
+            'bootstrap': {
+                'dcs': {
+                    'loop_wait': 2,
+                    'postgresql': {
+                        'parameters': {
+                            'wal_keep_segments': 100,
+                            'archive_mode': 'on',
+                            'archive_command': (PatroniPoolController.ARCHIVE_RESTORE_SCRIPT +
+                                                ' --mode archive ' +
+                                                '--dirname {} --filename %f --pathname %p').format(
+                                                    os.path.join(self._work_directory, 'data', 'wal_archive'))
+                        }
+                    }
+                }
+            }
+        })
         if config['postgresql'].get('callbacks', {}).get('on_role_change'):
             config['postgresql']['callbacks']['on_role_change'] += ' ' + str(self.__PORT)
 
         with open(patroni_config_path, 'w') as f:
             yaml.safe_dump(config, f, default_flow_style=False)
 
-        user = config['postgresql'].get('authentication', config['postgresql']).get('superuser', {})
-        self._connkwargs = {k: user[n] for n, k in [('username', 'user'), ('password', 'password')] if n in user}
-        self._connkwargs.update({'host': host, 'port': self.__PORT, 'dbname': 'postgres'})
+        self._connkwargs = config['postgresql'].get('authentication', config['postgresql']).get('superuser', {})
+        self._connkwargs.update({'host': host, 'port': self.__PORT, 'dbname': 'postgres',
+                                 'user': self._connkwargs.pop('username', None)})
 
         self._replication = config['postgresql'].get('authentication', config['postgresql']).get('replication', {})
-        self._replication.update({'host': host, 'port': self.__PORT, 'dbname': 'postgres'})
+        self._replication.update({'host': host, 'port': self.__PORT, 'user': self._replication.pop('username', None)})
+        self._restapi_url = 'http://{0}'.format(config['restapi']['connect_address'])
+        if self._context.certfile:
+            self._restapi_url = self._restapi_url.replace('http://', 'https://')
 
         return patroni_config_path
 
     def _connection(self):
         if not self._conn or self._conn.closed != 0:
             self._conn = psycopg.connect(**self._connkwargs)
-            self._conn.autocommit = True
         return self._conn
 
     def _cursor(self):
@@ -269,7 +333,10 @@ class PatroniController(AbstractController):
 
     @property
     def backup_source(self):
-        return 'postgres://{username}:{password}@{host}:{port}/{dbname}'.format(**self._replication)
+        def escape(value):
+            return re.sub(r'([\'\\ ])', r'\\\1', str(value))
+
+        return ' '.join('{0}={1}'.format(k, escape(v)) for k, v in self._replication.items())
 
     def backup(self, dest=os.path.join('data', 'basebackup')):
         subprocess.call(PatroniPoolController.BACKUP_SCRIPT + ['--walmethod=none',
@@ -308,6 +375,7 @@ class AbstractDcsController(AbstractController):
 
     def __init__(self, context, mktemp=True):
         work_directory = mktemp and tempfile.mkdtemp() or None
+        self._paused = False
         super(AbstractDcsController, self).__init__(context, self.name(), work_directory, context.pctl.output_dir)
 
     def _is_accessible(self):
@@ -319,11 +387,22 @@ class AbstractDcsController(AbstractController):
         if self._work_directory:
             shutil.rmtree(self._work_directory)
 
-    def path(self, key=None, scope='batman'):
-        return self._CLUSTER_NODE.format(scope) + (key and '/' + key or '')
+    def path(self, key=None, scope='batman', group=None):
+        citus_group = '/{0}'.format(group) if group is not None else ''
+        return self._CLUSTER_NODE.format(scope) + citus_group + (key and '/' + key or '')
+
+    def start_outage(self):
+        if not self._paused and self._handle:
+            self._handle.suspend()
+            self._paused = True
+
+    def stop_outage(self):
+        if self._paused and self._handle:
+            self._handle.resume()
+            self._paused = False
 
     @abc.abstractmethod
-    def query(self, key, scope='batman'):
+    def query(self, key, scope='batman', group=None):
         """ query for a value of a given key """
 
     @abc.abstractmethod
@@ -357,8 +436,8 @@ class ConsulController(AbstractDcsController):
         self._config_file = self._work_directory + '.json'
         with open(self._config_file, 'wb') as f:
             f.write(b'{"session_ttl_min":"5s","server":true,"bootstrap":true,"advertise_addr":"127.0.0.1"}')
-        return subprocess.Popen(['consul', 'agent', '-config-file', self._config_file, '-data-dir',
-                                 self._work_directory], stdout=self._log, stderr=subprocess.STDOUT)
+        return psutil.Popen(['consul', 'agent', '-config-file', self._config_file, '-data-dir',
+                             self._work_directory], stdout=self._log, stderr=subprocess.STDOUT)
 
     def stop(self, kill=False, timeout=15):
         super(ConsulController, self).stop(kill=kill, timeout=timeout)
@@ -371,11 +450,11 @@ class ConsulController(AbstractDcsController):
         except Exception:
             return False
 
-    def path(self, key=None, scope='batman'):
-        return super(ConsulController, self).path(key, scope)[1:]
+    def path(self, key=None, scope='batman', group=None):
+        return super(ConsulController, self).path(key, scope, group)[1:]
 
-    def query(self, key, scope='batman'):
-        _, value = self._client.kv.get(self.path(key, scope))
+    def query(self, key, scope='batman', group=None):
+        _, value = self._client.kv.get(self.path(key, scope, group))
         return value and value['Value'].decode('utf-8')
 
     def cleanup_service_tree(self):
@@ -394,8 +473,8 @@ class AbstractEtcdController(AbstractDcsController):
         self._client_cls = client_cls
 
     def _start(self):
-        return subprocess.Popen(["etcd", "--debug", "--data-dir", self._work_directory],
-                                stdout=self._log, stderr=subprocess.STDOUT)
+        return psutil.Popen(["etcd", "--enable-v2=true", "--data-dir", self._work_directory],
+                            stdout=self._log, stderr=subprocess.STDOUT)
 
     def _is_running(self):
         from patroni.dcs.etcd import DnsCachingResolver
@@ -415,10 +494,10 @@ class EtcdController(AbstractEtcdController):
         super(EtcdController, self).__init__(context, EtcdClient)
         os.environ['PATRONI_ETCD_HOST'] = 'localhost:2379'
 
-    def query(self, key, scope='batman'):
+    def query(self, key, scope='batman', group=None):
         import etcd
         try:
-            return self._client.get(self.path(key, scope)).value
+            return self._client.get(self.path(key, scope, group)).value
         except etcd.EtcdKeyNotFound:
             return None
 
@@ -439,9 +518,9 @@ class Etcd3Controller(AbstractEtcdController):
         super(Etcd3Controller, self).__init__(context, Etcd3Client)
         os.environ['PATRONI_ETCD3_HOST'] = 'localhost:2379'
 
-    def query(self, key, scope='batman'):
+    def query(self, key, scope='batman', group=None):
         import base64
-        response = self._client.range(self.path(key, scope))
+        response = self._client.range(self.path(key, scope, group))
         for k in response.get('kvs', []):
             return base64.b64decode(k['value']).decode('utf-8') if 'value' in k else None
 
@@ -452,7 +531,43 @@ class Etcd3Controller(AbstractEtcdController):
             assert False, "exception when cleaning up etcd contents: {0}".format(e)
 
 
-class KubernetesController(AbstractDcsController):
+class AbstractExternalDcsController(AbstractDcsController):
+
+    def __init__(self, context, mktemp=True):
+        super(AbstractExternalDcsController, self).__init__(context, mktemp)
+        self._wrapper = ['sudo']
+
+    def _start(self):
+        return self._external_pid
+
+    def start_outage(self):
+        if not self._paused:
+            subprocess.call(self._wrapper + ['kill', '-SIGSTOP', self._external_pid])
+            self._paused = True
+
+    def stop_outage(self):
+        if self._paused:
+            subprocess.call(self._wrapper + ['kill', '-SIGCONT', self._external_pid])
+            self._paused = False
+
+    def _has_started(self):
+        return True
+
+    @abc.abstractmethod
+    def process_name():
+        """process name to search with pgrep"""
+
+    def _is_running(self):
+        if not self._handle:
+            self._external_pid = subprocess.check_output(['pgrep', '-nf', self.process_name()]).decode('utf-8').strip()
+            return False
+        return True
+
+    def stop(self):
+        pass
+
+
+class KubernetesController(AbstractExternalDcsController):
 
     def __init__(self, context):
         super(KubernetesController, self).__init__(context)
@@ -461,19 +576,48 @@ class KubernetesController(AbstractDcsController):
         self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
         os.environ['PATRONI_KUBERNETES_LABELS'] = json.dumps(self._labels)
         os.environ['PATRONI_KUBERNETES_USE_ENDPOINTS'] = 'true'
-        os.environ['PATRONI_KUBERNETES_BYPASS_API_SERVICE'] = 'true'
+        os.environ.setdefault('PATRONI_KUBERNETES_BYPASS_API_SERVICE', 'true')
 
         from patroni.dcs.kubernetes import k8s_client, k8s_config
-        k8s_config.load_kube_config(context='local')
+        k8s_config.load_kube_config(context=os.environ.setdefault('PATRONI_KUBERNETES_CONTEXT', 'kind-kind'))
         self._client = k8s_client
         self._api = self._client.CoreV1Api()
 
-    def _start(self):
-        pass
+    def process_name(self):
+        return "localkube"
 
-    def create_pod(self, name, scope):
+    def _is_running(self):
+        if not self._handle:
+            context = os.environ.get('PATRONI_KUBERNETES_CONTEXT')
+            if context.startswith('kind-'):
+                container = '{0}-control-plane'.format(context[5:])
+                api_process = 'kube-apiserver'
+            elif context.startswith('k3d-'):
+                container = '{0}-server-0'.format(context)
+                api_process = 'k3s'
+            else:
+                return super(KubernetesController, self)._is_running()
+            try:
+                docker = 'docker'
+                with open(os.devnull, 'w') as null:
+                    if subprocess.call([docker, 'info'], stdout=null, stderr=null) != 0:
+                        raise Exception
+            except Exception:
+                docker = 'podman'
+                with open(os.devnull, 'w') as null:
+                    if subprocess.call([docker, 'info'], stdout=null, stderr=null) != 0:
+                        raise Exception
+            self._wrapper = [docker, 'exec', container]
+            self._external_pid = subprocess.check_output(self._wrapper + ['pidof', api_process]).decode('utf-8').strip()
+            return False
+        return True
+
+    def create_pod(self, name, scope, group=None):
+        self.delete_pod(name)
         labels = self._labels.copy()
         labels['cluster-name'] = scope
+        if group is not None:
+            labels['citus-group'] = str(group)
         metadata = self._client.V1ObjectMeta(namespace=self._namespace, name=name, labels=labels)
         spec = self._client.V1PodSpec(containers=[self._client.V1Container(name=name, image='empty')])
         body = self._client.V1Pod(metadata=metadata, spec=spec)
@@ -490,12 +634,14 @@ class KubernetesController(AbstractDcsController):
             except Exception:
                 break
 
-    def query(self, key, scope='batman'):
+    def query(self, key, scope='batman', group=None):
         if key.startswith('members/'):
             pod = self._api.read_namespaced_pod(key[8:], self._namespace)
             return (pod.metadata.annotations or {}).get('status', '')
         else:
             try:
+                if group is not None:
+                    scope = '{0}-{1}'.format(scope, group)
                 ep = scope + {'leader': '', 'history': '-config', 'initialize': '-config'}.get(key, '-' + key)
                 e = self._api.read_namespaced_endpoints(ep, self._namespace)
                 if key != 'sync':
@@ -520,11 +666,8 @@ class KubernetesController(AbstractDcsController):
             if len(result.items) < 1:
                 break
 
-    def _is_running(self):
-        return True
 
-
-class ZooKeeperController(AbstractDcsController):
+class ZooKeeperController(AbstractExternalDcsController):
 
     """ handles all zookeeper related tasks, used for the tests setup and cleanup """
 
@@ -536,13 +679,13 @@ class ZooKeeperController(AbstractDcsController):
         import kazoo.client
         self._client = kazoo.client.KazooClient()
 
-    def _start(self):
-        pass  # TODO: implement later
+    def process_name(self):
+        return "zookeeper"
 
-    def query(self, key, scope='batman'):
+    def query(self, key, scope='batman', group=None):
         import kazoo.exceptions
         try:
-            return self._client.get(self.path(key, scope))[0].decode('utf-8')
+            return self._client.get(self.path(key, scope, group))[0].decode('utf-8')
         except kazoo.exceptions.NoNodeError:
             return None
 
@@ -556,6 +699,9 @@ class ZooKeeperController(AbstractDcsController):
             assert False, "exception when cleaning up zookeeper contents: {0}".format(e)
 
     def _is_running(self):
+        if not super(ZooKeeperController, self)._is_running():
+            return False
+
         # if zookeeper is running, but we didn't start it
         if self._client.connected:
             return True
@@ -605,12 +751,12 @@ class RaftController(AbstractDcsController):
         del env['PATRONI_RAFT_PARTNER_ADDRS']
         env['PATRONI_RAFT_SELF_ADDR'] = self.CONTROLLER_ADDR
         env['PATRONI_RAFT_DATA_DIR'] = self._work_directory
-        return subprocess.Popen([sys.executable, '-m', 'coverage', 'run',
-                                '--source=patroni', '-p', 'patroni_raft_controller.py'],
-                                stdout=self._log, stderr=subprocess.STDOUT, env=env)
+        return psutil.Popen([sys.executable, '-m', 'coverage', 'run',
+                             '--source=patroni', '-p', 'patroni_raft_controller.py'],
+                            stdout=self._log, stderr=subprocess.STDOUT, env=env)
 
-    def query(self, key, scope='batman'):
-        ret = self._raft.get(self.path(key, scope))
+    def query(self, key, scope='batman', group=None):
+        ret = self._raft.get(self.path(key, scope, group))
         return ret and ret['value']
 
     def set(self, key, value):
@@ -626,15 +772,18 @@ class RaftController(AbstractDcsController):
             self.start()
 
         ready_event = threading.Event()
-        self._raft = KVStoreTTL(ready_event.set, None, None, partner_addrs=[self.CONTROLLER_ADDR], password=self.PASSWORD)
+        self._raft = KVStoreTTL(ready_event.set, None, None,
+                                partner_addrs=[self.CONTROLLER_ADDR], password=self.PASSWORD)
         self._raft.startAutoTick()
         ready_event.wait()
 
 
 class PatroniPoolController(object):
 
-    BACKUP_SCRIPT = [sys.executable, 'features/backup_create.py']
-    ARCHIVE_RESTORE_SCRIPT = ' '.join((sys.executable, os.path.abspath('features/archive-restore.py')))
+    PYTHON = sys.executable.replace('\\', '/')
+    BACKUP_SCRIPT = [PYTHON, 'features/backup_create.py']
+    BACKUP_RESTORE_SCRIPT = ' '.join((PYTHON, os.path.abspath('features/backup_restore.py'))).replace('\\', '/')
+    ARCHIVE_RESTORE_SCRIPT = ' '.join((PYTHON, os.path.abspath('features/archive-restore.py')))
 
     def __init__(self, context):
         self._context = context
@@ -643,8 +792,17 @@ class PatroniPoolController(object):
         self._patroni_path = None
         self._processes = {}
         self.create_and_set_output_directory('')
+        self._check_postgres_ssl()
         self.known_dcs = {subclass.name(): subclass for subclass in AbstractDcsController.get_subclasses()}
 
+    def _check_postgres_ssl(self):
+        try:
+            subprocess.check_output(['postgres', '-D', os.devnull, '-c', 'ssl=on'], stderr=subprocess.STDOUT)
+            raise Exception  # this one should never happen because the previous line will always raise and exception
+        except Exception as e:
+            self._context.postgres_supports_ssl = isinstance(e, subprocess.CalledProcessError)\
+                    and 'SSL is not supported by this build' not in e.output.decode()
+
     @property
     def patroni_path(self):
         if self._patroni_path is None:
@@ -695,7 +853,8 @@ class PatroniPoolController(object):
             'bootstrap': {
                 'method': 'pg_basebackup',
                 'pg_basebackup': {
-                    'command': " ".join(self.BACKUP_SCRIPT) + ' --walmethod=stream --dbname=' + f.backup_source
+                    'command': " ".join(self.BACKUP_SCRIPT +
+                                        ['--walmethod=stream', '--dbname="{0}"'.format(f.backup_source)])
                 },
                 'dcs': {
                     'postgresql': {
@@ -710,7 +869,7 @@ class PatroniPoolController(object):
                     'archive_mode': 'on',
                     'archive_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode archive ' +
                                         '--dirname {} --filename %f --pathname %p').format(
-                                        os.path.join(self.patroni_path, 'data', 'wal_archive'))
+                                        os.path.join(self.patroni_path, 'data', 'wal_archive_clone').replace('\\', '/'))
                 },
                 'authentication': {
                     'superuser': {'password': 'zalando1'},
@@ -726,14 +885,14 @@ class PatroniPoolController(object):
             'bootstrap': {
                 'method': 'backup_restore',
                 'backup_restore': {
-                    'command': (sys.executable + ' features/backup_restore.py --sourcedir=' +
-                                os.path.join(self.patroni_path, 'data', 'basebackup')),
+                    'command': (self.BACKUP_RESTORE_SCRIPT + ' --sourcedir=' +
+                                os.path.join(self.patroni_path, 'data', 'basebackup').replace('\\', '/')),
                     'recovery_conf': {
                         'recovery_target_action': 'promote',
                         'recovery_target_timeline': 'latest',
                         'restore_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode restore ' +
                                             '--dirname {} --filename %f --pathname %p').format(
-                                            os.path.join(self.patroni_path, 'data', 'wal_archive'))
+                            os.path.join(self.patroni_path, 'data', 'wal_archive_clone').replace('\\', '/'))
                     }
                 }
             },
@@ -746,6 +905,25 @@ class PatroniPoolController(object):
         }
         self.start(name, custom_config=custom_config)
 
+    def bootstrap_from_backup_no_leader(self, name, cluster_name):
+        custom_config = {
+            'scope': cluster_name,
+            'postgresql': {
+                'recovery_conf': {
+                    'restore_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode restore ' +
+                                        '--dirname {} --filename %f --pathname %p').format(
+                                        os.path.join(self.patroni_path, 'data', 'wal_archive').replace('\\', '/'))
+                },
+                'create_replica_methods': ['no_leader_bootstrap'],
+                'no_leader_bootstrap': {
+                    'command': (self.BACKUP_RESTORE_SCRIPT + ' --sourcedir=' +
+                                os.path.join(self.patroni_path, 'data', 'basebackup').replace('\\', '/')),
+                    'no_leader': '1'
+                }
+            }
+        }
+        self.start(name, custom_config=custom_config)
+
     @property
     def dcs(self):
         if self._dcs is None:
@@ -872,10 +1050,32 @@ class WatchdogMonitor(object):
 
 # actions to execute on start/stop of the tests and before running individual features
 def before_all(context):
-    os.environ.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
-    context.ci = any(a in os.environ for a in ('TRAVIS_BUILD_NUMBER', 'BUILD_NUMBER', 'GITHUB_ACTIONS'))
+    context.ci = os.name == 'nt' or\
+        any(a in os.environ for a in ('TRAVIS_BUILD_NUMBER', 'BUILD_NUMBER', 'GITHUB_ACTIONS'))
     context.timeout_multiplier = 5 if context.ci else 1  # MacOS sometimes is VERY slow
     context.pctl = PatroniPoolController(context)
+
+    context.keyfile = os.path.join(context.pctl.output_dir, 'patroni.key')
+    context.certfile = os.path.join(context.pctl.output_dir, 'patroni.crt')
+    try:
+        with open(os.devnull, 'w') as null:
+            ret = subprocess.call(['openssl', 'req', '-nodes', '-new', '-x509', '-subj', '/CN=batman.patroni',
+                                   '-keyout', context.keyfile, '-out', context.certfile], stdout=null, stderr=null)
+            if ret != 0:
+                raise Exception
+    except Exception:
+        context.keyfile = context.certfile = None
+
+    os.environ.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
+    ctl = {'auth': os.environ['PATRONI_RESTAPI_USERNAME'] + ':' + os.environ['PATRONI_RESTAPI_PASSWORD']}
+    if context.certfile:
+        os.environ.update({'PATRONI_RESTAPI_CAFILE': context.certfile,
+                           'PATRONI_RESTAPI_CERTFILE': context.certfile,
+                           'PATRONI_RESTAPI_KEYFILE': context.keyfile,
+                           'PATRONI_RESTAPI_VERIFY_CLIENT': 'required',
+                           'PATRONI_CTL_INSECURE': 'on'})
+        ctl.update({'cacert': context.certfile, 'certfile': context.certfile, 'keyfile': context.keyfile})
+    context.request_executor = PatroniRequest({'ctl': ctl}, True)
     context.dcs_ctl = context.pctl.known_dcs[context.pctl.dcs](context)
     context.dcs_ctl.start()
     try:
@@ -893,13 +1093,45 @@ def after_all(context):
 
 def before_feature(context, feature):
     """ create per-feature output directory to collect Patroni and PostgreSQL logs """
+    if feature.name == 'watchdog' and os.name == 'nt':
+        return feature.skip("Watchdog isn't supported on Windows")
+    elif feature.name == 'citus':
+        lib = subprocess.check_output(['pg_config', '--pkglibdir']).decode('utf-8').strip()
+        if not os.path.exists(os.path.join(lib, 'citus.so')):
+            return feature.skip("Citus extenstion isn't available")
     context.pctl.create_and_set_output_directory(feature.name)
 
 
 def after_feature(context, feature):
-    """ stop all Patronis, remove their data directory and cleanup the keys in etcd """
+    """ send SIGCONT to a dcs if neccessary,
+    stop all Patronis remove their data directory and cleanup the keys in etcd """
+    context.dcs_ctl.stop_outage()
     context.pctl.stop_all()
-    shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data'))
+    data = os.path.join(context.pctl.patroni_path, 'data')
+    if os.path.exists(data):
+        shutil.rmtree(data)
     context.dcs_ctl.cleanup_service_tree()
-    if feature.status == 'failed':
+
+    found = False
+    logs = glob.glob(context.pctl.output_dir + '/patroni_*.log')
+    for log in logs:
+        with open(log) as f:
+            for line in f:
+                if 'please report it as a BUG' in line:
+                    print(':'.join([log, line.rstrip()]))
+                    found = True
+
+    if feature.status == 'failed' or found:
         shutil.copytree(context.pctl.output_dir, context.pctl.output_dir + '_failed')
+    if found:
+        raise Exception('Unexpected errors in Patroni log files')
+
+
+def before_scenario(context, scenario):
+    if 'slot-advance' in scenario.effective_tags:
+        for p in context.pctl._processes.values():
+            if p._conn and p._conn.server_version < 110000:
+                scenario.skip('pg_replication_slot_advance() is not supported on {0}'.format(p._conn.server_version))
+                break
+    if 'dcs-failsafe' in scenario.effective_tags and not context.dcs_ctl._handle:
+        scenario.skip('it is not possible to control state of {0} from tests'.format(context.dcs_ctl.name()))

features/ignored_slots.feature

@@ -3,7 +3,7 @@ Feature: ignored slots
     Given I start postgres1
     Then postgres1 is a leader after 10 seconds
     And there is a non empty initialize key in DCS after 15 seconds
-    When I issue a PATCH request to http://127.0.0.1:8009/config with {"loop_wait": 2, "ignore_slots": [{"name": "unmanaged_slot_0", "database": "postgres", "plugin": "test_decoding", "type": "logical"}, {"name": "unmanaged_slot_1", "database": "postgres", "plugin": "test_decoding"}, {"name": "unmanaged_slot_2", "database": "postgres"}, {"name": "unmanaged_slot_3"}], "postgresql": {"parameters": {"wal_level": "logical"}}}
+    When I issue a PATCH request to http://127.0.0.1:8009/config with {"ignore_slots": [{"name": "unmanaged_slot_0", "database": "postgres", "plugin": "test_decoding", "type": "logical"}, {"name": "unmanaged_slot_1", "database": "postgres", "plugin": "test_decoding"}, {"name": "unmanaged_slot_2", "database": "postgres"}, {"name": "unmanaged_slot_3"}], "postgresql": {"parameters": {"wal_level": "logical"}}}
     Then I receive a response code 200
     And Response on GET http://127.0.0.1:8009/config contains ignore_slots after 10 seconds
     # Make sure the wal_level has been changed.
@@ -52,7 +52,7 @@ Feature: ignored slots
     And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin
     And postgres1 does not have a logical replication slot named dummy_slot
 
-    # 3. After a failover the server (now a master) still has the slot.
+    # 3. After a failover the server (now a primary) still has the slot.
     When I shut down postgres0
     Then "members/postgres1" key in DCS has role=master after 3 seconds
     And postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin

features/patroni_api.feature

@@ -14,9 +14,9 @@ Scenario: check API requests on a stand-alone server
 	Then I receive a response code 200
 	When I issue a GET request to http://127.0.0.1:8008/replica
 	Then I receive a response code 503
-	When I run patronictl.py reinit batman postgres0 --force
-	Then I receive a response returncode 0
-	And I receive a response output "Failed: reinitialize for member postgres0, status code=503, (I am the leader, can not reinitialize)"
+	When I issue a POST request to http://127.0.0.1:8008/reinitialize with {"force": true}
+	Then I receive a response code 503
+	And I receive a response text I am the leader, can not reinitialize
 	When I run patronictl.py switchover batman --master postgres0 --force
 	Then I receive a response returncode 1
 	And I receive a response output "Error: No candidates found to switchover to"
@@ -35,13 +35,13 @@ Scenario: check local configuration reload
 	Then I receive a response code 202
 
 Scenario: check dynamic configuration change via DCS
-	Given I run patronictl.py edit-config -s 'ttl=10' -s 'loop_wait=2' -p 'max_connections=101' --force batman
+	Given I run patronictl.py edit-config -s 'ttl=10' -p 'max_connections=101' --force batman
 	Then I receive a response returncode 0
-	And I receive a response output "+loop_wait: 2"
+	And I receive a response output "+ttl: 10"
 	And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 11 seconds
 	When I issue a GET request to http://127.0.0.1:8008/config
 	Then I receive a response code 200
-	And I receive a response loop_wait 2
+	And I receive a response ttl 10
 	When I issue a GET request to http://127.0.0.1:8008/patroni
 	Then I receive a response code 200
 	And I receive a response tags {'new_tag': 'new_value'}
@@ -94,11 +94,11 @@ Scenario: check the switchover via the API in the pause mode
 	And postgres0 role is the secondary after 10 seconds
 	And replication works from postgres1 to postgres0 after 20 seconds
 	And "members/postgres0" key in DCS has state=running after 10 seconds
-	When I issue a GET request to http://127.0.0.1:8008/master
+	When I issue a GET request to http://127.0.0.1:8008/primary
 	Then I receive a response code 503
 	When I issue a GET request to http://127.0.0.1:8008/replica
 	Then I receive a response code 200
-	When I issue a GET request to http://127.0.0.1:8009/master
+	When I issue a GET request to http://127.0.0.1:8009/primary
 	Then I receive a response code 200
 	When I issue a GET request to http://127.0.0.1:8009/replica
 	Then I receive a response code 503
@@ -109,18 +109,18 @@ Scenario: check the scheduled switchover
 	And I receive a response output "Can't schedule switchover in the paused state"
 	When I run patronictl.py resume batman
 	Then I receive a response returncode 0
-	Given I issue a scheduled switchover from postgres1 to postgres0 in 5 seconds
+	Given I issue a scheduled switchover from postgres1 to postgres0 in 10 seconds
 	Then I receive a response returncode 0
 	And postgres0 is a leader after 20 seconds
 	And postgres0 role is the primary after 10 seconds
 	And postgres1 role is the secondary after 10 seconds
 	And replication works from postgres0 to postgres1 after 25 seconds
 	And "members/postgres1" key in DCS has state=running after 10 seconds
-	When I issue a GET request to http://127.0.0.1:8008/master
+	When I issue a GET request to http://127.0.0.1:8008/primary
 	Then I receive a response code 200
 	When I issue a GET request to http://127.0.0.1:8008/replica
 	Then I receive a response code 503
-	When I issue a GET request to http://127.0.0.1:8009/master
+	When I issue a GET request to http://127.0.0.1:8009/primary
 	Then I receive a response code 503
 	When I issue a GET request to http://127.0.0.1:8009/replica
 	Then I receive a response code 200

features/standby_cluster.feature

@@ -3,7 +3,7 @@ Feature: standby cluster
     Given I start postgres1
     Then postgres1 is a leader after 10 seconds
     And there is a non empty initialize key in DCS after 15 seconds
-    When I issue a PATCH request to http://127.0.0.1:8009/config with {"loop_wait": 2, "slots": {"pm_1": {"type": "physical"}}, "postgresql": {"parameters": {"wal_level": "logical"}}}
+    When I issue a PATCH request to http://127.0.0.1:8009/config with {"slots": {"pm_1": {"type": "physical"}}, "postgresql": {"parameters": {"wal_level": "logical"}}}
     Then I receive a response code 200
     And Response on GET http://127.0.0.1:8009/config contains slots after 10 seconds
     And I sleep for 3 seconds
@@ -14,7 +14,7 @@ Feature: standby cluster
     Then "members/postgres0" key in DCS has state=running after 10 seconds
     And replication works from postgres1 to postgres0 after 15 seconds
 
-  @skip
+  @slot-advance
   Scenario: check permanent logical slots are synced to the replica
     Given I run patronictl.py restart batman postgres1 --force
     Then Logical slot test_logical is in sync between postgres0 and postgres1 after 10 seconds
@@ -35,7 +35,7 @@ Feature: standby cluster
     When I add the table foo to postgres0
     Then table foo is present on postgres1 after 20 seconds
     And I sleep for 3 seconds
-    When I issue a GET request to http://127.0.0.1:8009/master
+    When I issue a GET request to http://127.0.0.1:8009/primary
     Then I receive a response code 503
     When I issue a GET request to http://127.0.0.1:8009/standby_leader
     Then I receive a response code 200
@@ -50,7 +50,7 @@ Feature: standby cluster
     When I kill postgres1
     And I kill postmaster on postgres1
     Then postgres2 is replicating from postgres0 after 32 seconds
-    When I issue a GET request to http://127.0.0.1:8010/master
+    When I issue a GET request to http://127.0.0.1:8010/primary
     Then I receive a response code 503
     And I sleep for 3 seconds
     When I issue a GET request to http://127.0.0.1:8010/standby_leader

features/steps/basic_replication.py

@@ -28,7 +28,7 @@ def stop_postgres(context, name):
 def add_table(context, table_name, pg_name):
     # parse the configuration file and get the port
     try:
-        context.pctl.query(pg_name, "CREATE TABLE {0}()".format(table_name))
+        context.pctl.query(pg_name, "CREATE TABLE public.{0}()".format(table_name))
     except pg.Error as e:
         assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
 
@@ -37,9 +37,9 @@ def add_table(context, table_name, pg_name):
 def toggle_wal_replay(context, action, pg_name):
     # pause or resume the wal replay process
     try:
-       version = context.pctl.query(pg_name, "select pg_catalog.pg_read_file('PG_VERSION', 0, 2)").fetchone()
-       wal = version and version[0] and int(version[0].split('.')[0]) < 10 and "xlog" or "wal"
-       context.pctl.query(pg_name, "SELECT pg_{0}_replay_{1}()".format(wal, action))
+        version = context.pctl.query(pg_name, "SHOW server_version_num").fetchone()[0]
+        wal_name = 'xlog' if int(version)/10000 < 10 else 'wal'
+        context.pctl.query(pg_name, "SELECT pg_{0}_replay_{1}()".format(wal_name, action))
     except pg.Error as e:
         assert False, "Error during {0} wal recovery on {1}: {2}".format(action, pg_name, e)
 
@@ -47,10 +47,10 @@ def toggle_wal_replay(context, action, pg_name):
 @step('I {action:w} table on {pg_name:w}')
 def crdr_mytest(context, action, pg_name):
     try:
-       if (action == "create"):
-           context.pctl.query(pg_name, "create table if not exists mytest(id Numeric)")
-       else:
-           context.pctl.query(pg_name, "drop table if exists mytest")
+        if (action == "create"):
+            context.pctl.query(pg_name, "create table if not exists public.mytest(id numeric)")
+        else:
+            context.pctl.query(pg_name, "drop table if exists public.mytest")
     except pg.Error as e:
         assert False, "Error {0} table mytest on {1}: {2}".format(action, pg_name, e)
 
@@ -59,7 +59,7 @@ def crdr_mytest(context, action, pg_name):
 def initiate_load(context, pg_name):
     # perform dummy load
     try:
-       context.pctl.query(pg_name, "begin; insert into mytest select r::numeric from generate_series(1, 350000) r; commit;")
+        context.pctl.query(pg_name, "insert into public.mytest select r::numeric from generate_series(1, 350000) r")
     except pg.Error as e:
         assert False, "Error loading test data on {0}: {1}".format(pg_name, e)
 
@@ -68,7 +68,7 @@ def initiate_load(context, pg_name):
 def table_is_present_on(context, table_name, pg_name, max_replication_delay):
     max_replication_delay *= context.timeout_multiplier
     for _ in range(int(max_replication_delay)):
-        if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
+        if context.pctl.query(pg_name, "SELECT 1 FROM public.{0}".format(table_name), fail_ok=True) is not None:
             break
         sleep(1)
     else:
@@ -83,10 +83,10 @@ def check_role(context, pg_name, pg_role, max_promotion_timeout):
         "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
 
 
-@step('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
-@then('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
-def replication_works(context, master, replica, time_limit):
+@step('replication works from {primary:w} to {replica:w} after {time_limit:d} seconds')
+@then('replication works from {primary:w} to {replica:w} after {time_limit:d} seconds')
+def replication_works(context, primary, replica, time_limit):
     context.execute_steps(u"""
         When I add the table test_{0} to {1}
         Then table test_{0} is present on {2} after {3} seconds
-    """.format(int(time()), master, replica, time_limit))
+    """.format(int(time()), primary, replica, time_limit))

features/steps/citus.py

@@ -0,0 +1,117 @@
+import json
+import time
+
+from behave import step, then
+from dateutil import tz
+from datetime import datetime
+from functools import partial
+from threading import Thread, Event
+
+tzutc = tz.tzutc()
+
+
+@step('{name:w} is a leader in a group {group:d} after {time_limit:d} seconds')
+@then('{name:w} is a leader in a group {group:d} after {time_limit:d} seconds')
+def is_a_group_leader(context, name, group, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    while (context.dcs_ctl.query("leader", group=group) != name):
+        time.sleep(1)
+        assert time.time() < max_time, "{0} is not a leader in dcs after {1} seconds".format(name, time_limit)
+
+
+@step('"{name}" key in a group {group:d} in DCS has {key:w}={value} after {time_limit:d} seconds')
+def check_group_member(context, name, group, key, value, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    dcs_value = None
+    response = None
+    while time.time() < max_time:
+        try:
+            response = json.loads(context.dcs_ctl.query(name, group=group))
+            dcs_value = response.get(key)
+            if dcs_value == value:
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    assert False, ("{0} in a group {1} does not have {2}={3} (found {4}) in dcs" +
+                   " after {5} seconds").format(name, group, key, value, response, time_limit)
+
+
+@step('I start {name:w} in citus group {group:d}')
+def start_citus(context, name, group):
+    return context.pctl.start(name, custom_config={"citus": {"database": "postgres", "group": int(group)}})
+
+
+@step('{name1:w} is registered in the coordinator {name2:w} as the worker in group {group:d}')
+def check_registration(context, name1, name2, group):
+    worker_port = int(context.pctl.query(name1, "SHOW port").fetchone()[0])
+    r = context.pctl.query(name2, "SELECT nodeport FROM pg_catalog.pg_dist_node WHERE groupid = {0}".format(group))
+    assert worker_port == r.fetchone()[0],\
+        "Worker {0} is not registered in pg_dist_node on the coordinator {1}".format(name1, name2)
+
+
+@step('I create a distributed table on {name:w}')
+def create_distributed_table(context, name):
+    context.pctl.query(name, 'CREATE TABLE public.d(id int not null)')
+    context.pctl.query(name, "SELECT create_distributed_table('public.d', 'id')")
+
+
+@step('I cleanup a distributed table on {name:w}')
+def cleanup_distributed_table(context, name):
+    context.pctl.query(name, 'TRUNCATE public.d')
+
+
+def insert_thread(query_func, context):
+    while True:
+        if context.thread_stop_event.is_set():
+            break
+
+        context.insert_counter += 1
+        query_func('INSERT INTO public.d VALUES({0})'.format(context.insert_counter))
+
+        context.thread_stop_event.wait(0.01)
+
+
+@step('I start a thread inserting data on {name:w}')
+def start_insert_thread(context, name):
+    context.thread_stop_event = Event()
+    context.insert_counter = 0
+    query_func = partial(context.pctl.query, name)
+    thread_func = partial(insert_thread, query_func, context)
+    context.thread = Thread(target=thread_func)
+    context.thread.daemon = True
+    context.thread.start()
+
+
+@then('a thread is still alive')
+def thread_is_alive(context):
+    assert context.thread.is_alive(), "Thread is not alive"
+
+
+@step("I stop a thread")
+def stop_insert_thread(context):
+    context.thread_stop_event.set()
+    context.thread.join(1*context.timeout_multiplier)
+    assert not context.thread.is_alive(), "Thread is still alive"
+
+
+@step("a distributed table on {name:w} has expected rows")
+def count_rows(context, name):
+    rows = context.pctl.query(name, "SELECT COUNT(*) FROM public.d").fetchone()[0]
+    assert rows == context.insert_counter, "Distributed table doesn't have expected amount of rows"
+
+
+@step("There is a transaction in progress on {name:w} changing pg_dist_node")
+def check_transaction(context, name):
+    cur = context.pctl.query(name, "SELECT xact_start FROM pg_stat_activity WHERE pid <> pg_backend_pid()"
+                                   " AND state = 'idle in transaction' AND query ~ 'citus_update_node'")
+    assert cur.rowcount == 1, "There is no idle in transaction updating pg_dist_node"
+    context.xact_start = cur.fetchone()[0]
+
+
+@step("a transaction finishes in {timeout:d} seconds")
+def check_transaction_timeout(context, timeout):
+    assert (datetime.now(tzutc) - context.xact_start).seconds > timeout,\
+            "a transaction finished earlier than in {0} seconds".format(timeout)

features/steps/dcs_failsafe_mode.py

@@ -0,0 +1,16 @@
+from behave import step
+
+
+@step('DCS is down')
+def start_dcs_outage(context):
+    context.dcs_ctl.start_outage()
+
+
+@step('DCS is up')
+def stop_dcs_outage(context):
+    context.dcs_ctl.stop_outage()
+
+
+@step('I start {name:w} in a cluster {cluster_name:w} from backup with no_leader')
+def start_cluster_from_backup_no_leader(context, name, cluster_name):
+    context.pctl.bootstrap_from_backup_no_leader(name, cluster_name)

features/steps/patroni_api.py

@@ -1,5 +1,4 @@
 import json
-import os
 import parse
 import shlex
 import subprocess
@@ -10,10 +9,8 @@ import yaml
 from behave import register_type, step, then
 from dateutil import tz
 from datetime import datetime, timedelta
-from patroni.request import PatroniRequest
 
 tzutc = tz.tzutc()
-request_executor = PatroniRequest({'ctl': {'auth': 'username:password'}})
 
 
 @parse.with_pattern(r'https?://(?:\w|\.|:|/)+')
@@ -73,11 +70,13 @@ def do_post_empty(context, url):
 
 @step('I issue a {request_method:w} request to {url:url} with {data}')
 def do_request(context, request_method, url, data):
+    if context.certfile:
+        url = url.replace('http://', 'https://')
     data = data and json.loads(data)
     try:
-        r = request_executor.request(request_method, url, data)
+        r = context.request_executor.request(request_method, url, data)
         if request_method == 'PATCH' and r.status == 409:
-            r = request_executor.request(request_method, url, data)
+            r = context.request_executor.request(request_method, url, data)
     except Exception:
         context.status_code = context.response = None
     else:
@@ -88,10 +87,7 @@ def do_request(context, request_method, url, data):
 def do_run(context, cmd):
     cmd = [sys.executable, '-m', 'coverage', 'run', '--source=patroni', '-p'] + shlex.split(cmd)
     try:
-        # XXX: Dirty hack! We need to take name/passwd from the config!
-        env = os.environ.copy()
-        env.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
-        response = subprocess.check_output(cmd, stderr=subprocess.STDOUT, env=env)
+        response = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         context.status_code = 0
     except subprocess.CalledProcessError as e:
         response = e.output
@@ -113,6 +109,8 @@ def check_response(context, component, data):
         assert data.strip('"') in context.response, "response {0} does not contain {1}".format(context.response, data)
     else:
         assert component in context.response, "{0} is not part of the response".format(component)
+        if context.certfile:
+            data = data.replace('http://', 'https://')
         assert str(context.response[component]) == str(data), "{0} does not contain {1}".format(component, data)
 
 
@@ -135,11 +133,27 @@ def add_tag_to_config(context, tag, value, pg_name):
     context.pctl.add_tag_to_config(pg_name, tag, value)
 
 
-@then('Response on GET {url} contains {value} after {timeout:d} seconds')
-def check_http_response(context, url, value, timeout, negate=False):
+@then('Status code on GET {url:url} is {code:d} after {timeout:d} seconds')
+def check_http_code(context, url, code, timeout):
+    if context.certfile:
+        url = url.replace('http://', 'https://')
     timeout *= context.timeout_multiplier
     for _ in range(int(timeout)):
-        r = request_executor.request('GET', url)
+        r = context.request_executor.request('GET', url)
+        if int(code) == int(r.status):
+            break
+        time.sleep(1)
+    else:
+        assert False, "HTTP Status Code is not {0} after {1} seconds".format(code, timeout)
+
+
+@then('Response on GET {url:url} contains {value} after {timeout:d} seconds')
+def check_http_response(context, url, value, timeout, negate=False):
+    if context.certfile:
+        url = url.replace('http://', 'https://')
+    timeout *= context.timeout_multiplier
+    for _ in range(int(timeout)):
+        r = context.request_executor.request('GET', url)
         if (value in r.data.decode('utf-8')) != negate:
             break
         time.sleep(1)

features/steps/standby_cluster.py

@@ -1,17 +1,12 @@
 import os
-import sys
 import time
 
 from behave import step
 
 
-select_replication_query = """
-SELECT * FROM pg_catalog.pg_stat_replication
-WHERE application_name = '{0}'
-"""
-
-executable = sys.executable if os.name != 'nt' else sys.executable.replace('\\', '/')
-callback = executable + " features/callback2.py "
+def callbacks(context, name):
+    return {c: '{0} features/callback2.py {1}'.format(context.pctl.PYTHON, name)
+            for c in ('on_start', 'on_stop', 'on_restart', 'on_role_change')}
 
 
 @step('I start {name:w} in a cluster {cluster_name:w}')
@@ -19,10 +14,10 @@ def start_patroni(context, name, cluster_name):
     return context.pctl.start(name, custom_config={
         "scope": cluster_name,
         "postgresql": {
-            "callbacks": {c: callback + name for c in ('on_start', 'on_stop', 'on_restart', 'on_role_change')},
+            "callbacks": callbacks(context, name),
             "backup_restore": {
-                "command": (executable + " features/backup_restore.py --sourcedir=" +
-                            os.path.join(context.pctl.patroni_path, 'data', 'basebackup'))}
+                "command": (context.pctl.PYTHON + " features/backup_restore.py --sourcedir=" +
+                            os.path.join(context.pctl.patroni_path, 'data', 'basebackup').replace('\\', '/'))}
         }
     })
 
@@ -49,7 +44,7 @@ def start_patroni_standby_cluster(context, name, cluster_name, name2):
             }
         },
         "postgresql": {
-            "callbacks": {c: callback + name for c in ('on_start', 'on_stop', 'on_restart', 'on_role_change')}
+            "callbacks": callbacks(context, name)
         }
     })
     return context.pctl.start(name)
@@ -62,7 +57,7 @@ def check_replication_status(context, pg_name1, pg_name2, timeout):
     while time.time() < bound_time:
         cur = context.pctl.query(
             pg_name2,
-            select_replication_query.format(pg_name1),
+            "SELECT * FROM pg_catalog.pg_stat_replication WHERE application_name = '{0}'".format(pg_name1),
             fail_ok=True
         )
 

features/steps/watchdog.py

@@ -15,7 +15,7 @@ def polling_loop(timeout, interval=1):
 
 @step('I start {name:w} with watchdog')
 def start_patroni_with_watchdog(context, name):
-    return context.pctl.start(name, custom_config={'watchdog': True})
+    return context.pctl.start(name, custom_config={'watchdog': True, 'bootstrap': {'dcs': {'ttl': 20}}})
 
 
 @step('{name:w} watchdog has been pinged after {timeout:d} seconds')
@@ -31,6 +31,11 @@ def watchdog_was_closed(context, name):
     assert context.pctl.get_watchdog(name).was_closed
 
 
+@step('{name:w} watchdog has a {timeout:d} second timeout')
+def watchdog_has_timeout(context, name, timeout):
+    assert context.pctl.get_watchdog(name).timeout == timeout
+
+
 @step('I reset {name:w} watchdog state')
 def watchdog_reset_pinged(context, name):
     context.pctl.get_watchdog(name).reset()

features/watchdog.feature

@@ -6,6 +6,14 @@ Feature: watchdog
     Then postgres0 is a leader after 10 seconds
     And postgres0 role is the primary after 10 seconds
     And postgres0 watchdog has been pinged after 10 seconds
+    And postgres0 watchdog has a 15 second timeout
+
+  Scenario: watchdog is reconfigured after global ttl changed
+    Given I run patronictl.py edit-config batman -s ttl=30 --force
+    Then I receive a response returncode 0
+    And I receive a response output "+ttl: 30"
+    When I sleep for 4 seconds
+    Then postgres0 watchdog has a 25 second timeout
 
   Scenario: watchdog is disabled during pause
     Given I run patronictl.py pause batman

kubernetes/Dockerfile

@@ -1,10 +1,9 @@
-FROM postgres:11
-MAINTAINER Alexander Kukushkin <alexander.kukushkin@zalando.de>
+FROM postgres:15
+LABEL maintainer="Alexander Kukushkin <akukushkin@microsoft.com>"
 
 RUN export DEBIAN_FRONTEND=noninteractive \
     && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
     && apt-get update -y \
-    && apt-get upgrade -y \
     && apt-cache depends patroni | sed -n -e 's/.* Depends: \(python3-.\+\)$/\1/p' \
             | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
             | xargs apt-get install -y vim-tiny curl jq locales git python3-pip python3-wheel \
@@ -25,7 +24,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/* /root/.cache
 
-ADD entrypoint.sh /
+COPY entrypoint.sh /
 
 EXPOSE 5432 8008
 ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 EDITOR=/usr/bin/editor

kubernetes/Dockerfile.citus

@@ -0,0 +1,42 @@
+FROM postgres:15
+LABEL maintainer="Alexander Kukushkin <[email protected]>"
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && apt-get update -y \
+    && apt-get upgrade -y \
+    && apt-cache depends patroni | sed -n -e 's/.* Depends: \(python3-.\+\)$/\1/p' \
+            | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
+            | xargs apt-get install -y busybox vim-tiny curl jq less locales git python3-pip python3-wheel \
+    ## Make sure we have a en_US.UTF-8 locale available
+    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
+    && curl https://install.citusdata.com/community/deb.sh | bash \
+    && apt-get -y install postgresql-15-citus-11.1 \
+    && pip3 install setuptools \
+    && pip3 install 'git+https://github.com/zalando/patroni.git#egg=patroni[kubernetes]' \
+    && PGHOME=/home/postgres \
+    && mkdir -p $PGHOME \
+    && chown postgres $PGHOME \
+    && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \
+    && /bin/busybox --install -s \
+    # Set permissions for OpenShift
+    && chmod 775 $PGHOME \
+    && chmod 664 /etc/passwd \
+    # Clean up
+    && apt-get remove -y git python3-pip python3-wheel \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* /root/.cache
+
+ADD entrypoint.sh /
+ENV PGSSLMODE=verify-ca PGSSLKEY=/etc/ssl/private/ssl-cert-snakeoil.key PGSSLCERT=/etc/ssl/certs/ssl-cert-snakeoil.pem PGSSLROOTCERT=/etc/ssl/certs/ssl-cert-snakeoil.pem
+
+RUN sed -i 's/^postgresql:/&\n  basebackup:\n    checkpoint: fast/' /entrypoint.sh \
+    && sed -i "s|^    postgresql:|&\n      pg_hba:\n      - local all all trust\n      - hostssl replication all all md5 clientcert=$PGSSLMODE\n      - hostssl all all all md5 clientcert=$PGSSLMODE\n      parameters:\n        max_connections: 100\n        shared_buffers: 16MB\n        ssl: 'on'\n        ssl_ca_file: $PGSSLROOTCERT\n        ssl_cert_file: $PGSSLCERT\n        ssl_key_file: $PGSSLKEY\n        citus.node_conninfo: 'sslrootcert=$PGSSLROOTCERT sslkey=$PGSSLKEY sslcert=$PGSSLCERT sslmode=$PGSSLMODE'|"  /entrypoint.sh \
+    && sed -i "s#^    \(superuser\|replication\):#&\n      sslmode: $PGSSLMODE\n      sslkey: $PGSSLKEY\n      sslcert: $PGSSLCERT\n      sslrootcert: $PGSSLROOTCERT#" /entrypoint.sh
+
+EXPOSE 5432 8008
+ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 EDITOR=/usr/bin/editor
+USER postgres
+WORKDIR /home/postgres
+CMD ["/bin/bash", "/entrypoint.sh"]

kubernetes/README.md

@@ -0,0 +1,154 @@
+# Kubernetes deployment examples
+Below you will find examples of Patroni deployments using [kind](https://kind.sigs.k8s.io/).
+
+# Patroni on K8s
+The Patroni cluster deployment with a StatefulSet consisting of three Pods.
+
+Example session:
+
+    $ kind create cluster
+    Creating cluster "kind" ...
+     ✓ Ensuring node image (kindest/node:v1.25.3) 🖼
+     ✓ Preparing nodes 📦
+     ✓ Writing configuration 📜
+     ✓ Starting control-plane 🕹️
+     ✓ Installing CNI 🔌
+     ✓ Installing StorageClass 💾
+    Set kubectl context to "kind-kind"
+    You can now use your cluster with:
+
+    kubectl cluster-info --context kind-kind
+
+    Thanks for using kind! 😊
+
+    $ docker build -t patroni .
+    Sending build context to Docker daemon  138.8kB
+    Step 1/9 : FROM postgres:15
+    ...
+    Successfully built e9bfe69c5d2b
+    Successfully tagged patroni:latest
+
+    $ kind load docker-image patroni
+    Image: "" with ID "sha256:e9bfe69c5d2b319dec0cf564fb895484537664775e18f37f9b707914cc5537e6" not yet present on node "kind-control-plane", loading...
+
+    $ kubectl apply -f patroni_k8s.yaml
+    service/patronidemo-config created
+    statefulset.apps/patronidemo created
+    endpoints/patronidemo created
+    service/patronidemo created
+    service/patronidemo-repl created
+    secret/patronidemo created
+    serviceaccount/patronidemo created
+    role.rbac.authorization.k8s.io/patronidemo created
+    rolebinding.rbac.authorization.k8s.io/patronidemo created
+    clusterrole.rbac.authorization.k8s.io/patroni-k8s-ep-access created
+    clusterrolebinding.rbac.authorization.k8s.io/patroni-k8s-ep-access created
+
+    $ kubectl get pods -L role
+    NAME            READY   STATUS    RESTARTS   AGE   ROLE
+    patronidemo-0   1/1     Running   0          34s   master
+    patronidemo-1   1/1     Running   0          30s   replica
+    patronidemo-2   1/1     Running   0          26s   replica
+
+    $ kubectl exec -ti patronidemo-0 -- bash
+    postgres@patronidemo-0:~$ patronictl list
+    + Cluster: patronidemo (7186662553319358497) ----+----+-----------+
+    | Member        | Host       | Role    | State   | TL | Lag in MB |
+    +---------------+------------+---------+---------+----+-----------+
+    | patronidemo-0 | 10.244.0.5 | Leader  | running |  1 |           |
+    | patronidemo-1 | 10.244.0.6 | Replica | running |  1 |         0 |
+    | patronidemo-2 | 10.244.0.7 | Replica | running |  1 |         0 |
+    +---------------+------------+---------+---------+----+-----------+
+
+# Citus on K8s
+The Citus cluster with the StatefulSets, one coordinator with three Pods and two workers with two pods each.
+
+Example session:
+
+    $ kind create cluster
+    Creating cluster "kind" ...
+     ✓ Ensuring node image (kindest/node:v1.25.3) 🖼
+     ✓ Preparing nodes 📦
+     ✓ Writing configuration 📜
+     ✓ Starting control-plane 🕹️
+     ✓ Installing CNI 🔌
+     ✓ Installing StorageClass 💾
+    Set kubectl context to "kind-kind"
+    You can now use your cluster with:
+
+    kubectl cluster-info --context kind-kind
+
+    Thanks for using kind! 😊
+
+    demo@localhost:~/git/patroni/kubernetes$ docker build -f Dockerfile.citus -t patroni-citus-k8s .
+    Sending build context to Docker daemon  138.8kB
+    Step 1/11 : FROM postgres:15
+    ...
+    Successfully built 8cd73e325028
+    Successfully tagged patroni-citus-k8s:latest
+
+    $ kind load docker-image patroni-citus-k8s
+    Image: "" with ID "sha256:8cd73e325028d7147672494965e53453f5540400928caac0305015eb2c7027c7" not yet present on node "kind-control-plane", loading...
+
+    $ kubectl apply -f citus_k8s.yaml
+    service/citusdemo-0-config created
+    service/citusdemo-1-config created
+    service/citusdemo-2-config created
+    statefulset.apps/citusdemo-0 created
+    statefulset.apps/citusdemo-1 created
+    statefulset.apps/citusdemo-2 created
+    endpoints/citusdemo-0 created
+    service/citusdemo-0 created
+    endpoints/citusdemo-1 created
+    service/citusdemo-1 created
+    endpoints/citusdemo-2 created
+    service/citusdemo-2 created
+    service/citusdemo-workers created
+    secret/citusdemo created
+    serviceaccount/citusdemo created
+    role.rbac.authorization.k8s.io/citusdemo created
+    rolebinding.rbac.authorization.k8s.io/citusdemo created
+    clusterrole.rbac.authorization.k8s.io/patroni-k8s-ep-access created
+    clusterrolebinding.rbac.authorization.k8s.io/patroni-k8s-ep-access created
+
+    $ kubectl get sts
+    NAME          READY   AGE
+    citusdemo-0   1/3     6s  # coodinator (group=0)
+    citusdemo-1   1/2     6s  # worker (group=1)
+    citusdemo-2   1/2     6s  # worker (group=2)
+
+    $ kubectl get pods -l cluster-name=citusdemo -L role
+    NAME            READY   STATUS    RESTARTS   AGE    ROLE
+    citusdemo-0-0   1/1     Running   0          105s   master
+    citusdemo-0-1   1/1     Running   0          101s   replica
+    citusdemo-0-2   1/1     Running   0          96s    replica
+    citusdemo-1-0   1/1     Running   0          105s   master
+    citusdemo-1-1   1/1     Running   0          101s   replica
+    citusdemo-2-0   1/1     Running   0          105s   master
+    citusdemo-2-1   1/1     Running   0          101s   replica
+
+    $ kubectl exec -ti citusdemo-0-0 -- bash
+    postgres@citusdemo-0-0:~$ patronictl list
+    + Citus cluster: citusdemo -----------+--------------+---------+----+-----------+
+    | Group | Member        | Host        | Role         | State   | TL | Lag in MB |
+    +-------+---------------+-------------+--------------+---------+----+-----------+
+    |     0 | citusdemo-0-0 | 10.244.0.10 | Leader       | running |  1 |           |
+    |     0 | citusdemo-0-1 | 10.244.0.12 | Replica      | running |  1 |         0 |
+    |     0 | citusdemo-0-2 | 10.244.0.14 | Sync Standby | running |  1 |         0 |
+    |     1 | citusdemo-1-0 | 10.244.0.8  | Leader       | running |  1 |           |
+    |     1 | citusdemo-1-1 | 10.244.0.11 | Sync Standby | running |  1 |         0 |
+    |     2 | citusdemo-2-0 | 10.244.0.9  | Leader       | running |  1 |           |
+    |     2 | citusdemo-2-1 | 10.244.0.13 | Sync Standby | running |  1 |         0 |
+    +-------+---------------+-------------+--------------+---------+----+-----------+
+
+    postgres@citusdemo-0-0:~$ psql citus
+    psql (15.1 (Debian 15.1-1.pgdg110+1))
+    Type "help" for help.
+
+    citus=# table pg_dist_node;
+     nodeid | groupid |  nodename   | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards
+    --------+---------+-------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------
+          1 |       0 | 10.244.0.10 |     5432 | default  | t           | t        | primary  | default     | t              | f
+          2 |       1 | 10.244.0.8  |     5432 | default  | t           | t        | primary  | default     | t              | t
+          3 |       2 | 10.244.0.9  |     5432 | default  | t           | t        | primary  | default     | t              | t
+    (3 rows)

kubernetes/citus_k8s.yaml

@@ -0,0 +1,590 @@
+# headless services to avoid deletion of citusdemo-*-config endpoints
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-0-config
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '0'
+spec:
+  clusterIP: None
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-1-config
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '1'
+spec:
+  clusterIP: None
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-2-config
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '2'
+spec:
+  clusterIP: None
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: &cluster_name citusdemo-0
+  labels: &labels
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '0'
+    citus-type: coordinator
+spec:
+  replicas: 3
+  serviceName: *cluster_name
+  selector:
+    matchLabels:
+      <<: *labels
+  template:
+    metadata:
+      labels:
+        <<: *labels
+    spec:
+      serviceAccountName: citusdemo
+      containers:
+      - name: *cluster_name
+        image: patroni-citus-k8s  # docker build -f Dockerfile.citus -t patroni-citus-k8s .
+        imagePullPolicy: IfNotPresent
+        readinessProbe:
+          httpGet:
+            scheme: HTTP
+            path: /readiness
+            port: 8008
+          initialDelaySeconds: 3
+          periodSeconds: 10
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 3
+        ports:
+        - containerPort: 8008
+          protocol: TCP
+        - containerPort: 5432
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /home/postgres/pgdata
+          name: pgdata
+        env:
+        - name: PATRONI_KUBERNETES_POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: PATRONI_KUBERNETES_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: PATRONI_KUBERNETES_BYPASS_API_SERVICE
+          value: 'true'
+        - name: PATRONI_KUBERNETES_USE_ENDPOINTS
+          value: 'true'
+        - name: PATRONI_KUBERNETES_LABELS
+          value: '{application: patroni, cluster-name: citusdemo}'
+        - name: PATRONI_CITUS_DATABASE
+          value: citus
+        - name: PATRONI_CITUS_GROUP
+          value: '0'
+        - name: PATRONI_SUPERUSER_USERNAME
+          value: postgres
+        - name: PATRONI_SUPERUSER_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: citusdemo
+              key: superuser-password
+        - name: PATRONI_REPLICATION_USERNAME
+          value: standby
+        - name: PATRONI_REPLICATION_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: citusdemo
+              key: replication-password
+        - name: PATRONI_SCOPE
+          value: citusdemo
+        - name: PATRONI_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: PATRONI_POSTGRESQL_DATA_DIR
+          value: /home/postgres/pgdata/pgroot/data
+        - name: PATRONI_POSTGRESQL_PGPASS
+          value: /tmp/pgpass
+        - name: PATRONI_POSTGRESQL_LISTEN
+          value: '0.0.0.0:5432'
+        - name: PATRONI_RESTAPI_LISTEN
+          value: '0.0.0.0:8008'
+      terminationGracePeriodSeconds: 0
+      volumes:
+      - name: pgdata
+        emptyDir: {}
+#  volumeClaimTemplates:
+#  - metadata:
+#      labels:
+#        application: spilo
+#        spilo-cluster: *cluster_name
+#      annotations:
+#        volume.alpha.kubernetes.io/storage-class: anything
+#      name: pgdata
+#    spec:
+#      accessModes:
+#      - ReadWriteOnce
+#      resources:
+#        requests:
+#          storage: 5Gi
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: &cluster_name citusdemo-1
+  labels: &labels
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '1'
+    citus-type: worker
+spec:
+  replicas: 2
+  serviceName: *cluster_name
+  selector:
+    matchLabels:
+      <<: *labels
+  template:
+    metadata:
+      labels:
+        <<: *labels
+    spec:
+      serviceAccountName: citusdemo
+      containers:
+      - name: *cluster_name
+        image: patroni-citus-k8s  # docker build -f Dockerfile.citus -t patroni-citus-k8s .
+        imagePullPolicy: IfNotPresent
+        readinessProbe:
+          httpGet:
+            scheme: HTTP
+            path: /readiness
+            port: 8008
+          initialDelaySeconds: 3
+          periodSeconds: 10
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 3
+        ports:
+        - containerPort: 8008
+          protocol: TCP
+        - containerPort: 5432
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /home/postgres/pgdata
+          name: pgdata
+        env:
+        - name: PATRONI_KUBERNETES_POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: PATRONI_KUBERNETES_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: PATRONI_KUBERNETES_BYPASS_API_SERVICE
+          value: 'true'
+        - name: PATRONI_KUBERNETES_USE_ENDPOINTS
+          value: 'true'
+        - name: PATRONI_KUBERNETES_LABELS
+          value: '{application: patroni, cluster-name: citusdemo}'
+        - name: PATRONI_CITUS_DATABASE
+          value: citus
+        - name: PATRONI_CITUS_GROUP
+          value: '1'
+        - name: PATRONI_SUPERUSER_USERNAME
+          value: postgres
+        - name: PATRONI_SUPERUSER_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: citusdemo
+              key: superuser-password
+        - name: PATRONI_REPLICATION_USERNAME
+          value: standby
+        - name: PATRONI_REPLICATION_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: citusdemo
+              key: replication-password
+        - name: PATRONI_SCOPE
+          value: citusdemo
+        - name: PATRONI_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: PATRONI_POSTGRESQL_DATA_DIR
+          value: /home/postgres/pgdata/pgroot/data
+        - name: PATRONI_POSTGRESQL_PGPASS
+          value: /tmp/pgpass
+        - name: PATRONI_POSTGRESQL_LISTEN
+          value: '0.0.0.0:5432'
+        - name: PATRONI_RESTAPI_LISTEN
+          value: '0.0.0.0:8008'
+      terminationGracePeriodSeconds: 0
+      volumes:
+      - name: pgdata
+        emptyDir: {}
+#  volumeClaimTemplates:
+#  - metadata:
+#      labels:
+#        application: spilo
+#        spilo-cluster: *cluster_name
+#      annotations:
+#        volume.alpha.kubernetes.io/storage-class: anything
+#      name: pgdata
+#    spec:
+#      accessModes:
+#      - ReadWriteOnce
+#      resources:
+#        requests:
+#          storage: 5Gi
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: &cluster_name citusdemo-2
+  labels: &labels
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '2'
+    citus-type: worker
+spec:
+  replicas: 2
+  serviceName: *cluster_name
+  selector:
+    matchLabels:
+      <<: *labels
+  template:
+    metadata:
+      labels:
+        <<: *labels
+    spec:
+      serviceAccountName: citusdemo
+      containers:
+      - name: *cluster_name
+        image: patroni-citus-k8s  # docker build -f Dockerfile.citus -t patroni-citus-k8s .
+        imagePullPolicy: IfNotPresent
+        readinessProbe:
+          httpGet:
+            scheme: HTTP
+            path: /readiness
+            port: 8008
+          initialDelaySeconds: 3
+          periodSeconds: 10
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 3
+        ports:
+        - containerPort: 8008
+          protocol: TCP
+        - containerPort: 5432
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /home/postgres/pgdata
+          name: pgdata
+        env:
+        - name: PATRONI_KUBERNETES_POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: PATRONI_KUBERNETES_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: PATRONI_KUBERNETES_BYPASS_API_SERVICE
+          value: 'true'
+        - name: PATRONI_KUBERNETES_USE_ENDPOINTS
+          value: 'true'
+        - name: PATRONI_KUBERNETES_LABELS
+          value: '{application: patroni, cluster-name: citusdemo}'
+        - name: PATRONI_CITUS_DATABASE
+          value: citus
+        - name: PATRONI_CITUS_GROUP
+          value: '2'
+        - name: PATRONI_SUPERUSER_USERNAME
+          value: postgres
+        - name: PATRONI_SUPERUSER_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: citusdemo
+              key: superuser-password
+        - name: PATRONI_REPLICATION_USERNAME
+          value: standby
+        - name: PATRONI_REPLICATION_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: citusdemo
+              key: replication-password
+        - name: PATRONI_SCOPE
+          value: citusdemo
+        - name: PATRONI_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: PATRONI_POSTGRESQL_DATA_DIR
+          value: /home/postgres/pgdata/pgroot/data
+        - name: PATRONI_POSTGRESQL_PGPASS
+          value: /tmp/pgpass
+        - name: PATRONI_POSTGRESQL_LISTEN
+          value: '0.0.0.0:5432'
+        - name: PATRONI_RESTAPI_LISTEN
+          value: '0.0.0.0:8008'
+      terminationGracePeriodSeconds: 0
+      volumes:
+      - name: pgdata
+        emptyDir: {}
+#  volumeClaimTemplates:
+#  - metadata:
+#      labels:
+#        application: spilo
+#        spilo-cluster: *cluster_name
+#      annotations:
+#        volume.alpha.kubernetes.io/storage-class: anything
+#      name: pgdata
+#    spec:
+#      accessModes:
+#      - ReadWriteOnce
+#      resources:
+#        requests:
+#          storage: 5Gi
+          
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: citusdemo-0
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '0'
+    citus-type: coordinator
+subsets: []
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-0
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '0'
+    citus-type: coordinator
+spec:
+  type: ClusterIP
+  ports:
+  - port: 5432
+    targetPort: 5432
+
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: citusdemo-1
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '1'
+    citus-type: worker
+subsets: []
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-1
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '1'
+    citus-type: worker
+spec:
+  type: ClusterIP
+  ports:
+  - port: 5432
+    targetPort: 5432
+
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: citusdemo-2
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '2'
+    citus-type: worker
+subsets: []
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-2
+  labels:
+    application: patroni
+    cluster-name: citusdemo
+    citus-group: '2'
+    citus-type: worker
+spec:
+  type: ClusterIP
+  ports:
+  - port: 5432
+    targetPort: 5432
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: citusdemo-workers
+  labels: &labels
+    application: patroni
+    cluster-name: citusdemo
+    citus-type: worker
+    role: master
+spec:
+  type: ClusterIP
+  selector:
+    <<: *labels
+  ports:
+  - port: 5432
+    targetPort: 5432
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: &cluster_name citusdemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+type: Opaque
+data:
+  superuser-password: emFsYW5kbw==
+  replication-password: cmVwLXBhc3M=
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: citusdemo
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: citusdemo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
+  # delete and deletecollection are required only for 'patronictl remove'
+  - delete
+  - deletecollection
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - patch
+  - update
+  # the following three privileges are necessary only when using endpoints
+  - create
+  - list
+  - watch
+  # delete and deletecollection are required only for for 'patronictl remove'
+  - delete
+  - deletecollection
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+# The following privilege is only necessary for creation of headless service
+# for citusdemo-config endpoint, in order to prevent cleaning it up by the
+# k8s master. You can avoid giving this privilege by explicitly creating the
+# service like it is done in this manifest (lines 2..10)
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: citusdemo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: citusdemo
+subjects:
+- kind: ServiceAccount
+  name: citusdemo
+
+# Following privileges are only required if deployed not in the "default"
+# namespace and you want Patroni to bypass kubernetes service
+# (PATRONI_KUBERNETES_BYPASS_API_SERVICE=true)
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: patroni-k8s-ep-access
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  resourceNames:
+  - kubernetes
+  verbs:
+  - get
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: patroni-k8s-ep-access
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: patroni-k8s-ep-access
+subjects:
+- kind: ServiceAccount
+  name: citusdemo
+# The namespace must be specified explicitly.
+# If deploying to the different namespace you have to change it.
+  namespace: default

kubernetes/openshift-example/README.md

@@ -1,5 +1,5 @@
 # Patroni OpenShift Configuration
-Patroni can be run in OpenShift. Based on the kubernetes configuration, the Dockerfile and Entrypoint has been modified to support the dynamic UID/GID configuration that is applied in OpenShift. This can be run under the standard `restricted` SCC. 
+Patroni can be run in OpenShift. Based on the kubernetes configuration, the Dockerfile and Entrypoint has been modified to support the dynamic UID/GID configuration that is applied in OpenShift. This can be run under the standard `restricted` SCC.
 
 # Examples
 
@@ -11,39 +11,39 @@ oc new-project patroni-test
 
 ## Build the image
 
-Note: Update the references when merged upstream. 
-Note: If deploying as a template for multiple users, the following commands should be performed in a shared namespace like `openshift`. 
+Note: Update the references when merged upstream.
+Note: If deploying as a template for multiple users, the following commands should be performed in a shared namespace like `openshift`.
 
 ```
 oc import-image postgres:10 --confirm -n openshift
 oc new-build https://github.com/zalando/patroni --context-dir=kubernetes -n openshift
 ```
 
-## Deploy the Image 
-Two configuration templates exist in [templates](templates) directory: 
-- Patroni Ephemeral
-- Patroni Persistent
+## Deploy the Image
+Two configuration templates exist in [templates](templates) directory:
+-   Patroni Ephemeral
+-   Patroni Persistent
 
-The only difference is whether or not the statefulset requests persistent storage. 
+The only difference is whether or not the statefulset requests persistent storage.
 
 ## Create the Template
-Install the template into the `openshift` namespace if this should be shared across projects: 
+Install the template into the `openshift` namespace if this should be shared across projects:
 
 ```
 oc create -f templates/template_patroni_ephemeral.yml -n openshift
 ```
 
-Then, from your own project: 
+Then, from your own project:
 
 ```
 oc new-app patroni-pgsql-ephemeral
 ```
 
-Once the pods are running, two configmaps should be available: 
+Once the pods are running, two configmaps should be available:
 
 ```
 $ oc get configmap
 NAME                DATA      AGE
 patroniocp-config   0         1m
 patroniocp-leader   0         1m
-```
+```

kubernetes/openshift-example/test/README.md

@@ -1,2 +1,2 @@
 # Jenkins Test
-This pipeline test will create a separate deployment config for a pgbench pod and execute a test against the patroni cluster. This is a sample and should be customized. 
+This pipeline test will create a separate deployment config for a pgbench pod and execute a test against the patroni cluster. This is a sample and should be customized.

mkbinary.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 set -e
 
-pip install --ignore-installed setuptools==19.2 pyinstaller
-pyinstaller --clean --onefile patroni.spec
+pip install --ignore-installed pyinstaller
+pyinstaller --clean patroni.spec

patroni.spec

@@ -8,7 +8,7 @@ def hiddenimports():
     sys.path.insert(0, '.')
     try:
         import patroni.dcs
-        return patroni.dcs.dcs_modules()
+        return patroni.dcs.dcs_modules() + ['http.server']
     finally:
         sys.path.pop(0)
 

patroni/__main__.py

@@ -3,7 +3,7 @@ import os
 import signal
 import time
 
-from .daemon import AbstractPatroniDaemon, abstract_main
+from patroni.daemon import AbstractPatroniDaemon, abstract_main
 
 logger = logging.getLogger(__name__)
 
@@ -11,13 +11,13 @@ logger = logging.getLogger(__name__)
 class Patroni(AbstractPatroniDaemon):
 
     def __init__(self, config):
-        from .api import RestApiServer
-        from .dcs import get_dcs
-        from .ha import Ha
-        from .postgresql import Postgresql
-        from .request import PatroniRequest
-        from .version import __version__
-        from .watchdog import Watchdog
+        from patroni.api import RestApiServer
+        from patroni.dcs import get_dcs
+        from patroni.ha import Ha
+        from patroni.postgresql import Postgresql
+        from patroni.request import PatroniRequest
+        from patroni.version import __version__
+        from patroni.watchdog import Watchdog
 
         super(Patroni, self).__init__(config)
 
@@ -47,6 +47,7 @@ class Patroni(AbstractPatroniDaemon):
                 elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
                     if self.config.set_dynamic_configuration(self.config['bootstrap']['dcs']):
                         self.dcs.reload_config(self.config)
+                        self.watchdog.reload_config(self.config)
                 break
             except DCSError:
                 logger.warning('Can not get cluster from dcs')
@@ -137,7 +138,7 @@ def patroni_main():
 
 def main():
     if os.getpid() != 1:
-        from . import check_psycopg
+        from patroni import check_psycopg
 
         check_psycopg()
         return patroni_main()

patroni/api.py

@@ -38,6 +38,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         self.log_request(status_code)
 
     def _write_response(self, status_code, body, content_type='text/html', headers=None):
+        # TODO: try-catch ConnectionResetError: [Errno 104] Connection reset by peer and log it in DEBUG level
         self.send_response(status_code)
         headers = headers or {}
         if content_type:
@@ -96,7 +97,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_GET(self, write_status_code_only=False):
         """Default method for processing all GET requests which can not be routed to other methods"""
 
-        path = '/master' if self.path == '/' else self.path
+        path = '/primary' if self.path == '/' else self.path
         response = self.get_postgresql_status()
 
         patroni = self.server.patroni
@@ -113,8 +114,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response.get('role') == 'replica' and response.get('state') == 'running' else 503
 
         if not cluster and patroni.ha.is_paused():
-            leader_status_code = 200 if response.get('role') in ('master', 'standby_leader') else 503
-            primary_status_code = 200 if response.get('role') == 'master' else 503
+            leader_status_code = 200 if response.get('role') in ('master', 'primary', 'standby_leader') else 503
+            primary_status_code = 200 if response.get('role') in ('master', 'primary') else 503
             standby_leader_status_code = 200 if response.get('role') == 'standby_leader' else 503
         elif patroni.ha.is_leader():
             leader_status_code = 200
@@ -141,7 +142,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
             ignore_tags = True
         elif 'replica' in path:
             status_code = replica_status_code
-        elif 'read-only' in path:
+        elif 'read-only' in path and 'sync' not in path:
             status_code = 200 if 200 in (primary_status_code, standby_leader_status_code) else replica_status_code
         elif 'health' in path:
             status_code = 200 if response.get('state') == 'running' else 503
@@ -185,8 +186,20 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_OPTIONS(self):
         self.do_GET(write_status_code_only=True)
 
+    def do_HEAD(self):
+        self.do_GET(write_status_code_only=True)
+
     def do_GET_liveness(self):
-        self._write_status_code_only(200)
+        patroni = self.server.patroni
+        is_primary = patroni.postgresql.role in ('master', 'primary') and patroni.postgresql.is_running()
+        # We can tolerate Patroni problems longer on the replica.
+        # On the primary the liveness probe most likely will start failing only after the leader key expired.
+        # It should not be a big problem because replicas will see that the primary is still alive via REST API call.
+        liveness_threshold = patroni.dcs.ttl * (1 if is_primary else 2)
+
+        # In maintenance mode (pause) we are fine if heartbeat loop stuck.
+        status_code = 200 if patroni.ha.is_paused() or patroni.next_run + liveness_threshold > time.time() else 503
+        self._write_status_code_only(status_code)
 
     def do_GET_readiness(self):
         patroni = self.server.patroni
@@ -242,7 +255,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         metrics.append("# HELP patroni_master Value is 1 if this node is the leader, 0 otherwise.")
         metrics.append("# TYPE patroni_master gauge")
-        metrics.append("patroni_master{0} {1}".format(scope_label, int(postgres['role'] == 'master')))
+        metrics.append("patroni_master{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary'))))
+
+        metrics.append("# HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise.")
+        metrics.append("# TYPE patroni_primary gauge")
+        metrics.append("patroni_primary{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary'))))
 
         metrics.append("# HELP patroni_xlog_location Current location of the Postgres"
                        " transaction log, 0 if this node is not the leader.")
@@ -289,6 +306,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
         metrics.append("# TYPE patroni_cluster_unlocked gauge")
         metrics.append("patroni_cluster_unlocked{0} {1}".format(scope_label, int(postgres.get('cluster_unlocked', 0))))
 
+        metrics.append("# HELP patroni_failsafe_mode_is_active Value is 1 if the cluster is unlocked, 0 if locked.")
+        metrics.append("# TYPE patroni_failsafe_mode_is_active gauge")
+        metrics.append("patroni_failsafe_mode_is_active{0} {1}"
+                       .format(scope_label, int(postgres.get('failsafe_mode_is_active', 0))))
+
         metrics.append("# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.")
         metrics.append("# TYPE patroni_postgres_timeline counter")
         metrics.append("patroni_postgres_timeline{0} {1}".format(scope_label, postgres.get('timeline', 0)))
@@ -355,6 +377,32 @@ class RestApiHandler(BaseHTTPRequestHandler):
         self.server.patroni.sighup_handler()
         self._write_response(202, 'reload scheduled')
 
+    def do_GET_failsafe(self):
+        failsafe = self.server.patroni.dcs.failsafe
+        if isinstance(failsafe, dict):
+            self._write_json_response(200, failsafe)
+        else:
+            self.send_error(502)
+
+    @check_access
+    def do_POST_failsafe(self):
+        if self.server.patroni.ha.is_failsafe_mode():
+            request = self._read_json_content()
+            if request:
+                message = self.server.patroni.ha.update_failsafe(request) or 'Accepted'
+                code = 200 if message == 'Accepted' else 500
+                self._write_response(code, message)
+        else:
+            self.send_error(502)
+
+    @check_access
+    def do_POST_sigterm(self):
+        """Only for behave testing on windows"""
+
+        if os.name == 'nt' and os.getenv('BEHAVE_DEBUG'):
+            self.server.patroni.api_sigterm()
+        self._write_response(202, 'shutdown scheduled')
+
     @staticmethod
     def parse_schedule(schedule, action):
         """ parses the given schedule and validates at """
@@ -399,9 +447,9 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     status_code = _
                     break
             elif k == 'role':
-                if request[k] not in ('master', 'replica'):
+                if request[k] not in ('master', 'primary', 'replica'):
                     status_code = 400
-                    data = "PostgreSQL role should be either master or replica"
+                    data = "PostgreSQL role should be either primary or replica"
                     break
             elif k == 'postgres_version':
                 try:
@@ -571,6 +619,18 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_POST_switchover(self):
         self.do_POST_failover(action='switchover')
 
+    @check_access
+    def do_POST_citus(self):
+        request = self._read_json_content()
+        if not request:
+            return
+
+        patroni = self.server.patroni
+        if patroni.postgresql.citus_handler.is_coordinator() and patroni.ha.is_leader():
+            cluster = patroni.dcs.get_cluster(True)
+            patroni.postgresql.citus_handler.handle_event(cluster, request)
+        self._write_response(200, 'OK')
+
     def parse_request(self):
         """Override parse_request method to enrich basic functionality of `BaseHTTPRequestHandler` class
 
@@ -608,7 +668,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
             stmt = ("SELECT " + postgresql.POSTMASTER_START_TIME + ", " + postgresql.TL_LSN + ","
                     " pg_catalog.pg_last_xact_replay_timestamp(),"
                     " pg_catalog.array_to_json(pg_catalog.array_agg(pg_catalog.row_to_json(ri))) "
-                    "FROM (SELECT (SELECT rolname FROM pg_authid WHERE oid = usesysid) AS usename,"
+                    "FROM (SELECT (SELECT rolname FROM pg_catalog.pg_authid WHERE oid = usesysid) AS usename,"
                     " application_name, client_addr, w.state, sync_state, sync_priority"
                     " FROM pg_catalog.pg_stat_get_wal_senders() w, pg_catalog.pg_stat_get_activity(pid)) AS ri")
 
@@ -649,6 +709,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         if not cluster or cluster.is_unlocked():
             result['cluster_unlocked'] = True
+        if self.server.patroni.ha.failsafe_is_active():
+            result['failsafe_mode_is_active'] = True
         result['dcs_last_seen'] = self.server.patroni.dcs.last_seen
         return result
 
@@ -713,16 +775,17 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
     def __members_ips(self):
         cluster = self.patroni.dcs.cluster
         if self.__allowlist_include_members and cluster:
-            for member in cluster.members:
-                if member.api_url:
-                    try:
-                        r = urlparse(member.api_url)
-                        host = r.hostname
-                        port = r.port or (443 if r.scheme == 'https' else 80)
-                        for ip in self.__resolve_ips(host, port):
-                            yield ip
-                    except Exception as e:
-                        logger.debug('Failed to parse url %s: %r', member.api_url, e)
+            for cluster in [cluster] + list(cluster.workers.values()):
+                for member in cluster.members:
+                    if member.api_url:
+                        try:
+                            r = urlparse(member.api_url)
+                            host = r.hostname
+                            port = r.port or (443 if r.scheme == 'https' else 80)
+                            for ip in self.__resolve_ips(host, port):
+                                yield ip
+                        except Exception as e:
+                            logger.debug('Failed to parse url %s: %r', member.api_url, e)
 
     def check_access(self, rh):
         if self.__allowlist or self.__allowlist_include_members:
@@ -812,32 +875,23 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
                 else:
                     logger.error('Bad value in the "restapi.verify_client": %s', verify_client)
             self.__ssl_serial_number = self.get_certificate_serial_number()
-            self.socket = ctx.wrap_socket(self.socket, server_side=True)
+            self.socket = ctx.wrap_socket(self.socket, server_side=True, do_handshake_on_connect=False)
         if reloading_config:
             self.start()
 
     def process_request_thread(self, request, client_address):
-        if isinstance(request, tuple):
-            sock, newsock = request
-            try:
-                request = sock.context.wrap_socket(newsock, do_handshake_on_connect=sock.do_handshake_on_connect,
-                                                   suppress_ragged_eofs=sock.suppress_ragged_eofs, server_side=True)
-            except socket.error:
-                return
+        enable_keepalive(request, 10, 3)
+        if hasattr(request, 'context'):  # SSLSocket
+            request.do_handshake()
         super(RestApiServer, self).process_request_thread(request, client_address)
 
-    def get_request(self):
-        sock = self.socket
-        newsock, addr = socket.socket.accept(sock)
-        enable_keepalive(newsock, 10, 3)
-        if hasattr(sock, 'context'):  # SSLSocket, we want to do the deferred handshake from a thread
-            newsock = (sock, newsock)
-        return newsock, addr
-
     def shutdown_request(self, request):
-        if isinstance(request, tuple):
-            _, request = request  # SSLSocket
-        return super(RestApiServer, self).shutdown_request(request)
+        if hasattr(request, 'context'):  # SSLSocket
+            try:
+                request.unwrap()
+            except Exception as e:
+                logger.debug('Failed to shutdown SSL connection: %r', e)
+        super(RestApiServer, self).shutdown_request(request)
 
     def get_certificate_serial_number(self):
         if self.__ssl_options.get('certfile'):

patroni/config.py

@@ -2,6 +2,7 @@ import json
 import logging
 import os
 import shutil
+import six
 import tempfile
 import yaml
 
@@ -32,7 +33,7 @@ _AUTH_ALLOWED_PARAMETERS = (
 
 def default_validator(conf):
     if not conf:
-        return "Config is empty."
+        raise ConfigParseError("Config is empty.")
 
 
 class Config(object):
@@ -58,16 +59,21 @@ class Config(object):
     PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION'
 
     __CACHE_FILENAME = 'patroni.dynamic.json'
+    __REMAP_KEYS = {
+        'master_start_timeout': 'primary_start_timeout',
+        'master_stop_timeout': 'primary_stop_timeout'
+    }
     __DEFAULT_CONFIG = {
         'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10,
         'maximum_lag_on_failover': 1048576,
         'maximum_lag_on_syncnode': -1,
         'check_timeline': False,
-        'master_start_timeout': 300,
-        'master_stop_timeout': 0,
+        'primary_start_timeout': 300,
+        'primary_stop_timeout': 0,
         'synchronous_mode': False,
         'synchronous_mode_strict': False,
         'synchronous_node_count': 1,
+        'failsafe_mode': False,
         'standby_cluster': {
             'create_replica_methods': '',
             'host': '',
@@ -102,9 +108,9 @@ class Config(object):
             config_env = os.environ.pop(self.PATRONI_CONFIG_VARIABLE, None)
             self._local_configuration = config_env and yaml.safe_load(config_env) or self.__environment_configuration
         if validator:
-            error = validator(self._local_configuration)
-            if error:
-                raise ConfigParseError(error)
+            errors = validator(self._local_configuration)
+            if errors:
+                raise ConfigParseError("\n".join(errors))
 
         self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration)
         self._data_dir = self.__effective_configuration.get('postgresql', {}).get('data_dir', "")
@@ -223,18 +229,22 @@ class Config(object):
         config = deepcopy(self.__DEFAULT_CONFIG)
 
         for name, value in dynamic_configuration.items():
+            # allow copying master_start_timeout->primary_start_timeout when the latter isn't in dynamic_configuration
+            if name in self.__REMAP_KEYS and self.__REMAP_KEYS[name] not in dynamic_configuration:
+                name = self.__REMAP_KEYS[name]
             if name == 'postgresql':
                 for name, value in (value or {}).items():
                     if name == 'parameters':
                         config['postgresql'][name].update(self._process_postgresql_parameters(value))
-                    elif name not in ('connect_address', 'listen', 'data_dir', 'pgpass', 'authentication'):
+                    elif name not in ('connect_address', 'proxy_address', 'listen',
+                                      'config_dir', 'data_dir', 'pgpass', 'authentication'):
                         config['postgresql'][name] = deepcopy(value)
             elif name == 'standby_cluster':
                 for name, value in (value or {}).items():
                     if name in self.__DEFAULT_CONFIG['standby_cluster']:
                         config['standby_cluster'][name] = deepcopy(value)
             elif name in config:  # only variables present in __DEFAULT_CONFIG allowed to be overridden from DCS
-                if name in ('synchronous_mode', 'synchronous_mode_strict'):
+                if name in ('synchronous_mode', 'synchronous_mode_strict', 'failsafe_mode'):
                     config[name] = value
                 else:
                     config[name] = int(value)
@@ -271,7 +281,8 @@ class Config(object):
                                         'cafile', 'ciphers', 'verify_client', 'http_extra_headers',
                                         'https_extra_headers', 'allowlist', 'allowlist_include_members'])
         _set_section_values('ctl', ['insecure', 'cacert', 'certfile', 'keyfile', 'keyfile_password'])
-        _set_section_values('postgresql', ['listen', 'connect_address', 'config_dir', 'data_dir', 'pgpass', 'bin_dir'])
+        _set_section_values('postgresql', ['listen', 'connect_address', 'proxy_address',
+                                           'config_dir', 'data_dir', 'pgpass', 'bin_dir'])
         _set_section_values('log', ['level', 'traceback_level', 'format', 'dateformat', 'max_queue_size',
                                     'dir', 'file_size', 'file_num', 'loggers'])
         _set_section_values('raft', ['data_dir', 'self_addr', 'partner_addrs', 'password', 'bind_addr'])
@@ -351,18 +362,24 @@ class Config(object):
                 if suffix in ('HOST', 'HOSTS', 'PORT', 'USE_PROXIES', 'PROTOCOL', 'SRV', 'SRV_SUFFIX', 'URL', 'PROXY',
                               'CACERT', 'CERT', 'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'CONSISTENCY',
                               'REGISTER_SERVICE', 'SERVICE_CHECK_INTERVAL', 'SERVICE_CHECK_TLS_SERVER_NAME',
-                              'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP',
-                              'PORTS', 'LABELS', 'BYPASS_API_SERVICE', 'KEY_PASSWORD', 'USE_SSL', 'SET_ACLS') and name:
+                              'SERVICE_TAGS', 'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL',
+                              'POD_IP', 'PORTS', 'LABELS', 'BYPASS_API_SERVICE', 'KEY_PASSWORD', 'USE_SSL', 'SET_ACLS',
+                              'GROUP', 'DATABASE') and name:
                     value = os.environ.pop(param)
-                    if suffix == 'PORT':
+                    if name == 'CITUS':
+                        if suffix == 'GROUP':
+                            value = parse_int(value)
+                        elif suffix != 'DATABASE':
+                            continue
+                    elif suffix == 'PORT':
                         value = value and parse_int(value)
-                    elif suffix in ('HOSTS', 'PORTS', 'CHECKS'):
+                    elif suffix in ('HOSTS', 'PORTS', 'CHECKS', 'SERVICE_TAGS'):
                         value = value and _parse_list(value)
                     elif suffix in ('LABELS', 'SET_ACLS'):
                         value = _parse_dict(value)
                     elif suffix in ('USE_PROXIES', 'REGISTER_SERVICE', 'USE_ENDPOINTS', 'BYPASS_API_SERVICE', 'VERIFY'):
                         value = parse_bool(value)
-                    if value:
+                    if value is not None:
                         ret[name.lower()][suffix.lower()] = value
         for dcs in ('etcd', 'etcd3'):
             if dcs in ret:
@@ -390,7 +407,11 @@ class Config(object):
     def _build_effective_configuration(self, dynamic_configuration, local_configuration):
         config = self._safe_copy_dynamic_configuration(dynamic_configuration)
         for name, value in local_configuration.items():
-            if name == 'postgresql':
+            if name == 'citus':  # remove invalid citus configuration
+                if isinstance(value, dict) and isinstance(value.get('group'), six.integer_types)\
+                        and isinstance(value.get('database'), six.string_types):
+                    config[name] = value
+            elif name == 'postgresql':
                 for name, value in (value or {}).items():
                     if name == 'parameters':
                         config['postgresql'][name].update(self._process_postgresql_parameters(value, True))
@@ -428,6 +449,12 @@ class Config(object):
         if 'name' not in config and 'name' in pg_config:
             config['name'] = pg_config['name']
 
+        # when bootstrapping the new Citus cluster (coordinator/worker) enable sync replication in global configuration
+        if 'citus' in config:
+            bootstrap = config.setdefault('bootstrap', {})
+            dcs = bootstrap.setdefault('dcs', {})
+            dcs.setdefault('synchronous_mode', True)
+
         updated_fields = (
             'name',
             'scope',
@@ -435,7 +462,8 @@ class Config(object):
             'synchronous_mode',
             'synchronous_mode_strict',
             'synchronous_node_count',
-            'maximum_lag_on_syncnode'
+            'maximum_lag_on_syncnode',
+            'citus'
         )
 
         pg_config.update({p: config[p] for p in updated_fields if p in config})

patroni/ctl.py

@@ -34,9 +34,8 @@ except ImportError:  # pragma: no cover
 
 from .dcs import get_dcs as _get_dcs
 from .exceptions import PatroniException
-from .postgresql import Postgresql
 from .postgresql.misc import postgres_version_to_int
-from .utils import cluster_as_json, find_executable, patch_config, polling_loop
+from .utils import cluster_as_json, find_executable, patch_config, polling_loop, is_standby_cluster
 from .request import PatroniRequest
 from .version import __version__
 
@@ -60,6 +59,18 @@ class PatronictlPrettyTable(PrettyTable):
         self.__hline_num = 0
         self.__hline = None
 
+    def __build_header(self, line):
+        header = self.__table_header[:len(line) - 2]
+        return "".join([line[0], header, line[1 + len(header):]])
+
+    def _stringify_hrule(self, *args, **kwargs):
+        ret = super(PatronictlPrettyTable, self)._stringify_hrule(*args, **kwargs)
+        where = args[1] if len(args) > 1 else kwargs.get('where')
+        if where == 'top_' and self.__table_header:
+            ret = self.__build_header(ret)
+            self.__hline_num += 1
+        return ret
+
     def _is_first_hline(self):
         return self.__hline_num == 0
 
@@ -71,8 +82,7 @@ class PatronictlPrettyTable(PrettyTable):
 
         # Inject nice table header
         if self._is_first_hline() and self.__table_header:
-            header = self.__table_header[:len(ret) - 2]
-            ret = "".join([ret[0], header, ret[1 + len(header):]])
+            ret = self.__build_header(ret)
 
         self.__hline_num += 1
         return ret
@@ -99,7 +109,7 @@ def parse_dcs(dcs):
     return yaml.safe_load(default['template'].format(host=parsed.hostname or 'localhost', port=port or default['port']))
 
 
-def load_config(path, dcs):
+def load_config(path, dcs_url):
     from patroni.config import Config
 
     if not (os.path.exists(path) and os.access(path, os.R_OK)):
@@ -112,53 +122,54 @@ def load_config(path, dcs):
         logging.debug('Loading configuration from file %s', path)
     config = Config(path, validator=None).copy()
 
-    dcs = parse_dcs(dcs) or parse_dcs(config.get('dcs_api')) or {}
-    if dcs:
+    dcs_url = parse_dcs(dcs_url) or {}
+    if dcs_url:
         for d in DCS_DEFAULTS:
             config.pop(d, None)
-        config.update(dcs)
+        config.update(dcs_url)
     return config
 
 
-def store_config(config, path):
-    dir_path = os.path.dirname(path)
-    if dir_path and not os.path.isdir(dir_path):
-        os.makedirs(dir_path)
-    with open(path, 'w') as fd:
-        yaml.dump(config, fd)
-
-
 option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, tsv, json, yaml)', default='pretty')
 option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
 option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
 option_force = click.option('--force', is_flag=True, help='Do not ask for confirmation at any point')
 arg_cluster_name = click.argument('cluster_name', required=False,
                                   default=lambda: click.get_current_context().obj.get('scope'))
+option_default_citus_group = click.option('--group', required=False, type=int, help='Citus group',
+                                          default=lambda: click.get_current_context().obj.get('citus', {}).get('group'))
+option_citus_group = click.option('--group', required=False, type=int, help='Citus group')
 option_insecure = click.option('-k', '--insecure', is_flag=True, help='Allow connections to SSL sites without certs')
+role_choice = click.Choice(['leader', 'primary', 'standby-leader', 'replica', 'standby', 'any', 'master'])
 
 
 @click.group()
 @click.option('--config-file', '-c', help='Configuration file',
               envvar='PATRONICTL_CONFIG_FILE', default=CONFIG_FILE_PATH)
-@click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
+@click.option('--dcs-url', '--dcs', '-d', 'dcs_url', help='The DCS connect url', envvar='DCS_URL')
 @option_insecure
 @click.pass_context
-def ctl(ctx, config_file, dcs, insecure):
+def ctl(ctx, config_file, dcs_url, insecure):
     level = 'WARNING'
     for name in ('LOGLEVEL', 'PATRONI_LOGLEVEL', 'PATRONI_LOG_LEVEL'):
         level = os.environ.get(name, level)
     logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=level)
     logging.captureWarnings(True)  # Capture eventual SSL warning
-    ctx.obj = load_config(config_file, dcs)
+    ctx.obj = load_config(config_file, dcs_url)
     # backward compatibility for configuration file where ctl section is not define
     ctx.obj.setdefault('ctl', {})['insecure'] = ctx.obj.get('ctl', {}).get('insecure') or insecure
 
 
-def get_dcs(config, scope):
+def get_dcs(config, scope, group):
     config.update({'scope': scope, 'patronictl': True})
+    if group is not None:
+        config['citus'] = {'group': group}
     config.setdefault('name', scope)
     try:
-        return _get_dcs(config)
+        dcs = _get_dcs(config)
+        if config.get('citus') and group is None:
+            dcs.get_cluster = dcs._get_citus_cluster
+        return dcs
     except PatroniException as e:
         raise PatroniCtlException(str(e))
 
@@ -183,9 +194,10 @@ def print_output(columns, rows, alignment=None, fmt='pretty', header=None, delim
             for row in rows:
                 if row[i]:
                     row[i] = format_config_for_editing(row[i], fmt != 'pretty').strip()
-        if list_cluster and fmt != 'tsv':  # skip cluster name if pretty-printing
-            columns = columns[1:] if columns else []
-            rows = [row[1:] for row in rows]
+        if list_cluster and fmt != 'tsv':  # skip cluster name and maybe Citus group if pretty-printing
+            skip_cols = 2 if ' (group: ' in header else 1
+            columns = columns[skip_cols:] if columns else []
+            rows = [row[skip_cols:] for row in rows]
 
         if fmt == 'tsv':
             for r in ([columns] if columns else []) + rows:
@@ -229,21 +241,29 @@ def watching(w, watch, max_count=None, clear=True):
         yield 0
 
 
-def get_all_members(cluster, role='master'):
-    if role == 'master':
-        if cluster.leader is not None and cluster.leader.name:
-            yield cluster.leader
+def get_all_members(obj, cluster, group, role='leader'):
+    clusters = {0: cluster}
+    if obj.get('citus') and group is None:
+        clusters.update(cluster.workers)
+    if role in ('leader', 'master', 'primary', 'standby-leader'):
+        role = {'primary': 'master', 'standby-leader': 'standby_leader'}.get(role, role)
+        for cluster in clusters.values():
+            if cluster.leader is not None and cluster.leader.name and\
+                    (role == 'leader' or
+                     cluster.leader.data.get('role') != 'master' and role == 'standby_leader' or
+                     cluster.leader.data.get('role') != 'standby_leader' and role == 'master'):
+                yield cluster.leader.member
         return
 
-    leader_name = (cluster.leader.member.name if cluster.leader else None)
-    for m in cluster.members:
-        if role == 'any' or role == 'replica' and m.name != leader_name:
-            yield m
+    for cluster in clusters.values():
+        leader_name = (cluster.leader.member.name if cluster.leader else None)
+        for m in cluster.members:
+            if role == 'any' or role in ('replica', 'standby') and m.name != leader_name:
+                yield m
 
 
-def get_any_member(cluster, role='master', member=None):
-    members = get_all_members(cluster, role)
-    for m in members:
+def get_any_member(obj, cluster, group, role='leader', member=None):
+    for m in get_all_members(obj, cluster, group, role):
         if member is None or m.name == member:
             return m
 
@@ -257,8 +277,8 @@ def get_all_members_leader_first(cluster):
             yield member
 
 
-def get_cursor(cluster, connect_parameters, role='master', member=None):
-    member = get_any_member(cluster, role=role, member=member)
+def get_cursor(obj, cluster, group, connect_parameters, role='leader', member=None):
+    member = get_any_member(obj, cluster, group, role=role, member=member)
     if member is None:
         return None
 
@@ -271,15 +291,15 @@ def get_cursor(cluster, connect_parameters, role='master', member=None):
 
     from . import psycopg
     conn = psycopg.connect(**params)
-    conn.autocommit = True
     cursor = conn.cursor()
-    if role == 'any':
+    if role in ('any', 'leader'):
         return cursor
 
     cursor.execute('SELECT pg_catalog.pg_is_in_recovery()')
     in_recovery = cursor.fetchone()[0]
 
-    if in_recovery and role == 'replica' or not in_recovery and role == 'master':
+    if in_recovery and role in ('replica', 'standby', 'standby-leader')\
+            or not in_recovery and role in ('master', 'primary'):
         return cursor
 
     conn.close()
@@ -287,32 +307,31 @@ def get_cursor(cluster, connect_parameters, role='master', member=None):
     return None
 
 
-def get_members(cluster, cluster_name, member_names, role, force, action, ask_confirmation=True):
-    candidates = {m.name: m for m in cluster.members}
+def get_members(obj, cluster, cluster_name, member_names, role, force, action, ask_confirmation=True, group=None):
+    members = list(get_all_members(obj, cluster, group, role))
 
+    candidates = {m.name for m in members}
     if not force or role:
         if not member_names and not candidates:
             raise PatroniCtlException('{0} cluster doesn\'t have any members'.format(cluster_name))
-        output_members(cluster, cluster_name)
+        output_members(obj, cluster, cluster_name, group=group)
 
-    if role:
-        role_names = [m.name for m in get_all_members(cluster, role)]
-        if member_names:
-            member_names = list(set(member_names) & set(role_names))
-            if not member_names:
-                raise PatroniCtlException('No {0} among provided members'.format(role))
-        else:
-            member_names = role_names
+    if member_names:
+        member_names = list(set(member_names) & candidates)
+        if not member_names:
+            raise PatroniCtlException('No {0} among provided members'.format(role))
+    elif action != 'reinitialize':
+        member_names = list(candidates)
 
     if not member_names and not force:
         member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(action,
-                        ', '.join(candidates.keys())), type=str, default='')]
+                        ', '.join(candidates)), type=str, default='')]
 
     for member_name in member_names:
         if member_name not in candidates:
             raise PatroniCtlException('{0} is not a member of cluster'.format(member_name))
 
-    members = [candidates[n] for n in member_names]
+    members = [m for m in members if m.name in member_names]
     if ask_confirmation:
         confirm_members_action(members, force, action)
     return members
@@ -333,20 +352,22 @@ def confirm_members_action(members, force, action, scheduled_at=None):
                 raise PatroniCtlException('Aborted {0}'.format(action))
 
 
-@ctl.command('dsn', help='Generate a dsn for the provided member, defaults to a dsn of the master')
-@click.option('--role', '-r', help='Give a dsn of any member with this role', type=click.Choice(['master', 'replica',
-              'any']), default=None)
+@ctl.command('dsn', help='Generate a dsn for the provided member, defaults to a dsn of the leader')
+@click.option('--role', '-r', help='Give a dsn of any member with this role', type=role_choice, default=None)
 @click.option('--member', '-m', help='Generate a dsn for this member', type=str)
 @arg_cluster_name
+@option_citus_group
 @click.pass_obj
-def dsn(obj, cluster_name, role, member):
-    if role is not None and member is not None:
-        raise PatroniCtlException('--role and --member are mutually exclusive options')
+def dsn(obj, cluster_name, group, role, member):
+    if member is not None:
+        if role is not None:
+            raise PatroniCtlException('--role and --member are mutually exclusive options')
+        role = 'any'
     if member is None and role is None:
-        role = 'master'
+        role = 'leader'
 
-    cluster = get_dcs(obj, cluster_name).get_cluster()
-    m = get_any_member(cluster, role=role, member=member)
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
+    m = get_any_member(obj, cluster, group, role=role, member=member)
     if m is None:
         raise PatroniCtlException('Can not find a suitable member')
 
@@ -356,14 +377,14 @@ def dsn(obj, cluster_name, role, member):
 
 @ctl.command('query', help='Query a Patroni PostgreSQL member')
 @arg_cluster_name
+@option_citus_group
 @click.option('--format', 'fmt', help='Output format (pretty, tsv, json, yaml)', default='tsv')
 @click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb'))
 @click.option('--password', help='force password prompt', is_flag=True)
 @click.option('-U', '--username', help='database user name', type=str)
 @option_watch
 @option_watchrefresh
-@click.option('--role', '-r', help='The role of the query', type=click.Choice(['master', 'replica', 'any']),
-              default=None)
+@click.option('--role', '-r', help='The role of the query', type=role_choice, default=None)
 @click.option('--member', '-m', help='Query a specific member', type=str)
 @click.option('--delimiter', help='The column delimiter', default='\t')
 @click.option('--command', '-c', help='The SQL commands to execute')
@@ -372,6 +393,7 @@ def dsn(obj, cluster_name, role, member):
 def query(
     obj,
     cluster_name,
+    group,
     role,
     member,
     w,
@@ -384,10 +406,12 @@ def query(
     dbname,
     fmt='tsv',
 ):
-    if role is not None and member is not None:
-        raise PatroniCtlException('--role and --member are mutually exclusive options')
+    if member is not None:
+        if role is not None:
+            raise PatroniCtlException('--role and --member are mutually exclusive options')
+        role = 'any'
     if member is None and role is None:
-        role = 'master'
+        role = 'leader'
 
     if p_file is not None and command is not None:
         raise PatroniCtlException('--file and --command are mutually exclusive options')
@@ -406,25 +430,25 @@ def query(
     if p_file is not None:
         command = p_file.read()
 
-    dcs = get_dcs(obj, cluster_name)
+    dcs = get_dcs(obj, cluster_name, group)
 
     cursor = None
     for _ in watching(w, watch, clear=False):
         if cursor is None:
             cluster = dcs.get_cluster()
 
-        output, header = query_member(cluster, cursor, member, role, command, connect_parameters)
+        output, header = query_member(obj, cluster, group, cursor, member, role, command, connect_parameters)
         print_output(header, output, fmt=fmt, delimiter=delimiter)
 
 
-def query_member(cluster, cursor, member, role, command, connect_parameters):
+def query_member(obj, cluster, group, cursor, member, role, command, connect_parameters):
     from . import psycopg
     try:
         if cursor is None:
-            cursor = get_cursor(cluster, connect_parameters, role=role, member=member)
+            cursor = get_cursor(obj, cluster, group, connect_parameters, role=role, member=member)
 
         if cursor is None:
-            if role is None:
+            if member is not None:
                 message = 'No connection to member {0} is available'.format(member)
             else:
                 message = 'No connection to role={0} is available'.format(role)
@@ -444,13 +468,16 @@ def query_member(cluster, cursor, member, role, command, connect_parameters):
 
 @ctl.command('remove', help='Remove cluster from DCS')
 @click.argument('cluster_name')
+@option_citus_group
 @option_format
 @click.pass_obj
-def remove(obj, cluster_name, fmt):
-    dcs = get_dcs(obj, cluster_name)
+def remove(obj, cluster_name, group, fmt):
+    dcs = get_dcs(obj, cluster_name, group)
     cluster = dcs.get_cluster()
 
-    output_members(cluster, cluster_name, fmt=fmt)
+    if obj.get('citus') and group is None:
+        raise PatroniCtlException('For Citus clusters the --group must me specified')
+    output_members(obj, cluster, cluster_name, fmt=fmt)
 
     confirm = click.prompt('Please confirm the cluster name to remove', type=str)
     if confirm != cluster_name:
@@ -464,9 +491,9 @@ def remove(obj, cluster_name, fmt):
         raise PatroniCtlException('You did not exactly type "{0}"'.format(message))
 
     if cluster.leader and cluster.leader.name:
-        confirm = click.prompt('This cluster currently is healthy. Please specify the master name to continue')
+        confirm = click.prompt('This cluster currently is healthy. Please specify the leader name to continue')
         if confirm != cluster.leader.name:
-            raise PatroniCtlException('You did not specify the current master of the cluster')
+            raise PatroniCtlException('You did not specify the current leader of the cluster')
 
     dcs.delete_cluster()
 
@@ -499,14 +526,14 @@ def parse_scheduled(scheduled):
 @ctl.command('reload', help='Reload cluster member configuration')
 @click.argument('cluster_name')
 @click.argument('member_names', nargs=-1)
-@click.option('--role', '-r', help='Reload only members with this role', default='any',
-              type=click.Choice(['master', 'replica', 'any']))
+@option_citus_group
+@click.option('--role', '-r', help='Reload only members with this role', type=role_choice, default='any')
 @option_force
 @click.pass_obj
-def reload(obj, cluster_name, member_names, force, role):
-    cluster = get_dcs(obj, cluster_name).get_cluster()
+def reload(obj, cluster_name, member_names, group, force, role):
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
 
-    members = get_members(cluster, cluster_name, member_names, role, force, 'reload')
+    members = get_members(obj, cluster, cluster_name, member_names, role, force, 'reload', group=group)
 
     for member in members:
         r = request_patroni(member, 'post', 'reload')
@@ -525,8 +552,8 @@ def reload(obj, cluster_name, member_names, force, role):
 @ctl.command('restart', help='Restart cluster member')
 @click.argument('cluster_name')
 @click.argument('member_names', nargs=-1)
-@click.option('--role', '-r', help='Restart only members with this role', default='any',
-              type=click.Choice(['master', 'replica', 'any']))
+@option_citus_group
+@click.option('--role', '-r', help='Restart only members with this role', type=role_choice, default='any')
 @click.option('--any', 'p_any', help='Restart a single member only', is_flag=True)
 @click.option('--scheduled', help='Timestamp of a scheduled restart in unambiguous format (e.g. ISO 8601)',
               default=None)
@@ -537,10 +564,10 @@ def reload(obj, cluster_name, member_names, force, role):
               help='Return error and fail over if necessary when restarting takes longer than this.')
 @option_force
 @click.pass_obj
-def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, version, pending, timeout):
-    cluster = get_dcs(obj, cluster_name).get_cluster()
+def restart(obj, cluster_name, group, member_names, force, role, p_any, scheduled, version, pending, timeout):
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
 
-    members = get_members(cluster, cluster_name, member_names, role, force, 'restart', False)
+    members = get_members(obj, cluster, cluster_name, member_names, role, force, 'restart', False, group=group)
     if scheduled is None and not force:
         next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
         scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ',
@@ -598,13 +625,14 @@ def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, vers
 
 @ctl.command('reinit', help='Reinitialize cluster member')
 @click.argument('cluster_name')
+@option_citus_group
 @click.argument('member_names', nargs=-1)
 @option_force
 @click.option('--wait', help='Wait until reinitialization completes', is_flag=True)
 @click.pass_obj
-def reinit(obj, cluster_name, member_names, force, wait):
-    cluster = get_dcs(obj, cluster_name).get_cluster()
-    members = get_members(cluster, cluster_name, member_names, None, force, 'reinitialize')
+def reinit(obj, cluster_name, group, member_names, force, wait):
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
+    members = get_members(obj, cluster, cluster_name, member_names, 'replica', force, 'reinitialize', group=group)
 
     wait_on_members = []
     for member in members:
@@ -635,31 +663,42 @@ def reinit(obj, cluster_name, member_names, force, wait):
                 wait_on_members.remove(member)
 
 
-def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, force, scheduled=None):
+def _do_failover_or_switchover(obj, action, cluster_name, group, leader, candidate, force, scheduled=None):
     """
         We want to trigger a failover or switchover for the specified cluster name.
 
-        We verify that the cluster name, master name and candidate name are correct.
+        We verify that the cluster name, leader name and candidate name are correct.
         If so, we trigger an action and keep the client up to date.
     """
 
-    dcs = get_dcs(obj, cluster_name)
+    dcs = get_dcs(obj, cluster_name, group)
     cluster = dcs.get_cluster()
+    click.echo('Current cluster topology')
+    output_members(obj, cluster, cluster_name, group=group)
+
+    if obj.get('citus') and group is None:
+        if force:
+            raise PatroniCtlException('For Citus clusters the --group must me specified')
+        else:
+            group = click.prompt('Citus group', type=int)
+            dcs = get_dcs(obj, cluster_name, group)
+            cluster = dcs.get_cluster()
 
     if action == 'switchover' and (cluster.leader is None or not cluster.leader.name):
-        raise PatroniCtlException('This cluster has no master')
+        raise PatroniCtlException('This cluster has no leader')
 
-    if master is None:
+    if leader is None:
         if force or action == 'failover':
-            master = cluster.leader and cluster.leader.name
+            leader = cluster.leader and cluster.leader.name
         else:
-            master = click.prompt('Master', type=str, default=cluster.leader.member.name)
+            prompt = 'Standby Leader' if is_standby_cluster(cluster.config) else 'Primary'
+            leader = click.prompt(prompt, type=str, default=cluster.leader.member.name)
 
-    if master is not None and cluster.leader and cluster.leader.member.name != master:
-        raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(master, cluster_name))
+    if leader is not None and cluster.leader and cluster.leader.member.name != leader:
+        raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(leader, cluster_name))
 
     # excluding members with nofailover tag
-    candidate_names = [str(m.name) for m in cluster.members if m.name != master and not m.nofailover]
+    candidate_names = [str(m.name) for m in cluster.members if m.name != leader and not m.nofailover]
     # We sort the names for consistent output to the client
     candidate_names.sort()
 
@@ -672,7 +711,7 @@ def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, for
     if action == 'failover' and not candidate:
         raise PatroniCtlException('Failover could be performed only to a specific candidate')
 
-    if candidate == master:
+    if candidate == leader:
         raise PatroniCtlException(action.title() + ' target and source are the same.')
 
     if candidate and candidate not in candidate_names:
@@ -693,16 +732,13 @@ def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, for
                 raise PatroniCtlException("Can't schedule switchover in the paused state")
             scheduled_at_str = scheduled_at.isoformat()
 
-    failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at_str}
+    failover_value = {'leader': leader, 'candidate': candidate, 'scheduled_at': scheduled_at_str}
 
     logging.debug(failover_value)
 
     # By now we have established that the leader exists and the candidate exists
-    click.echo('Current cluster topology')
-    output_members(dcs.get_cluster(), cluster_name)
-
     if not force:
-        demote_msg = ', demoting current master ' + master if master else ''
+        demote_msg = ', demoting current leader ' + leader if leader else ''
         if scheduled_at_str:
             if not click.confirm('Are you sure you want to schedule {0} of cluster {1} at {2}{3}?'
                                  .format(action, cluster_name, scheduled_at_str, demote_msg)):
@@ -734,32 +770,34 @@ def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, for
         logging.exception(r)
         logging.warning('Failing over to DCS')
         click.echo('{0} Could not {1} using Patroni api, falling back to DCS'.format(timestamp(), action))
-        dcs.manual_failover(master, candidate, scheduled_at=scheduled_at)
+        dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at)
 
-    output_members(cluster, cluster_name)
+    output_members(obj, cluster, cluster_name, group=group)
 
 
 @ctl.command('failover', help='Failover to a replica')
 @arg_cluster_name
-@click.option('--master', help='The name of the current master', default=None)
+@option_citus_group
+@click.option('--leader', '--primary', '--master', 'leader', help='The name of the current leader', default=None)
 @click.option('--candidate', help='The name of the candidate', default=None)
 @option_force
 @click.pass_obj
-def failover(obj, cluster_name, master, candidate, force):
-    action = 'switchover' if master else 'failover'
-    _do_failover_or_switchover(obj, action, cluster_name, master, candidate, force)
+def failover(obj, cluster_name, group, leader, candidate, force):
+    action = 'switchover' if leader else 'failover'
+    _do_failover_or_switchover(obj, action, cluster_name, group, leader, candidate, force)
 
 
 @ctl.command('switchover', help='Switchover to a replica')
 @arg_cluster_name
-@click.option('--master', help='The name of the current master', default=None)
+@option_citus_group
+@click.option('--leader', '--primary', '--master', 'leader', help='The name of the current leader', default=None)
 @click.option('--candidate', help='The name of the candidate', default=None)
 @click.option('--scheduled', help='Timestamp of a scheduled switchover in unambiguous format (e.g. ISO 8601)',
               default=None)
 @option_force
 @click.pass_obj
-def switchover(obj, cluster_name, master, candidate, force, scheduled):
-    _do_failover_or_switchover(obj, 'switchover', cluster_name, master, candidate, force, scheduled)
+def switchover(obj, cluster_name, group, leader, candidate, force, scheduled):
+    _do_failover_or_switchover(obj, 'switchover', cluster_name, group, leader, candidate, force, scheduled)
 
 
 def generate_topology(level, member, topology):
@@ -789,48 +827,7 @@ def topology_sort(members):
         yield member
 
 
-def output_members(cluster, name, extended=False, fmt='pretty'):
-    rows = []
-    logging.debug(cluster)
-    initialize = {None: 'uninitialized', '': 'initializing'}.get(cluster.initialize, cluster.initialize)
-    cluster = cluster_as_json(cluster)
-
-    columns = ['Cluster', 'Member', 'Host', 'Role', 'State', 'TL', 'Lag in MB']
-    for c in ('Pending restart', 'Scheduled restart', 'Tags'):
-        if extended or any(m.get(c.lower().replace(' ', '_')) for m in cluster['members']):
-            columns.append(c)
-
-    # Show Host as 'host:port' if somebody is running on non-standard port or two nodes are running on the same host
-    members = [m for m in cluster['members'] if 'host' in m]
-    append_port = any('port' in m and m['port'] != 5432 for m in members) or\
-        len(set(m['host'] for m in members)) < len(members)
-
-    sort = topology_sort if fmt == 'topology' else iter
-    for m in sort(cluster['members']):
-        logging.debug(m)
-
-        lag = m.get('lag', '')
-        m.update(cluster=name, member=m['name'], host=m.get('host', ''), tl=m.get('timeline', ''),
-                 role=m['role'].replace('_', ' ').title(),
-                 lag_in_mb=round(lag/1024/1024) if isinstance(lag, six.integer_types) else lag,
-                 pending_restart='*' if m.get('pending_restart') else '')
-
-        if append_port and m['host'] and m.get('port'):
-            m['host'] = ':'.join([m['host'], str(m['port'])])
-
-        if 'scheduled_restart' in m:
-            value = m['scheduled_restart']['schedule']
-            if 'postgres_version' in m['scheduled_restart']:
-                value += ' if version < {0}'.format(m['scheduled_restart']['postgres_version'])
-            m['scheduled_restart'] = value
-
-        rows.append([m.get(n.lower().replace(' ', '_'), '') for n in columns])
-
-    print_output(columns, rows, {'Lag in MB': 'r', 'TL': 'r'}, fmt, ' Cluster: {0} ({1}) '.format(name, initialize))
-
-    if fmt not in ('pretty', 'topology'):  # Omit service info when using machine-readable formats
-        return
-
+def get_cluster_service_info(cluster):
     service_info = []
     if cluster.get('pause'):
         service_info.append('Maintenance mode: on')
@@ -841,44 +838,109 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
             if name in cluster['scheduled_switchover']:
                 info += '\n{0:>24}: {1}'.format(name, cluster['scheduled_switchover'][name])
         service_info.append(info)
+    return service_info
 
-    if service_info:
-        click.echo(' ' + '\n '.join(service_info))
+
+def output_members(obj, cluster, name, extended=False, fmt='pretty', group=None):
+    rows = []
+    logging.debug(cluster)
+
+    initialize = {None: 'uninitialized', '': 'initializing'}.get(cluster.initialize, cluster.initialize)
+    columns = ['Cluster', 'Member', 'Host', 'Role', 'State', 'TL', 'Lag in MB']
+
+    clusters = {group or 0: cluster_as_json(cluster)}
+
+    is_citus_cluster = obj.get('citus')
+    if is_citus_cluster:
+        columns.insert(1, 'Group')
+        if group is None:
+            clusters.update({g: cluster_as_json(c) for g, c in cluster.workers.items()})
+
+    all_members = [m for c in clusters.values() for m in c['members'] if 'host' in m]
+
+    for c in ('Pending restart', 'Scheduled restart', 'Tags'):
+        if extended or any(m.get(c.lower().replace(' ', '_')) for m in all_members):
+            columns.append(c)
+
+    # Show Host as 'host:port' if somebody is running on non-standard port or two nodes are running on the same host
+    append_port = any('port' in m and m['port'] != 5432 for m in all_members) or\
+        len(set(m['host'] for m in all_members)) < len(all_members)
+
+    sort = topology_sort if fmt == 'topology' else iter
+    for g, cluster in sorted(clusters.items()):
+        for member in sort(cluster['members']):
+            logging.debug(member)
+
+            lag = member.get('lag', '')
+            member.update(cluster=name, member=member['name'], group=g,
+                          host=member.get('host', ''), tl=member.get('timeline', ''),
+                          role=member['role'].replace('_', ' ').title(),
+                          lag_in_mb=round(lag/1024/1024) if isinstance(lag, six.integer_types) else lag,
+                          pending_restart='*' if member.get('pending_restart') else '')
+
+            if append_port and member['host'] and member.get('port'):
+                member['host'] = ':'.join([member['host'], str(member['port'])])
+
+            if 'scheduled_restart' in member:
+                value = member['scheduled_restart']['schedule']
+                if 'postgres_version' in member['scheduled_restart']:
+                    value += ' if version < {0}'.format(member['scheduled_restart']['postgres_version'])
+                member['scheduled_restart'] = value
+
+            rows.append([member.get(n.lower().replace(' ', '_'), '') for n in columns])
+
+    title = 'Citus cluster' if is_citus_cluster else 'Cluster'
+    group_title = '' if group is None else 'group: {0}, '.format(group)
+    title_details = group_title and ' ({0}{1})'.format(group_title, initialize)
+    title = ' {0}: {1}{2} '.format(title, name, title_details)
+    print_output(columns, rows, {'Group': 'r', 'Lag in MB': 'r', 'TL': 'r'}, fmt, title)
+
+    if fmt not in ('pretty', 'topology'):  # Omit service info when using machine-readable formats
+        return
+
+    for g, cluster in sorted(clusters.items()):
+        service_info = get_cluster_service_info(cluster)
+        if service_info:
+            if is_citus_cluster and group is None:
+                click.echo('Citus group: {0}'.format(g))
+            click.echo(' ' + '\n '.join(service_info))
 
 
 @ctl.command('list', help='List the Patroni members for a given Patroni')
 @click.argument('cluster_names', nargs=-1)
+@option_citus_group
 @click.option('--extended', '-e', help='Show some extra information', is_flag=True)
 @click.option('--timestamp', '-t', 'ts', help='Print timestamp', is_flag=True)
 @option_format
 @option_watch
 @option_watchrefresh
 @click.pass_obj
-def members(obj, cluster_names, fmt, watch, w, extended, ts):
+def members(obj, cluster_names, group, fmt, watch, w, extended, ts):
     if not cluster_names:
         if 'scope' in obj:
             cluster_names = [obj['scope']]
         if not cluster_names:
             return logging.warning('Listing members: No cluster names were provided')
 
-    for cluster_name in cluster_names:
-        dcs = get_dcs(obj, cluster_name)
+    for _ in watching(w, watch):
+        if ts:
+            click.echo(timestamp(0))
 
-        for _ in watching(w, watch):
-            if ts:
-                click.echo(timestamp(0))
+        for cluster_name in cluster_names:
+            dcs = get_dcs(obj, cluster_name, group)
 
             cluster = dcs.get_cluster()
-            output_members(cluster, cluster_name, extended, fmt)
+            output_members(obj, cluster, cluster_name, extended, fmt, group)
 
 
 @ctl.command('topology', help='Prints ASCII topology for given cluster')
 @click.argument('cluster_names', nargs=-1)
+@option_citus_group
 @option_watch
 @option_watchrefresh
 @click.pass_obj
 @click.pass_context
-def topology(ctx, obj, cluster_names, watch, w):
+def topology(ctx, obj, cluster_names, group, watch, w):
     ctx.forward(members, fmt='topology')
 
 
@@ -886,83 +948,20 @@ def timestamp(precision=6):
     return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:precision - 7]
 
 
-@ctl.command('configure', help='Create configuration file')
-@click.option('--config-file', '-c', help='Configuration file', prompt='Configuration file', default=CONFIG_FILE_PATH)
-@click.option('--dcs', '-d', help='The DCS connect url', prompt='DCS connect url', default='etcd://localhost:2379')
-@click.option('--namespace', '-n', help='The namespace', prompt='Namespace', default='/service/')
-def configure(config_file, dcs, namespace):
-    store_config({'dcs_api': str(dcs), 'namespace': str(namespace)}, config_file)
-
-
-def touch_member(config, dcs):
-    ''' Rip-off of the ha.touch_member without inter-class dependencies '''
-    p = Postgresql(config['postgresql'])
-    p.set_state('running')
-    p.set_role('master')
-
-    def restapi_connection_string(config):
-        protocol = 'https' if config.get('certfile') else 'http'
-        connect_address = config.get('connect_address')
-        listen = config['listen']
-        return '{0}://{1}/patroni'.format(protocol, connect_address or listen)
-
-    data = {
-        'conn_url': p.connection_string,
-        'api_url': restapi_connection_string(config['restapi']),
-        'state': p.state,
-        'role': p.role
-    }
-
-    return dcs.touch_member(data, permanent=True)
-
-
-def set_defaults(config, cluster_name):
-    """fill-in some basic configuration parameters if config file is not set """
-    config['postgresql'].setdefault('name', cluster_name)
-    config['postgresql'].setdefault('scope', cluster_name)
-    config['postgresql'].setdefault('listen', '127.0.0.1')
-    config['postgresql']['authentication'] = {'replication': None}
-    config['restapi']['listen'] = ':' in config['restapi']['listen'] and config['restapi']['listen'] or '127.0.0.1:8008'
-
-
-@ctl.command('scaffold', help='Create a structure for the cluster in DCS')
-@click.argument('cluster_name')
-@click.option('--sysid', '-s', help='System ID of the cluster to put into the initialize key', default="")
-@click.pass_obj
-def scaffold(obj, cluster_name, sysid):
-    dcs = get_dcs(obj, cluster_name)
-    cluster = dcs.get_cluster()
-    if cluster and cluster.initialize is not None:
-        raise PatroniCtlException("This cluster is already initialized")
-
-    if not dcs.initialize(create_new=True, sysid=sysid):
-        # initialize key already exists, don't touch this cluster
-        raise PatroniCtlException("Initialize key for cluster {0} already exists".format(cluster_name))
-
-    set_defaults(obj, cluster_name)
-
-    # make sure the leader keys will never expire
-    if not (touch_member(obj, dcs) and dcs.attempt_to_acquire_leader(permanent=True)):
-        # we did initialize this cluster, but failed to write the leader or member keys, wipe it down completely.
-        dcs.delete_cluster()
-        raise PatroniCtlException("Unable to install permanent leader for cluster {0}".format(cluster_name))
-    click.echo("Cluster {0} has been created successfully".format(cluster_name))
-
-
 @ctl.command('flush', help='Discard scheduled events')
 @click.argument('cluster_name')
+@option_citus_group
 @click.argument('member_names', nargs=-1)
 @click.argument('target', type=click.Choice(['restart', 'switchover']))
-@click.option('--role', '-r', help='Flush only members with this role', default='any',
-              type=click.Choice(['master', 'replica', 'any']))
+@click.option('--role', '-r', help='Flush only members with this role', type=role_choice, default='any')
 @option_force
 @click.pass_obj
-def flush(obj, cluster_name, member_names, force, role, target):
-    dcs = get_dcs(obj, cluster_name)
+def flush(obj, cluster_name, group, member_names, force, role, target):
+    dcs = get_dcs(obj, cluster_name, group)
     cluster = dcs.get_cluster()
 
     if target == 'restart':
-        for member in get_members(cluster, cluster_name, member_names, role, force, 'flush'):
+        for member in get_members(obj, cluster, cluster_name, member_names, role, force, 'flush', group=group):
             if member.data.get('scheduled_restart'):
                 r = request_patroni(member, 'delete', 'restart')
                 check_response(r, member.name, 'flush scheduled restart')
@@ -1008,8 +1007,8 @@ def wait_until_pause_is_applied(dcs, paused, old_cluster):
     return click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
 
 
-def toggle_pause(config, cluster_name, paused, wait):
-    dcs = get_dcs(config, cluster_name)
+def toggle_pause(config, cluster_name, group, paused, wait):
+    dcs = get_dcs(config, cluster_name, group)
     cluster = dcs.get_cluster()
     if cluster.is_paused() == paused:
         raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not'))
@@ -1037,18 +1036,20 @@ def toggle_pause(config, cluster_name, paused, wait):
 
 @ctl.command('pause', help='Disable auto failover')
 @arg_cluster_name
+@option_default_citus_group
 @click.pass_obj
 @click.option('--wait', help='Wait until pause is applied on all nodes', is_flag=True)
-def pause(obj, cluster_name, wait):
-    return toggle_pause(obj, cluster_name, True, wait)
+def pause(obj, cluster_name, group, wait):
+    return toggle_pause(obj, cluster_name, group, True, wait)
 
 
 @ctl.command('resume', help='Resume auto failover')
 @arg_cluster_name
+@option_default_citus_group
 @click.option('--wait', help='Wait until pause is cleared on all nodes', is_flag=True)
 @click.pass_obj
-def resume(obj, cluster_name, wait):
-    return toggle_pause(obj, cluster_name, False, wait)
+def resume(obj, cluster_name, group, wait):
+    return toggle_pause(obj, cluster_name, group, False, wait)
 
 
 @contextmanager
@@ -1205,6 +1206,7 @@ def invoke_editor(before_editing, cluster_name):
 
 @ctl.command('edit-config', help="Edit cluster configuration")
 @arg_cluster_name
+@option_default_citus_group
 @click.option('--quiet', '-q', is_flag=True, help='Do not show changes')
 @click.option('--set', '-s', 'kvpairs', multiple=True,
               help='Set specific configuration value. Can be specified multiple times')
@@ -1216,8 +1218,8 @@ def invoke_editor(before_editing, cluster_name):
               ' Use - for stdin.')
 @option_force
 @click.pass_obj
-def edit_config(obj, cluster_name, force, quiet, kvpairs, pgkvpairs, apply_filename, replace_filename):
-    dcs = get_dcs(obj, cluster_name)
+def edit_config(obj, cluster_name, group, force, quiet, kvpairs, pgkvpairs, apply_filename, replace_filename):
+    dcs = get_dcs(obj, cluster_name, group)
     cluster = dcs.get_cluster()
 
     before_editing = format_config_for_editing(cluster.config.data)
@@ -1259,9 +1261,10 @@ def edit_config(obj, cluster_name, force, quiet, kvpairs, pgkvpairs, apply_filen
 
 @ctl.command('show-config', help="Show cluster configuration")
 @arg_cluster_name
+@option_default_citus_group
 @click.pass_obj
-def show_config(obj, cluster_name):
-    cluster = get_dcs(obj, cluster_name).get_cluster()
+def show_config(obj, cluster_name, group):
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
 
     click.echo(format_config_for_editing(cluster.config.data))
 
@@ -1269,16 +1272,17 @@ def show_config(obj, cluster_name):
 @ctl.command('version', help='Output version of patronictl command or a running Patroni instance')
 @click.argument('cluster_name', required=False)
 @click.argument('member_names', nargs=-1)
+@option_citus_group
 @click.pass_obj
-def version(obj, cluster_name, member_names):
+def version(obj, cluster_name, group, member_names):
     click.echo("patronictl version {0}".format(__version__))
 
     if not cluster_name:
         return
 
     click.echo("")
-    cluster = get_dcs(obj, cluster_name).get_cluster()
-    for m in cluster.members:
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
+    for m in get_all_members(obj, cluster, group, 'any'):
         if m.api_url:
             if not member_names or m.name in member_names:
                 try:
@@ -1294,10 +1298,11 @@ def version(obj, cluster_name, member_names):
 
 @ctl.command('history', help="Show the history of failovers/switchovers")
 @arg_cluster_name
+@option_default_citus_group
 @option_format
 @click.pass_obj
-def history(obj, cluster_name, fmt):
-    cluster = get_dcs(obj, cluster_name).get_cluster()
+def history(obj, cluster_name, group, fmt):
+    cluster = get_dcs(obj, cluster_name, group).get_cluster()
     history = cluster.history and cluster.history.lines or []
     table_header_row = ['TL', 'LSN', 'Reason', 'Timestamp', 'New Leader']
     for line in history:

patroni/daemon.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 import abc
 import os
 import signal
@@ -22,11 +24,15 @@ class AbstractPatroniDaemon(object):
     def sighup_handler(self, *args):
         self._received_sighup = True
 
-    def sigterm_handler(self, *args):
+    def api_sigterm(self):
         with self._sigterm_lock:
             if not self._received_sigterm:
                 self._received_sigterm = True
-                sys.exit()
+                return True
+
+    def sigterm_handler(self, *args):
+        if self.api_sigterm():
+            sys.exit()
 
     def setup_signal_handlers(self):
         self._received_sighup = False
@@ -83,16 +89,18 @@ def abstract_main(cls, validator=None):
                         help='Patroni may also read the configuration from the {0} environment variable'
                         .format(Config.PATRONI_CONFIG_VARIABLE))
     args = parser.parse_args()
+    validate_config = validator and args.validate_config
     try:
-        if validator and args.validate_config:
+        if validate_config:
             Config(args.configfile, validator=validator)
             sys.exit()
 
         config = Config(args.configfile)
     except ConfigParseError as e:
         if e.value:
-            print(e.value)
-        parser.print_help()
+            print(e.value, file=sys.stderr)
+        if not validate_config:
+            parser.print_help()
         sys.exit(1)
 
     controller = cls(config)

patroni/dcs/__init__.py

@@ -20,6 +20,8 @@ from threading import Event, Lock
 from ..exceptions import PatroniFatalException
 from ..utils import deep_compare, parse_bool, uri
 
+CITUS_COORDINATOR_GROUP_ID = 0
+citus_group_re = re.compile('^(0|[1-9][0-9]*)$')
 slot_name_re = re.compile('^[a-z0-9_]{1,63}$')
 logger = logging.getLogger(__name__)
 
@@ -94,6 +96,9 @@ def get_dcs(config):
                         # propagate some parameters
                         config[name].update({p: config[p] for p in ('namespace', 'name', 'scope', 'loop_wait',
                                              'patronictl', 'ttl', 'retry_timeout') if p in config})
+                        # From citus section we only need "group" parameter, but will propagate everything just in case.
+                        if isinstance(config.get('citus'), dict):
+                            config[name].update(config['citus'])
                         return item(config[name])
             except ImportError:
                 logger.debug('Failed to import %s', module_name)
@@ -224,8 +229,7 @@ class Member(namedtuple('Member', 'index,name,session,data')):
 
 
 class RemoteMember(Member):
-    """ Represents a remote master for a standby cluster
-    """
+    """Represents a remote member (typically a primary) for a standby cluster"""
     def __new__(cls, name, data):
         return super(RemoteMember, cls).__new__(cls, None, name, None, data)
 
@@ -278,7 +282,7 @@ class Leader(namedtuple('Leader', 'index,session,member')):
         version = self.member.version
         # 1.5.6 is the last version which doesn't expose checkpoint_after_promote: false
         if version and version > (1, 5, 6):
-            return self.data.get('role') == 'master' and 'checkpoint_after_promote' not in self.data
+            return self.data.get('role') in ('master', 'primary') and 'checkpoint_after_promote' not in self.data
 
 
 class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
@@ -444,7 +448,8 @@ class TimelineHistory(namedtuple('TimelineHistory', 'index,value,lines')):
         return TimelineHistory(index, value, lines)
 
 
-class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,failover,sync,history,slots')):
+class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,'
+                                    'failover,sync,history,slots,failsafe,workers')):
 
     """Immutable object (namedtuple) which represents PostgreSQL cluster.
     Consists of the following fields:
@@ -458,7 +463,14 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,f
     :param sync: reference to `SyncState` object, last observed synchronous replication state.
     :param history: reference to `TimelineHistory` object
     :param slots: state of permanent logical replication slots on the primary in the format: {"slot_name": int}
-    """
+    :param failsafe: failsafe topology. Node is allowed to become the leader only if its name is found in this list.
+    :param workers: workers of the Citus cluster, optional. Format: {int(group): Cluster()}"""
+
+    def __new__(cls, *args):
+        # Make workers argument optional
+        if len(cls._fields) == len(args) + 1:
+            args = args + ({},)
+        return super(Cluster, cls).__new__(cls, *args)
 
     @property
     def leader_name(self):
@@ -507,16 +519,16 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,f
 
     def get_replication_slots(self, my_name, role, nofailover, major_version, show_error=False):
         # if the replicatefrom tag is set on the member - we should not create the replication slot for it on
-        # the current master, because that member would replicate from elsewhere. We still create the slot if
+        # the current primary, because that member would replicate from elsewhere. We still create the slot if
         # the replicatefrom destination member is currently not a member of the cluster (fallback to the
-        # master), or if replicatefrom destination member happens to be the current master
+        # primary), or if replicatefrom destination member happens to be the current primary
         use_slots = self.use_slots
-        if role in ('master', 'standby_leader'):
+        if role in ('master', 'primary', 'standby_leader'):
             slot_members = [m.name for m in self.members if use_slots and m.name != my_name and
                             (m.replicatefrom is None or m.replicatefrom == my_name or
                              not self.has_member(m.replicatefrom))]
             permanent_slots = self.__permanent_slots if use_slots and \
-                role == 'master' else self.__permanent_physical_slots
+                role in ('master', 'primary') else self.__permanent_physical_slots
         else:
             # only manage slots for replicas that replicate from this one, except for the leader among them
             slot_members = [m.name for m in self.members if use_slots and
@@ -606,11 +618,11 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,f
     @property
     def timeline(self):
         """
-        >>> Cluster(0, 0, 0, 0, 0, 0, 0, 0, 0).timeline
+        >>> Cluster(0, 0, 0, 0, 0, 0, 0, 0, 0, None).timeline
         0
-        >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[]'), 0).timeline
+        >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[]'), 0, None).timeline
         1
-        >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[["a"]]'), 0).timeline
+        >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[["a"]]'), 0, None).timeline
         0
         """
         if self.history:
@@ -628,6 +640,20 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,f
         return next(iter(sorted(filter(lambda v: v, [m.version for m in self.members])) + [None]))
 
 
+class ReturnFalseException(Exception):
+    pass
+
+
+def catch_return_false_exception(func):
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except ReturnFalseException:
+            return False
+
+    return wrapper
+
+
 @six.add_metaclass(abc.ABCMeta)
 class AbstractDCS(object):
 
@@ -641,6 +667,7 @@ class AbstractDCS(object):
     _STATUS = 'status'  # JSON, contains "leader_lsn" and confirmed_flush_lsn of logical "slots" on the leader
     _LEADER_OPTIME = _OPTIME + '/' + _LEADER  # legacy
     _SYNC = 'sync'
+    _FAILSAFE = 'failsafe'
 
     def __init__(self, config):
         """
@@ -649,6 +676,7 @@ class AbstractDCS(object):
         """
         self._name = config['name']
         self._base_path = re.sub('/+', '/', '/'.join(['', config.get('namespace', 'service'), config['scope']]))
+        self._citus_group = str(config['group']) if isinstance(config.get('group'), six.integer_types) else None
         self._set_loop_wait(config.get('loop_wait', 10))
 
         self._ctl = bool(config.get('patronictl', False))
@@ -658,10 +686,15 @@ class AbstractDCS(object):
         self._last_lsn = ''
         self._last_seen = 0
         self._last_status = {}
+        self._last_failsafe = {}
         self.event = Event()
 
     def client_path(self, path):
-        return '/'.join([self._base_path, path.lstrip('/')])
+        components = [self._base_path]
+        if self._citus_group:
+            components.append(self._citus_group)
+        components.append(path.lstrip('/'))
+        return '/'.join(components)
 
     @property
     def initialize_path(self):
@@ -703,6 +736,10 @@ class AbstractDCS(object):
     def sync_path(self):
         return self.client_path(self._SYNC)
 
+    @property
+    def failsafe_path(self):
+        return self.client_path(self._FAILSAFE)
+
     @abc.abstractmethod
     def set_ttl(self, ttl):
         """Set the new ttl value for leader key"""
@@ -732,28 +769,69 @@ class AbstractDCS(object):
         return self._last_seen
 
     @abc.abstractmethod
-    def _load_cluster(self):
-        """Internally this method should build  `Cluster` object which
-           represents current state and topology of the cluster in DCS.
-           this method supposed to be called only by `get_cluster` method.
+    def _cluster_loader(self, path):
+        """Load and build the `Cluster` object from DCS, which
+        represents a single Patroni cluster.
 
-           raise `~DCSError` in case of communication or other problems with DCS.
-           If the current node was running as a master and exception raised,
-           instance would be demoted."""
+        :param path: the path in DCS where to load Cluster(s) from.
+        :returns: `Cluster`"""
+
+    def _citus_cluster_loader(self, path):
+        """Load and build `Cluster` onjects from DCS that represent all
+        Patroni clusters from a single Citus cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+        :returns: all Citus groups as `dict`, with group ids as keys"""
+
+    @abc.abstractmethod
+    def _load_cluster(self, path, loader):
+        """Internally this method should call the `loader` method that
+        will build `Cluster` object which represents current state and
+        topology of the cluster in DCS. This method supposed to be
+        called only by `get_cluster` method.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+        :param loader: one of `_cluster_loader` or `_citus_cluster_loader`
+        :raise: `~DCSError` in case of communication problems with DCS.
+        If the current node was running as a primary and exception
+        raised, instance would be demoted."""
 
     def _bypass_caches(self):
         """Used only in zookeeper"""
 
+    def is_citus_coordinator(self):
+        return self._citus_group == str(CITUS_COORDINATOR_GROUP_ID)
+
+    def get_citus_coordinator(self):
+        try:
+            path = '{0}/{1}/'.format(self._base_path, CITUS_COORDINATOR_GROUP_ID)
+            return self._load_cluster(path, self._cluster_loader)
+        except Exception as e:
+            logger.error('Failed to load Citus coordinator cluster from %s: %r', self.__class__.__name__, e)
+
+    def _get_citus_cluster(self):
+        groups = self._load_cluster(self._base_path + '/', self._citus_cluster_loader)
+        if isinstance(groups, Cluster):  # Zookeeper could return a cached version
+            cluster = groups
+        else:
+            cluster = groups.pop(CITUS_COORDINATOR_GROUP_ID,
+                                 Cluster(None, None, None, None, [], None, None, None, None, None))
+            cluster.workers.update(groups)
+        return cluster
+
     def get_cluster(self, force=False):
         if force:
             self._bypass_caches()
         try:
-            cluster = self._load_cluster()
+            cluster = self._get_citus_cluster() if self.is_citus_coordinator()\
+                else self._load_cluster(self.client_path(''), self._cluster_loader)
         except Exception:
             self.reset_cluster()
             raise
 
         self._last_seen = int(time.time())
+        self._last_status = {self._OPTIME: cluster.last_lsn, 'slots': cluster.slots}
+        self._last_failsafe = cluster.failsafe
 
         with self._cluster_thread_lock:
             self._cluster = cluster
@@ -796,23 +874,39 @@ class AbstractDCS(object):
             self._last_lsn = value[self._OPTIME]
             self._write_leader_optime(str(value[self._OPTIME]))
 
+    @abc.abstractmethod
+    def _write_failsafe(self, value):
+        """Write current cluster topology to DCS that will be used by failsafe mechanism (if enabled).
+
+        :param value: failsafe topology serialized in JSON format
+        :returns: `!True` on success."""
+
+    def write_failsafe(self, value):
+        if not (isinstance(self._last_failsafe, dict) and deep_compare(self._last_failsafe, value))\
+                and self._write_failsafe(json.dumps(value, separators=(',', ':'))):
+            self._last_failsafe = value
+
+    @property
+    def failsafe(self):
+        return self._last_failsafe
+
     @abc.abstractmethod
     def _update_leader(self):
         """Update leader key (or session) ttl
 
         :returns: `!True` if leader key (or session) has been updated successfully.
-            If not, `!False` must be returned and current instance would be demoted.
 
         You have to use CAS (Compare And Swap) operation in order to update leader key,
-        for example for etcd `prevValue` parameter must be used."""
+        for example for etcd `prevValue` parameter must be used.
+        If update fails due to DCS not being accessible or because it is not able to
+        process requests (hopefuly temporary), the ~DCSError exception should be raised."""
 
-    def update_leader(self, last_lsn, slots=None):
+    def update_leader(self, last_lsn, slots=None, failsafe=None):
         """Update leader key (or session) ttl and optime/leader
 
         :param last_lsn: absolute WAL LSN in bytes
         :param slots: dict with permanent slots confirmed_flush_lsn
-        :returns: `!True` if leader key (or session) has been updated successfully.
-            If not, `!False` must be returned and current instance would be demoted."""
+        :returns: `!True` if leader key (or session) has been updated successfully."""
 
         ret = self._update_leader()
         if ret and last_lsn:
@@ -820,18 +914,23 @@ class AbstractDCS(object):
             if slots:
                 status['slots'] = slots
             self.write_status(status)
+
+        if ret and failsafe is not None:
+            self.write_failsafe(failsafe)
+
         return ret
 
     @abc.abstractmethod
-    def attempt_to_acquire_leader(self, permanent=False):
+    def attempt_to_acquire_leader(self):
         """Attempt to acquire leader lock
         This method should create `/leader` key with value=`~self._name`
-        :param permanent: if set to `!True`, the leader key will never expire.
-         Used in patronictl for the external master
         :returns: `!True` if key has been created successfully.
 
         Key must be created atomically. In case if key already exists it should not be
-        overwritten and `!False` must be returned"""
+        overwritten and `!False` must be returned.
+
+        If key creation fails due to DCS not being accessible or because it is not able to
+        process requests (hopefuly temporary), the ~DCSError exception should be raised"""
 
     @abc.abstractmethod
     def set_failover_value(self, value, index=None):
@@ -854,15 +953,13 @@ class AbstractDCS(object):
         """Create or update `/config` key"""
 
     @abc.abstractmethod
-    def touch_member(self, data, permanent=False):
+    def touch_member(self, data):
         """Update member key in DCS.
         This method should create or update key with the name = '/members/' + `~self._name`
         and value = data in a given DCS.
 
         :param data: information about instance (including connection strings)
         :param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used`
-        :param permanent: if set to `!True`, the member key will never expire.
-         Used in patronictl for the external master.
         :returns: `!True` on success otherwise `!False`
         """
 
@@ -929,7 +1026,7 @@ class AbstractDCS(object):
         """"""
 
     def watch(self, leader_index, timeout):
-        """If the current node is a master it should just sleep.
+        """If the current node is a leader it should just sleep.
         Any other node should watch for changes of leader key with a given timeout
 
         :param leader_index: index of a leader key

patroni/dcs/consul.py

@@ -8,13 +8,14 @@ import ssl
 import time
 import urllib3
 
-from collections import namedtuple
+from collections import defaultdict, namedtuple
 from consul import ConsulException, NotFound, base
 from urllib3.exceptions import HTTPError
 from six.moves.urllib.parse import urlencode, urlparse, quote
 from six.moves.http_client import HTTPException
 
-from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState, TimelineHistory
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
+        TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
 from ..exceptions import DCSError
 from ..utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port, uri, USER_AGENT
 
@@ -189,6 +190,7 @@ class Consul(AbstractDCS):
 
     def __init__(self, config):
         super(Consul, self).__init__(config)
+        self._base_path = self._base_path[1:]
         self._scope = config['scope']
         self._session = None
         self.__do_not_watch = False
@@ -236,6 +238,7 @@ class Consul(AbstractDCS):
         self._service_check_tls_server_name = config.get('service_check_tls_server_name', None)
         if not self._ctl:
             self.create_session()
+        self._previous_loop_token = self._client.token
 
     def retry(self, *args, **kwargs):
         return self._retry.copy()(*args, **kwargs)
@@ -270,7 +273,7 @@ class Consul(AbstractDCS):
 
     @property
     def ttl(self):
-        return self._client.http.ttl
+        return self._client.http.ttl * 2  # we multiply the value by 2 because it was divided in the `set_ttl()` method
 
     def set_retry_timeout(self, retry_timeout):
         self._retry.deadline = retry_timeout
@@ -285,9 +288,9 @@ class Consul(AbstractDCS):
         except Exception:
             logger.exception('adjust_ttl')
 
-    def _do_refresh_session(self):
+    def _do_refresh_session(self, force=False):
         """:returns: `!True` if it had to create new session"""
-        if self._session and self._last_session_refresh + self._loop_wait > time.time():
+        if not force and self._session and self._last_session_refresh + self._loop_wait > time.time():
             return False
 
         if self._session:
@@ -316,96 +319,108 @@ class Consul(AbstractDCS):
             logger.exception('refresh_session')
         raise ConsulError('Failed to renew/create session')
 
-    def client_path(self, path):
-        return super(Consul, self).client_path(path)[1:]
-
     @staticmethod
     def member(node):
         return Member.from_node(node['ModifyIndex'], os.path.basename(node['Key']), node.get('Session'), node['Value'])
 
-    def _load_cluster(self):
-        try:
-            path = self.client_path('/')
-            _, results = self.retry(self._client.kv.get, path, recurse=True)
+    def _cluster_from_nodes(self, nodes):
+        # get initialize flag
+        initialize = nodes.get(self._INITIALIZE)
+        initialize = initialize and initialize['Value']
 
-            if results is None:
-                raise NotFound
+        # get global dynamic configuration
+        config = nodes.get(self._CONFIG)
+        config = config and ClusterConfig.from_node(config['ModifyIndex'], config['Value'])
 
-            nodes = {}
-            for node in results:
-                node['Value'] = (node['Value'] or b'').decode('utf-8')
-                nodes[node['Key'][len(path):].lstrip('/')] = node
-
-            # get initialize flag
-            initialize = nodes.get(self._INITIALIZE)
-            initialize = initialize and initialize['Value']
-
-            # get global dynamic configuration
-            config = nodes.get(self._CONFIG)
-            config = config and ClusterConfig.from_node(config['ModifyIndex'], config['Value'])
-
-            # get timeline history
-            history = nodes.get(self._HISTORY)
-            history = history and TimelineHistory.from_node(history['ModifyIndex'], history['Value'])
-
-            # get last known leader lsn and slots
-            status = nodes.get(self._STATUS)
-            if status:
-                try:
-                    status = json.loads(status['Value'])
-                    last_lsn = status.get(self._OPTIME)
-                    slots = status.get('slots')
-                except Exception:
-                    slots = last_lsn = None
-            else:
-                last_lsn = nodes.get(self._LEADER_OPTIME)
-                last_lsn = last_lsn and last_lsn['Value']
-                slots = None
+        # get timeline history
+        history = nodes.get(self._HISTORY)
+        history = history and TimelineHistory.from_node(history['ModifyIndex'], history['Value'])
 
+        # get last known leader lsn and slots
+        status = nodes.get(self._STATUS)
+        if status:
             try:
-                last_lsn = int(last_lsn)
+                status = json.loads(status['Value'])
+                last_lsn = status.get(self._OPTIME)
+                slots = status.get('slots')
             except Exception:
-                last_lsn = 0
+                slots = last_lsn = None
+        else:
+            last_lsn = nodes.get(self._LEADER_OPTIME)
+            last_lsn = last_lsn and last_lsn['Value']
+            slots = None
 
-            # get list of members
-            members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
+        try:
+            last_lsn = int(last_lsn)
+        except Exception:
+            last_lsn = 0
 
-            # get leader
-            leader = nodes.get(self._LEADER)
-            if not self._ctl and leader and leader['Value'] == self._name \
-                    and self._session != leader.get('Session', 'x'):
-                logger.info('I am leader but not owner of the session. Removing leader node')
-                self._client.kv.delete(self.leader_path, cas=leader['ModifyIndex'])
-                leader = None
+        # get list of members
+        members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
 
-            if leader:
-                member = Member(-1, leader['Value'], None, {})
-                member = ([m for m in members if m.name == leader['Value']] or [member])[0]
-                leader = Leader(leader['ModifyIndex'], leader.get('Session'), member)
+        # get leader
+        leader = nodes.get(self._LEADER)
 
-            # failover key
-            failover = nodes.get(self._FAILOVER)
-            if failover:
-                failover = Failover.from_node(failover['ModifyIndex'], failover['Value'])
+        if leader:
+            member = Member(-1, leader['Value'], None, {})
+            member = ([m for m in members if m.name == leader['Value']] or [member])[0]
+            leader = Leader(leader['ModifyIndex'], leader.get('Session'), member)
 
-            # get synchronization state
-            sync = nodes.get(self._SYNC)
-            sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
+        # failover key
+        failover = nodes.get(self._FAILOVER)
+        if failover:
+            failover = Failover.from_node(failover['ModifyIndex'], failover['Value'])
 
-            return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+        # get synchronization state
+        sync = nodes.get(self._SYNC)
+        sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
+
+        # get failsafe topology
+        failsafe = nodes.get(self._FAILSAFE)
+        try:
+            failsafe = json.loads(failsafe['Value']) if failsafe else None
+        except Exception:
+            failsafe = None
+
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+
+    def _cluster_loader(self, path):
+        _, results = self.retry(self._client.kv.get, path, recurse=True)
+        if results is None:
+            raise NotFound
+        nodes = {}
+        for node in results:
+            node['Value'] = (node['Value'] or b'').decode('utf-8')
+            nodes[node['Key'][len(path):]] = node
+
+        return self._cluster_from_nodes(nodes)
+
+    def _citus_cluster_loader(self, path):
+        _, results = self.retry(self._client.kv.get, path, recurse=True)
+        clusters = defaultdict(dict)
+        for node in results or []:
+            key = node['Key'][len(path):].split('/', 1)
+            if len(key) == 2 and citus_group_re.match(key[0]):
+                node['Value'] = (node['Value'] or b'').decode('utf-8')
+                clusters[int(key[0])][key[1]] = node
+        return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()}
+
+    def _load_cluster(self, path, loader):
+        try:
+            return loader(path)
         except NotFound:
-            return Cluster(None, None, None, None, [], None, None, None, None)
+            return Cluster(None, None, None, None, [], None, None, None, None, None)
         except Exception:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')
 
     @catch_consul_errors
-    def touch_member(self, data, permanent=False):
+    def touch_member(self, data):
         cluster = self.cluster
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
 
         try:
-            create_member = not permanent and self.refresh_session()
+            create_member = self.refresh_session()
         except DCSError:
             return False
 
@@ -423,8 +438,7 @@ class Consul(AbstractDCS):
             return True
 
         try:
-            args = {} if permanent else {'acquire': self._session}
-            self._client.kv.put(self.member_path, json.dumps(data, separators=(',', ':')), **args)
+            self._client.kv.put(self.member_path, json.dumps(data, separators=(',', ':')), acquire=self._session)
             return True
         except InvalidSession:
             self._session = None
@@ -463,7 +477,12 @@ class Consul(AbstractDCS):
             check['TLSServerName'] = self._service_check_tls_server_name
         tags = self._service_tags[:]
         tags.append(role)
+        if role == 'master':
+            tags.append('primary')
+        elif role == 'primary':
+            tags.append('master')
         self._previous_loop_service_tags = self._service_tags
+        self._previous_loop_token = self._client.token
 
         params = {
             'service_id': '{0}/{1}'.format(self._scope, self._name),
@@ -479,7 +498,7 @@ class Consul(AbstractDCS):
             return self.deregister_service(params['service_id'])
 
         self._previous_loop_register_service = self._register_service
-        if role in ['master', 'replica', 'standby-leader']:
+        if role in ['master', 'primary', 'replica', 'standby-leader']:
             if state != 'running':
                 return
             return self.register_service(service_name, **params)
@@ -500,25 +519,36 @@ class Consul(AbstractDCS):
         if (
             force or update or self._register_service != self._previous_loop_register_service
             or self._service_tags != self._previous_loop_service_tags
+            or self._client.token != self._previous_loop_token
         ):
             return self._update_service(new_data)
 
-    @catch_consul_errors
-    def _do_attempt_to_acquire_leader(self, permanent):
+    def _do_attempt_to_acquire_leader(self, retry):
         try:
-            kwargs = {} if permanent else {'acquire': self._session}
-            return self.retry(self._client.kv.put, self.leader_path, self._name, **kwargs)
+            return retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session)
         except InvalidSession:
-            self._session = None
             logger.error('Our session disappeared from Consul. Will try to get a new one and retry attempt')
-            self.refresh_session()
-            return self.retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session)
+            self._session = None
+            retry.deadline = retry.stoptime - time.time()
 
-    def attempt_to_acquire_leader(self, permanent=False):
-        if not self._session and not permanent:
-            self.refresh_session()
+            retry(self._do_refresh_session)
 
-        ret = self._do_attempt_to_acquire_leader(permanent)
+            retry.deadline = retry.stoptime - time.time()
+            if retry.deadline < 1:
+                raise ConsulError('_do_attempt_to_acquire_leader timeout')
+
+            return retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session)
+
+    @catch_return_false_exception
+    def attempt_to_acquire_leader(self):
+        retry = self._retry.copy()
+        self._run_and_handle_exceptions(self._do_refresh_session, retry=retry)
+
+        retry.deadline = retry.stoptime - time.time()
+        if retry.deadline < 1:
+            raise ConsulError('attempt_to_acquire_leader timeout')
+
+        ret = self._run_and_handle_exceptions(self._do_attempt_to_acquire_leader, retry, retry=None)
         if not ret:
             logger.info('Could not take out TTL lock')
 
@@ -544,10 +574,42 @@ class Consul(AbstractDCS):
         return self._client.kv.put(self.status_path, value)
 
     @catch_consul_errors
+    def _write_failsafe(self, value):
+        return self._client.kv.put(self.failsafe_path, value)
+
+    @staticmethod
+    def _run_and_handle_exceptions(method, *args, **kwargs):
+        retry = kwargs.pop('retry', None)
+        try:
+            return retry(method, *args, **kwargs) if retry else method(*args, **kwargs)
+        except (RetryFailedError, InvalidSession, HTTPException, HTTPError, socket.error, socket.timeout) as e:
+            raise ConsulError(e)
+        except ConsulException:
+            raise ReturnFalseException
+
+    @catch_return_false_exception
     def _update_leader(self):
+        retry = self._retry.copy()
+
+        self._run_and_handle_exceptions(self._do_refresh_session, True, retry=retry)
+
         if self._session:
-            self.retry(self._client.session.renew, self._session)
-            self._last_session_refresh = time.time()
+            cluster = self.cluster
+            leader_session = cluster and isinstance(cluster.leader, Leader) and cluster.leader.session
+            if leader_session != self._session:
+                retry.deadline = retry.stoptime - time.time()
+                if retry.deadline < 1:
+                    raise ConsulError('update_leader timeout')
+                logger.warning('Recreating the leader key due to session mismatch')
+                if cluster.leader:
+                    self._run_and_handle_exceptions(self._client.kv.delete, self.leader_path, cas=cluster.leader.index)
+
+                retry.deadline = retry.stoptime - time.time()
+                if retry.deadline < 0.5:
+                    raise ConsulError('update_leader timeout')
+                self._run_and_handle_exceptions(self._client.kv.put, self.leader_path,
+                                                self._name, acquire=self._session)
+
         return bool(self._session)
 
     @catch_consul_errors

patroni/dcs/etcd.py

@@ -10,6 +10,8 @@ import six
 import socket
 import time
 
+from collections import defaultdict
+from copy import deepcopy
 from dns.exception import DNSException
 from dns import resolver
 from urllib3 import Timeout
@@ -19,7 +21,8 @@ from six.moves.http_client import HTTPException
 from six.moves.urllib_parse import urlparse
 from threading import Thread
 
-from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState, TimelineHistory
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
+        TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
 from ..exceptions import DCSError
 from ..request import get as requests_get
 from ..utils import Retry, RetryFailedError, split_host_port, uri, USER_AGENT
@@ -216,10 +219,13 @@ class AbstractEtcdClientWithFailover(etcd.Client):
                 return response
             except (HTTPError, HTTPException, socket.error, socket.timeout) as e:
                 self.http.clear()
-                # switch to the next etcd node because we don't know exactly what happened,
-                # whether the key didn't received an update or there is a network problem.
-                if not retry and i + 1 < len(machines_cache):
-                    self.set_base_uri(machines_cache[i + 1])
+                if not retry:
+                    if len(machines_cache) == 1:
+                        self.set_base_uri(self._base_uri)  # trigger Etcd3 watcher restart
+                    # switch to the next etcd node because we don't know exactly what happened,
+                    # whether the key didn't received an update or there is a network problem.
+                    elif i + 1 < len(machines_cache):
+                        self.set_base_uri(machines_cache[i + 1])
                 if (isinstance(fields, dict) and fields.get("wait") == "true" and
                         isinstance(e, (ReadTimeoutError, ProtocolError))):
                     logger.debug("Watch timed out.")
@@ -457,6 +463,18 @@ class AbstractEtcd(AbstractDCS):
         if isinstance(raise_ex, Exception):
             raise raise_ex
 
+    def _run_and_handle_exceptions(self, method, *args, **kwargs):
+        retry = kwargs.pop('retry', self.retry)
+        try:
+            return retry(method, *args, **kwargs) if retry else method(*args, **kwargs)
+        except (RetryFailedError, etcd.EtcdConnectionFailed) as e:
+            raise self._client.ERROR_CLS(e)
+        except etcd.EtcdException as e:
+            self._handle_exception(e)
+            raise ReturnFalseException
+        except Exception as e:
+            self._handle_exception(e, raise_ex=self._client.ERROR_CLS('unexpected error'))
+
     @staticmethod
     def set_socket_options(sock, socket_options):
         if socket_options:
@@ -464,6 +482,7 @@ class AbstractEtcd(AbstractDCS):
                 sock.setsockopt(*opt)
 
     def get_etcd_client(self, config, client_cls):
+        config = deepcopy(config)
         if 'proxy' in config:
             config['use_proxies'] = True
             config['url'] = config['proxy']
@@ -588,92 +607,111 @@ class Etcd(AbstractEtcd):
     def member(node):
         return Member.from_node(node.modifiedIndex, os.path.basename(node.key), node.ttl, node.value)
 
-    def _load_cluster(self):
+    def _cluster_from_nodes(self, etcd_index, nodes):
+        # get initialize flag
+        initialize = nodes.get(self._INITIALIZE)
+        initialize = initialize and initialize.value
+
+        # get global dynamic configuration
+        config = nodes.get(self._CONFIG)
+        config = config and ClusterConfig.from_node(config.modifiedIndex, config.value)
+
+        # get timeline history
+        history = nodes.get(self._HISTORY)
+        history = history and TimelineHistory.from_node(history.modifiedIndex, history.value)
+
+        # get last know leader lsn and slots
+        status = nodes.get(self._STATUS)
+        if status:
+            try:
+                status = json.loads(status.value)
+                last_lsn = status.get(self._OPTIME)
+                slots = status.get('slots')
+            except Exception:
+                slots = last_lsn = None
+        else:
+            last_lsn = nodes.get(self._LEADER_OPTIME)
+            last_lsn = last_lsn and last_lsn.value
+            slots = None
+
+        try:
+            last_lsn = int(last_lsn)
+        except Exception:
+            last_lsn = 0
+
+        # get list of members
+        members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
+
+        # get leader
+        leader = nodes.get(self._LEADER)
+        if leader:
+            member = Member(-1, leader.value, None, {})
+            member = ([m for m in members if m.name == leader.value] or [member])[0]
+            index = etcd_index if etcd_index > leader.modifiedIndex else leader.modifiedIndex + 1
+            leader = Leader(index, leader.ttl, member)
+
+        # failover key
+        failover = nodes.get(self._FAILOVER)
+        if failover:
+            failover = Failover.from_node(failover.modifiedIndex, failover.value)
+
+        # get synchronization state
+        sync = nodes.get(self._SYNC)
+        sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
+
+        # get failsafe topology
+        failsafe = nodes.get(self._FAILSAFE)
+        try:
+            failsafe = json.loads(failsafe.value) if failsafe else None
+        except Exception:
+            failsafe = None
+
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+
+    def _cluster_loader(self, path):
+        result = self.retry(self._client.read, path, recursive=True)
+        nodes = {node.key[len(result.key):].lstrip('/'): node for node in result.leaves}
+        return self._cluster_from_nodes(result.etcd_index, nodes)
+
+    def _citus_cluster_loader(self, path):
+        clusters = defaultdict(dict)
+        result = self.retry(self._client.read, path, recursive=True)
+        for node in result.leaves:
+            key = node.key[len(result.key):].lstrip('/').split('/', 1)
+            if len(key) == 2 and citus_group_re.match(key[0]):
+                clusters[int(key[0])][key[1]] = node
+        return {group: self._cluster_from_nodes(result.etcd_index, nodes) for group, nodes in clusters.items()}
+
+    def _load_cluster(self, path, loader):
         cluster = None
         try:
-            result = self.retry(self._client.read, self.client_path(''), recursive=True)
-            nodes = {node.key[len(result.key):].lstrip('/'): node for node in result.leaves}
-
-            # get initialize flag
-            initialize = nodes.get(self._INITIALIZE)
-            initialize = initialize and initialize.value
-
-            # get global dynamic configuration
-            config = nodes.get(self._CONFIG)
-            config = config and ClusterConfig.from_node(config.modifiedIndex, config.value)
-
-            # get timeline history
-            history = nodes.get(self._HISTORY)
-            history = history and TimelineHistory.from_node(history.modifiedIndex, history.value)
-
-            # get last know leader lsn and slots
-            status = nodes.get(self._STATUS)
-            if status:
-                try:
-                    status = json.loads(status.value)
-                    last_lsn = status.get(self._OPTIME)
-                    slots = status.get('slots')
-                except Exception:
-                    slots = last_lsn = None
-            else:
-                last_lsn = nodes.get(self._LEADER_OPTIME)
-                last_lsn = last_lsn and last_lsn.value
-                slots = None
-
-            try:
-                last_lsn = int(last_lsn)
-            except Exception:
-                last_lsn = 0
-
-            # get list of members
-            members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
-
-            # get leader
-            leader = nodes.get(self._LEADER)
-            if leader:
-                member = Member(-1, leader.value, None, {})
-                member = ([m for m in members if m.name == leader.value] or [member])[0]
-                index = result.etcd_index if result.etcd_index > leader.modifiedIndex else leader.modifiedIndex + 1
-                leader = Leader(index, leader.ttl, member)
-
-            # failover key
-            failover = nodes.get(self._FAILOVER)
-            if failover:
-                failover = Failover.from_node(failover.modifiedIndex, failover.value)
-
-            # get synchronization state
-            sync = nodes.get(self._SYNC)
-            sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
-
-            cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+            cluster = loader(path)
         except etcd.EtcdKeyNotFound:
-            cluster = Cluster(None, None, None, None, [], None, None, None, None)
+            cluster = Cluster(None, None, None, None, [], None, None, None, None, None)
         except Exception as e:
             self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
         self._has_failed = False
         return cluster
 
     @catch_etcd_errors
-    def touch_member(self, data, permanent=False):
+    def touch_member(self, data):
         data = json.dumps(data, separators=(',', ':'))
-        return self._client.set(self.member_path, data, None if permanent else self._ttl)
+        return self._client.set(self.member_path, data, self._ttl)
 
     @catch_etcd_errors
     def take_leader(self):
         return self.retry(self._client.write, self.leader_path, self._name, ttl=self._ttl)
 
-    def attempt_to_acquire_leader(self, permanent=False):
+    def _do_attempt_to_acquire_leader(self):
         try:
-            return bool(self.retry(self._client.write,
-                                   self.leader_path,
-                                   self._name,
-                                   ttl=None if permanent else self._ttl,
-                                   prevExist=False))
+            return bool(self.retry(self._client.write, self.leader_path, self._name, ttl=self._ttl, prevExist=False))
         except etcd.EtcdAlreadyExist:
             logger.info('Could not take out TTL lock')
-        except (RetryFailedError, etcd.EtcdException):
-            pass
-        return False
+            return False
+
+    @catch_return_false_exception
+    def attempt_to_acquire_leader(self):
+        return self._run_and_handle_exceptions(self._do_attempt_to_acquire_leader, retry=None)
 
     @catch_etcd_errors
     def set_failover_value(self, value, index=None):
@@ -691,9 +729,20 @@ class Etcd(AbstractEtcd):
     def _write_status(self, value):
         return self._client.set(self.status_path, value)
 
+    def _do_update_leader(self):
+        try:
+            return self.retry(self._client.write, self.leader_path, self._name,
+                              prevValue=self._name, ttl=self._ttl) is not None
+        except etcd.EtcdKeyNotFound:
+            return self._do_attempt_to_acquire_leader()
+
     @catch_etcd_errors
+    def _write_failsafe(self, value):
+        return self._client.set(self.failsafe_path, value)
+
+    @catch_return_false_exception
     def _update_leader(self):
-        return self.retry(self._client.write, self.leader_path, self._name, prevValue=self._name, ttl=self._ttl)
+        return self._run_and_handle_exceptions(self._do_update_leader, retry=None)
 
     @catch_etcd_errors
     def initialize(self, create_new=True, sysid=""):

patroni/dcs/etcd3.py

@@ -10,9 +10,12 @@ import sys
 import time
 import urllib3
 
+from collections import defaultdict
 from threading import Condition, Lock, Thread
+from urllib3.exceptions import ReadTimeoutError, ProtocolError
 
-from . import ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
+from . import ClusterConfig, Cluster, Failover, Leader, Member, SyncState,\
+        TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
 from .etcd import AbstractEtcdClientWithFailover, AbstractEtcd, catch_etcd_errors
 from ..exceptions import DCSError, PatroniException
 from ..utils import deep_compare, enable_keepalive, iter_response_objects, RetryFailedError, USER_AGENT
@@ -89,7 +92,7 @@ class Unavailable(Etcd3ClientError):
     code = GRPCCode.Unavailable
 
 
-# https://github.com/etcd-io/etcd/blob/master/etcdserver/api/v3rpc/rpctypes/error.go
+# https://github.com/etcd-io/etcd/commits/main/api/v3rpc/rpctypes/error.go
 class LeaseNotFound(NotFound):
     error = "etcdserver: requested lease not found"
 
@@ -315,6 +318,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
     def prefix(self, key, retry=None):
         return self.range(key, prefix_range_end(key), retry)
 
+    @_handle_auth_errors
     def lease_grant(self, ttl, retry=None):
         return self.call_rpc('/lease/grant', {'TTL': ttl}, retry)['ID']
 
@@ -349,7 +353,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
     def deleteprefix(self, key, retry=None):
         return self.deleterange(key, prefix_range_end(key), retry=retry)
 
-    def watchrange(self, key, range_end=None, start_revision=None, filters=None):
+    def watchrange(self, key, range_end=None, start_revision=None, filters=None, read_timeout=None):
         """returns: response object"""
         params = build_range_request(key, range_end)
         if start_revision is not None:
@@ -357,11 +361,11 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
         params['filters'] = filters or []
         kwargs = self._prepare_common_parameters(1, self.read_timeout)
         request_executor = self._prepare_request(kwargs, {'create_request': params})
-        kwargs.update(timeout=urllib3.Timeout(connect=kwargs['timeout']), retries=0)
+        kwargs.update(timeout=urllib3.Timeout(connect=kwargs['timeout'], read=read_timeout), retries=0)
         return request_executor(self._MPOST, self._base_uri + self.version_prefix + '/watch', **kwargs)
 
-    def watchprefix(self, key, start_revision=None, filters=None):
-        return self.watchrange(key, prefix_range_end(key), start_revision, filters)
+    def watchprefix(self, key, start_revision=None, filters=None, read_timeout=None):
+        return self.watchrange(key, prefix_range_end(key), start_revision, filters, read_timeout)
 
 
 class KVCache(Thread):
@@ -450,7 +454,14 @@ class KVCache(Thread):
     def _do_watch(self, revision):
         with self._response_lock:
             self._response = None
-        response = self._client.watchprefix(self._dcs.cluster_prefix, revision)
+        # We do most of requests with timeouts. The only exception /watch requests to Etcd v3.
+        # In order to interrupt the /watch request we do socket.shutdown() from the main thread,
+        # which doesn't work on Windows. Therefore we want to use the last resort, `read_timeout`.
+        # Setting it to TTL will help to partially mitigate the problem.
+        # Setting it to lower value is not nice because for idling clusters it will increase
+        # the numbers of interrupts and reconnects.
+        read_timeout = self._dcs.ttl if os.name == 'nt' else None
+        response = self._client.watchprefix(self._dcs.cluster_prefix, revision, read_timeout=read_timeout)
         with self._response_lock:
             if self._response is None:
                 self._response = response
@@ -472,7 +483,9 @@ class KVCache(Thread):
         try:
             self._do_watch(result['header']['revision'])
         except Exception as e:
-            logger.error('watchprefix failed: %r', e)
+            # Following exceptions are expected on Windows because the /watch request  is done with `read_timeout`
+            if not (os.name == 'nt' and isinstance(e, (ReadTimeoutError, ProtocolError))):
+                logger.error('watchprefix failed: %r', e)
         finally:
             with self.condition:
                 self._is_ready = False
@@ -546,13 +559,18 @@ class PatroniEtcd3Client(Etcd3Client):
                 raise RetryFailedError('Exceeded retry deadline')
             self._kv_cache.condition.wait(timeout)
 
-    def get_cluster(self):
-        if self._kv_cache:
+    def get_cluster(self, path):
+        if self._kv_cache and path.startswith(self._etcd3.cluster_prefix):
             with self._kv_cache.condition:
                 self._wait_cache(self._etcd3._retry.deadline)
-                return self._kv_cache.copy()
+                ret = self._kv_cache.copy()
         else:
-            return self._etcd3.retry(self.prefix, self._etcd3.cluster_prefix).get('kvs', [])
+            ret = self._etcd3.retry(self.prefix, path).get('kvs', [])
+        for node in ret:
+            node.update({'key': base64_decode(node['key']),
+                         'value': base64_decode(node.get('value', '')),
+                         'lease': node.get('lease')})
+        return ret
 
     def call_rpc(self, method, fields, retry=None):
         ret = super(PatroniEtcd3Client, self).call_rpc(method, fields, retry)
@@ -598,8 +616,8 @@ class Etcd3(AbstractEtcd):
         if self.__do_not_watch:
             self._lease = None
 
-    def _do_refresh_lease(self, retry=None):
-        if self._lease and self._last_lease_refresh + self._loop_wait > time.time():
+    def _do_refresh_lease(self, force=False, retry=None):
+        if not force and self._lease and self._last_lease_refresh + self._loop_wait > time.time():
             return False
 
         if self._lease and not self._client.lease_keepalive(self._lease, retry):
@@ -629,78 +647,94 @@ class Etcd3(AbstractEtcd):
 
     @property
     def cluster_prefix(self):
-        return self.client_path('')
+        return self._base_path + '/' if self.is_citus_coordinator() else self.client_path('')
 
     @staticmethod
     def member(node):
         return Member.from_node(node['mod_revision'], os.path.basename(node['key']), node['lease'], node['value'])
 
-    def _load_cluster(self):
+    def _cluster_from_nodes(self, nodes):
+        # get initialize flag
+        initialize = nodes.get(self._INITIALIZE)
+        initialize = initialize and initialize['value']
+
+        # get global dynamic configuration
+        config = nodes.get(self._CONFIG)
+        config = config and ClusterConfig.from_node(config['mod_revision'], config['value'])
+
+        # get timeline history
+        history = nodes.get(self._HISTORY)
+        history = history and TimelineHistory.from_node(history['mod_revision'], history['value'])
+
+        # get last know leader lsn and slots
+        status = nodes.get(self._STATUS)
+        if status:
+            try:
+                status = json.loads(status['value'])
+                last_lsn = status.get(self._OPTIME)
+                slots = status.get('slots')
+            except Exception:
+                slots = last_lsn = None
+        else:
+            last_lsn = nodes.get(self._LEADER_OPTIME)
+            last_lsn = last_lsn and last_lsn['value']
+            slots = None
+
+        try:
+            last_lsn = int(last_lsn)
+        except Exception:
+            last_lsn = 0
+
+        # get list of members
+        members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
+
+        # get leader
+        leader = nodes.get(self._LEADER)
+        if not self._ctl and leader and leader['value'] == self._name and self._lease != leader.get('lease'):
+            logger.warning('I am the leader but not owner of the lease')
+
+        if leader:
+            member = Member(-1, leader['value'], None, {})
+            member = ([m for m in members if m.name == leader['value']] or [member])[0]
+            leader = Leader(leader['mod_revision'], leader['lease'], member)
+
+        # failover key
+        failover = nodes.get(self._FAILOVER)
+        if failover:
+            failover = Failover.from_node(failover['mod_revision'], failover['value'])
+
+        # get synchronization state
+        sync = nodes.get(self._SYNC)
+        sync = SyncState.from_node(sync and sync['mod_revision'], sync and sync['value'])
+
+        # get failsafe topology
+        failsafe = nodes.get(self._FAILSAFE)
+        try:
+            failsafe = json.loads(failsafe['value']) if failsafe else None
+        except Exception:
+            failsafe = None
+
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+
+    def _cluster_loader(self, path):
+        nodes = {node['key'][len(path):]: node
+                 for node in self._client.get_cluster(path)
+                 if node['key'].startswith(path)}
+        return self._cluster_from_nodes(nodes)
+
+    def _citus_cluster_loader(self, path):
+        clusters = defaultdict(dict)
+        path = self._base_path + '/'
+        for node in self._client.get_cluster(path):
+            key = node['key'][len(path):].split('/', 1)
+            if len(key) == 2 and citus_group_re.match(key[0]):
+                clusters[int(key[0])][key[1]] = node
+        return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()}
+
+    def _load_cluster(self, path, loader):
         cluster = None
         try:
-            path_len = len(self.cluster_prefix)
-
-            nodes = {}
-            for node in self._client.get_cluster():
-                node['key'] = base64_decode(node['key'])
-                node['value'] = base64_decode(node.get('value', ''))
-                node['lease'] = node.get('lease')
-                nodes[node['key'][path_len:].lstrip('/')] = node
-
-            # get initialize flag
-            initialize = nodes.get(self._INITIALIZE)
-            initialize = initialize and initialize['value']
-
-            # get global dynamic configuration
-            config = nodes.get(self._CONFIG)
-            config = config and ClusterConfig.from_node(config['mod_revision'], config['value'])
-
-            # get timeline history
-            history = nodes.get(self._HISTORY)
-            history = history and TimelineHistory.from_node(history['mod_revision'], history['value'])
-
-            # get last know leader lsn and slots
-            status = nodes.get(self._STATUS)
-            if status:
-                try:
-                    status = json.loads(status['value'])
-                    last_lsn = status.get(self._OPTIME)
-                    slots = status.get('slots')
-                except Exception:
-                    slots = last_lsn = None
-            else:
-                last_lsn = nodes.get(self._LEADER_OPTIME)
-                last_lsn = last_lsn and last_lsn['value']
-                slots = None
-
-            try:
-                last_lsn = int(last_lsn)
-            except Exception:
-                last_lsn = 0
-
-            # get list of members
-            members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
-
-            # get leader
-            leader = nodes.get(self._LEADER)
-            if not self._ctl and leader and leader['value'] == self._name and self._lease != leader.get('lease'):
-                logger.warning('I am the leader but not owner of the lease')
-
-            if leader:
-                member = Member(-1, leader['value'], None, {})
-                member = ([m for m in members if m.name == leader['value']] or [member])[0]
-                leader = Leader(leader['mod_revision'], leader['lease'], member)
-
-            # failover key
-            failover = nodes.get(self._FAILOVER)
-            if failover:
-                failover = Failover.from_node(failover['mod_revision'], failover['value'])
-
-            # get synchronization state
-            sync = nodes.get(self._SYNC)
-            sync = SyncState.from_node(sync and sync['mod_revision'], sync and sync['value'])
-
-            cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+            cluster = loader(path)
         except UnsupportedEtcdVersion:
             raise
         except Exception as e:
@@ -709,12 +743,11 @@ class Etcd3(AbstractEtcd):
         return cluster
 
     @catch_etcd_errors
-    def touch_member(self, data, permanent=False):
-        if not permanent:
-            try:
-                self.refresh_lease()
-            except Etcd3Error:
-                return False
+    def touch_member(self, data):
+        try:
+            self.refresh_lease()
+        except Etcd3Error:
+            return False
 
         cluster = self.cluster
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
@@ -724,7 +757,7 @@ class Etcd3(AbstractEtcd):
 
         data = json.dumps(data, separators=(',', ':'))
         try:
-            return self._client.put(self.member_path, data, None if permanent else self._lease)
+            return self._client.put(self.member_path, data, self._lease)
         except LeaseNotFound:
             self._lease = None
             logger.error('Our lease disappeared from Etcd, can not "touch_member"')
@@ -733,21 +766,41 @@ class Etcd3(AbstractEtcd):
     def take_leader(self):
         return self.retry(self._client.put, self.leader_path, self._name, self._lease)
 
-    @catch_etcd_errors
-    def _do_attempt_to_acquire_leader(self, permanent):
+    def _do_attempt_to_acquire_leader(self, retry):
+        def _retry(*args, **kwargs):
+            kwargs['retry'] = retry
+            return retry(*args, **kwargs)
+
         try:
-            return self.retry(self._client.put, self.leader_path, self._name, None if permanent else self._lease, 0)
+            return _retry(self._client.put, self.leader_path, self._name, self._lease, 0)
         except LeaseNotFound:
-            self._lease = None
             logger.error('Our lease disappeared from Etcd. Will try to get a new one and retry attempt')
-            self.refresh_lease()
-            return self.retry(self._client.put, self.leader_path, self._name, None if permanent else self._lease, 0)
+            self._lease = None
+            retry.deadline = retry.stoptime - time.time()
 
-    def attempt_to_acquire_leader(self, permanent=False):
-        if not self._lease and not permanent:
-            self.refresh_lease()
+            _retry(self._do_refresh_lease)
 
-        ret = self._do_attempt_to_acquire_leader(permanent)
+            retry.deadline = retry.stoptime - time.time()
+            if retry.deadline < 1:
+                raise Etcd3Error('_do_attempt_to_acquire_leader timeout')
+
+            return _retry(self._client.put, self.leader_path, self._name, self._lease, 0)
+
+    @catch_return_false_exception
+    def attempt_to_acquire_leader(self):
+        retry = self._retry.copy()
+
+        def _retry(*args, **kwargs):
+            kwargs['retry'] = retry
+            return retry(*args, **kwargs)
+
+        self._run_and_handle_exceptions(self._do_refresh_lease, retry=_retry)
+
+        retry.deadline = retry.stoptime - time.time()
+        if retry.deadline < 1:
+            raise Etcd3Error('attempt_to_acquire_leader timeout')
+
+        ret = self._run_and_handle_exceptions(self._do_attempt_to_acquire_leader, retry, retry=None)
         if not ret:
             logger.info('Could not take out TTL lock')
         return ret
@@ -769,17 +822,32 @@ class Etcd3(AbstractEtcd):
         return self._client.put(self.status_path, value)
 
     @catch_etcd_errors
+    def _write_failsafe(self, value):
+        return self._client.put(self.failsafe_path, value)
+
+    @catch_return_false_exception
     def _update_leader(self):
-        if not self._lease:
-            self.refresh_lease()
-        elif self.retry(self._client.lease_keepalive, self._lease):
-            self._last_lease_refresh = time.time()
+        retry = self._retry.copy()
+
+        def _retry(*args, **kwargs):
+            kwargs['retry'] = retry
+            return retry(*args, **kwargs)
+
+        self._run_and_handle_exceptions(self._do_refresh_lease, True, retry=_retry)
 
         if self._lease:
             cluster = self.cluster
             leader_lease = cluster and isinstance(cluster.leader, Leader) and cluster.leader.session
             if leader_lease != self._lease:
-                self.take_leader()
+                retry.deadline = retry.stoptime - time.time()
+                if retry.deadline < 1:
+                    raise Etcd3Error('update_leader timeout')
+
+                try:
+                    self._run_and_handle_exceptions(self._client.put, self.leader_path,
+                                                    self._name, self._lease, retry=_retry)
+                except ReturnFalseException:
+                    pass
         return bool(self._lease)
 
     @catch_etcd_errors
@@ -798,7 +866,7 @@ class Etcd3(AbstractEtcd):
 
     @catch_etcd_errors
     def delete_cluster(self):
-        return self.retry(self._client.deleteprefix, self.cluster_prefix)
+        return self.retry(self._client.deleteprefix, self.client_path(''))
 
     @catch_etcd_errors
     def set_history_value(self, value):

patroni/dcs/exhibitor.py

@@ -19,7 +19,7 @@ class ExhibitorEnsembleProvider(object):
         self._uri_path = uri_path
         self._poll_interval = poll_interval
         self._exhibitors = hosts
-        self._master_exhibitors = hosts
+        self._boot_exhibitors = hosts
         self._zookeeper_hosts = ''
         self._next_poll = None
         while not self.poll():
@@ -32,7 +32,7 @@ class ExhibitorEnsembleProvider(object):
 
         json = self._query_exhibitors(self._exhibitors)
         if not json:
-            json = self._query_exhibitors(self._master_exhibitors)
+            json = self._query_exhibitors(self._boot_exhibitors)
 
         if isinstance(json, dict) and 'servers' in json and 'port' in json:
             self._next_poll = time.time() + self._poll_interval
@@ -68,7 +68,7 @@ class Exhibitor(ZooKeeper):
         config['hosts'] = self._ensemble_provider.zookeeper_hosts
         super(Exhibitor, self).__init__(config)
 
-    def _load_cluster(self):
+    def _load_cluster(self, path, loader):
         if self._ensemble_provider.poll():
             self._client.set_hosts(self._ensemble_provider.zookeeper_hosts)
-        return super(Exhibitor, self)._load_cluster()
+        return super(Exhibitor, self)._load_cluster(path, loader)

patroni/dcs/kubernetes.py

@@ -1,3 +1,5 @@
+import atexit
+import base64
 import datetime
 import functools
 import json
@@ -6,17 +8,20 @@ import os
 import random
 import socket
 import six
-import sys
+import tempfile
 import time
 import urllib3
 import yaml
 
+from collections import defaultdict
+from copy import deepcopy
 from urllib3 import Timeout
 from urllib3.exceptions import HTTPError
 from six.moves.http_client import HTTPException
 from threading import Condition, Lock, Thread
 
-from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState, TimelineHistory
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
+        TimelineHistory, CITUS_COORDINATOR_GROUP_ID, citus_group_re
 from ..exceptions import DCSError
 from ..utils import deep_compare, iter_response_objects, keepalive_socket_options,\
         Retry, RetryFailedError, tzutc, uri, USER_AGENT
@@ -28,12 +33,34 @@ SERVICE_HOST_ENV_NAME = 'KUBERNETES_SERVICE_HOST'
 SERVICE_PORT_ENV_NAME = 'KUBERNETES_SERVICE_PORT'
 SERVICE_TOKEN_FILENAME = '/var/run/secrets/kubernetes.io/serviceaccount/token'
 SERVICE_CERT_FILENAME = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'
+__temp_files = []
 
 
 class KubernetesError(DCSError):
     pass
 
 
+def _cleanup_temp_files():
+    global __temp_files
+    for temp_file in __temp_files:
+        try:
+            os.remove(temp_file)
+        except OSError:
+            pass
+    __temp_files = []
+
+
+def _create_temp_file(content):
+    if len(__temp_files) == 0:
+        atexit.register(_cleanup_temp_files)
+
+    fd, name = tempfile.mkstemp()
+    os.write(fd, content)
+    os.close(fd)
+    __temp_files.append(name)
+    return name
+
+
 # this function does the same mapping of snake_case => camelCase for > 97% of cases as autogenerated swagger code
 def to_camel_case(value):
     reserved = {'api', 'apiv3', 'cidr', 'cpu', 'csi', 'id', 'io', 'ip', 'ipc', 'pid', 'tls', 'uri', 'url', 'uuid'}
@@ -93,6 +120,13 @@ class K8sConfig(object):
             if c['name'] == name:
                 return c[section]
 
+    def _pool_config_from_file_or_data(self, config, file_key_name, pool_key_name):
+        data_key_name = file_key_name + '-data'
+        if data_key_name in config:
+            self.pool_config[pool_key_name] = _create_temp_file(base64.b64decode(config[data_key_name]))
+        elif file_key_name in config:
+            self.pool_config[pool_key_name] = config[file_key_name]
+
     def load_kube_config(self, context=None):
         with open(os.path.expanduser(KUBE_CONFIG_DEFAULT_LOCATION)) as f:
             config = yaml.safe_load(f)
@@ -103,10 +137,9 @@ class K8sConfig(object):
 
         self._server = cluster['server'].rstrip('/')
         if self._server.startswith('https'):
-            self.pool_config.update({v: user[k] for k, v in {'client-certificate': 'cert_file',
-                                                             'client-key': 'key_file'}.items() if k in user})
-            if 'certificate-authority' in cluster:
-                self.pool_config['ca_certs'] = cluster['certificate-authority']
+            self._pool_config_from_file_or_data(user, 'client-certificate', 'cert_file')
+            self._pool_config_from_file_or_data(user, 'client-key', 'key_file')
+            self._pool_config_from_file_or_data(cluster, 'certificate-authority', 'ca_certs')
             self.pool_config['cert_reqs'] = 'CERT_NONE' if cluster.get('insecure-skip-tls-verify') else 'CERT_REQUIRED'
         if user.get('token'):
             self._make_headers(token=user['token'])
@@ -208,7 +241,7 @@ class K8sClient(object):
 
         def set_base_uri(self, value):
             logger.info('Selected new K8s API server endpoint %s', value)
-            # We will connect by IP of the master node which is not listed as alternative name
+            # We will connect by IP of the K8s master node which is not listed as alternative name
             self.pool_manager.connection_pool_kw['assert_hostname'] = False
             self._base_uri = value
 
@@ -493,16 +526,10 @@ class CoreV1ApiProxy(object):
 
 
 def catch_kubernetes_errors(func):
-    def wrapper(*args, **kwargs):
+    def wrapper(self, *args, **kwargs):
         try:
-            return func(*args, **kwargs)
-        except k8s_client.rest.ApiException as e:
-            if e.status == 403:
-                logger.exception('Permission denied')
-            elif e.status != 409:  # Object exists or conflict in resource_version
-                logger.exception('Unexpected error from Kubernetes API')
-            return False
-        except (RetryFailedError, K8sException):
+            return self._run_and_handle_exceptions(func, self, *args, **kwargs)
+        except KubernetesError:
             return False
     return wrapper
 
@@ -662,8 +689,10 @@ class ObjectCache(Thread):
 
 class Kubernetes(AbstractDCS):
 
+    _CITUS_LABEL = 'citus-group'
+
     def __init__(self, config):
-        self._labels = config['labels']
+        self._labels = deepcopy(config['labels'])
         self._labels[config.get('scope_label', 'cluster-name')] = config['scope']
         self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
         self._namespace = config.get('namespace') or 'default'
@@ -671,13 +700,16 @@ class Kubernetes(AbstractDCS):
         self._ca_certs = os.environ.get('PATRONI_KUBERNETES_CACERT', config.get('cacert')) or SERVICE_CERT_FILENAME
         config['namespace'] = ''
         super(Kubernetes, self).__init__(config)
+        if self._citus_group:
+            self._labels[self._CITUS_LABEL] = self._citus_group
+
         self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
                             retry_exceptions=KubernetesRetriableException)
         self._ttl = None
         try:
             k8s_config.load_incluster_config(ca_certs=self._ca_certs)
         except k8s_config.ConfigException:
-            k8s_config.load_kube_config(context=config.get('context', 'local'))
+            k8s_config.load_kube_config(context=config.get('context', 'kind-kind'))
 
         self.__my_pod = None
         self.__ips = [] if config.get('patronictl') else [config.get('pod_ip')]
@@ -712,12 +744,25 @@ class Kubernetes(AbstractDCS):
         kwargs['_retry'] = retry
         return retry(*args, **kwargs)
 
+    @staticmethod
+    def _run_and_handle_exceptions(method, *args, **kwargs):
+        try:
+            return method(*args, **kwargs)
+        except k8s_client.rest.ApiException as e:
+            if e.status == 403:
+                logger.exception('Permission denied')
+            elif e.status != 409:  # Object exists or conflict in resource_version
+                logger.exception('Unexpected error from Kubernetes API')
+            return False
+        except (RetryFailedError, K8sException) as e:
+            raise KubernetesError(e)
+
     def client_path(self, path):
         return super(Kubernetes, self).client_path(path)[1:].replace('/', '-')
 
     @property
     def leader_path(self):
-        return self._base_path[1:] if self._api.use_endpoints else super(Kubernetes, self).leader_path
+        return super(Kubernetes, self).leader_path[:-7 if self._api.use_endpoints else None]
 
     def set_ttl(self, ttl):
         ttl = int(ttl)
@@ -749,88 +794,137 @@ class Kubernetes(AbstractDCS):
                 raise RetryFailedError('Exceeded retry deadline')
             self._condition.wait(timeout)
 
-    def _load_cluster(self):
+    def _cluster_from_nodes(self, group, nodes, pods):
+        members = [self.member(pod) for pod in pods]
+        path = self._base_path[1:] + '-'
+        if group:
+            path += group + '-'
+
+        config = nodes.get(path + self._CONFIG)
+        metadata = config and config.metadata
+        annotations = metadata and metadata.annotations or {}
+
+        # get initialize flag
+        initialize = annotations.get(self._INITIALIZE)
+
+        # get global dynamic configuration
+        config = ClusterConfig.from_node(metadata and metadata.resource_version,
+                                         annotations.get(self._CONFIG) or '{}',
+                                         metadata.resource_version if self._CONFIG in annotations else 0)
+
+        # get timeline history
+        history = TimelineHistory.from_node(metadata and metadata.resource_version,
+                                            annotations.get(self._HISTORY) or '[]')
+
+        leader_path = path[:-1] if self._api.use_endpoints else path + self._LEADER
+        leader = nodes.get(leader_path)
+        metadata = leader and leader.metadata
+        if leader_path == self.leader_path:  # We want to memorize leader_resource_version only for our cluster
+            self._leader_resource_version = metadata.resource_version if metadata else None
+        annotations = metadata and metadata.annotations or {}
+
+        # get last known leader lsn
+        last_lsn = annotations.get(self._OPTIME)
+        try:
+            last_lsn = 0 if last_lsn is None else int(last_lsn)
+        except Exception:
+            last_lsn = 0
+
+        # get permanent slots state (confirmed_flush_lsn)
+        slots = annotations.get('slots')
+        try:
+            slots = slots and json.loads(slots)
+        except Exception:
+            slots = None
+
+        # get failsafe topology
+        failsafe = annotations.get(self._FAILSAFE)
+        try:
+            failsafe = json.loads(failsafe) if failsafe else None
+        except Exception:
+            failsafe = None
+
+        # get leader
+        leader_record = {n: annotations.get(n) for n in (self._LEADER, 'acquireTime',
+                         'ttl', 'renewTime', 'transitions') if n in annotations}
+        # We want to memorize leader_observed_record and update leader_observed_time only for our cluster
+        if leader_path == self.leader_path and (leader_record or self._leader_observed_record)\
+                and leader_record != self._leader_observed_record:
+            self._leader_observed_record = leader_record
+            self._leader_observed_time = time.time()
+
+        leader = leader_record.get(self._LEADER)
+        try:
+            ttl = int(leader_record.get('ttl')) or self._ttl
+        except (TypeError, ValueError):
+            ttl = self._ttl
+
+        # We want to check validity of the leader record only for our own cluster
+        if leader_path == self.leader_path and\
+                not (metadata and self._leader_observed_time and self._leader_observed_time + ttl >= time.time()):
+            leader = None
+
+        if metadata:
+            member = Member(-1, leader, None, {})
+            member = ([m for m in members if m.name == leader] or [member])[0]
+            leader = Leader(metadata.resource_version, None, member)
+
+        # failover key
+        failover = nodes.get(path + self._FAILOVER)
+        metadata = failover and failover.metadata
+        failover = Failover.from_node(metadata and metadata.resource_version,
+                                      metadata and (metadata.annotations or {}).copy())
+
+        # get synchronization state
+        sync = nodes.get(path + self._SYNC)
+        metadata = sync and sync.metadata
+        sync = SyncState.from_node(metadata and metadata.resource_version,  metadata and metadata.annotations)
+
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+
+    def _cluster_loader(self, path):
+        return self._cluster_from_nodes(path['group'], path['nodes'], path['pods'])
+
+    def _citus_cluster_loader(self, path):
+        clusters = defaultdict(lambda: {'pods': [], 'nodes': {}})
+
+        for pod in path['pods']:
+            group = pod.metadata.labels.get(self._CITUS_LABEL)
+            if group and citus_group_re.match(group):
+                clusters[group]['pods'].append(pod)
+
+        for name, kind in path['nodes'].items():
+            group = kind.metadata.labels.get(self._CITUS_LABEL)
+            if group and citus_group_re.match(group):
+                clusters[group]['nodes'][name] = kind
+        return {int(group): self._cluster_from_nodes(group, value['nodes'], value['pods'])
+                for group, value in clusters.items()}
+
+    def __load_cluster(self, group, loader):
         stop_time = time.time() + self._retry.deadline
         self._api.refresh_api_servers_cache()
         try:
             with self._condition:
                 self._wait_caches(stop_time)
-
-                members = [self.member(pod) for pod in self._pods.copy().values()]
-                nodes = self._kinds.copy()
-
-            config = nodes.get(self.config_path)
-            metadata = config and config.metadata
-            annotations = metadata and metadata.annotations or {}
-
-            # get initialize flag
-            initialize = annotations.get(self._INITIALIZE)
-
-            # get global dynamic configuration
-            config = ClusterConfig.from_node(metadata and metadata.resource_version,
-                                             annotations.get(self._CONFIG) or '{}',
-                                             metadata.resource_version if self._CONFIG in annotations else 0)
-
-            # get timeline history
-            history = TimelineHistory.from_node(metadata and metadata.resource_version,
-                                                annotations.get(self._HISTORY) or '[]')
-
-            leader = nodes.get(self.leader_path)
-            metadata = leader and leader.metadata
-            self._leader_resource_version = metadata.resource_version if metadata else None
-            annotations = metadata and metadata.annotations or {}
-
-            # get last known leader lsn
-            last_lsn = annotations.get(self._OPTIME)
-            try:
-                last_lsn = 0 if last_lsn is None else int(last_lsn)
-            except Exception:
-                last_lsn = 0
-
-            # get permanent slots state (confirmed_flush_lsn)
-            slots = annotations.get('slots')
-            try:
-                slots = slots and json.loads(slots)
-            except Exception:
-                slots = None
-
-            # get leader
-            leader_record = {n: annotations.get(n) for n in (self._LEADER, 'acquireTime',
-                             'ttl', 'renewTime', 'transitions') if n in annotations}
-            if (leader_record or self._leader_observed_record) and leader_record != self._leader_observed_record:
-                self._leader_observed_record = leader_record
-                self._leader_observed_time = time.time()
-
-            leader = leader_record.get(self._LEADER)
-            try:
-                ttl = int(leader_record.get('ttl')) or self._ttl
-            except (TypeError, ValueError):
-                ttl = self._ttl
-
-            if not metadata or not self._leader_observed_time or self._leader_observed_time + ttl < time.time():
-                leader = None
-
-            if metadata:
-                member = Member(-1, leader, None, {})
-                member = ([m for m in members if m.name == leader] or [member])[0]
-                leader = Leader(metadata.resource_version, None, member)
-
-            # failover key
-            failover = nodes.get(self.failover_path)
-            metadata = failover and failover.metadata
-            failover = Failover.from_node(metadata and metadata.resource_version,
-                                          metadata and (metadata.annotations or {}).copy())
-
-            # get synchronization state
-            sync = nodes.get(self.sync_path)
-            metadata = sync and sync.metadata
-            sync = SyncState.from_node(metadata and metadata.resource_version,  metadata and metadata.annotations)
-
-            return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+                pods = [pod for pod in self._pods.copy().values()
+                        if not group or pod.metadata.labels.get(self._CITUS_LABEL) == group]
+                nodes = {name: kind for name, kind in self._kinds.copy().items()
+                         if not group or kind.metadata.labels.get(self._CITUS_LABEL) == group}
+            return loader({'group': group, 'pods': pods, 'nodes': nodes})
         except Exception:
             logger.exception('get_cluster')
             raise KubernetesError('Kubernetes API is not responding properly')
 
+    def _load_cluster(self, path, loader):
+        group = self._citus_group if path == self.client_path('') else None
+        return self.__load_cluster(group, loader)
+
+    def get_citus_coordinator(self):
+        try:
+            return self.__load_cluster(str(CITUS_COORDINATOR_GROUP_ID), self._cluster_loader)
+        except Exception as e:
+            logger.error('Failed to load Citus coordinator cluster from Kubernetes: %r', e)
+
     @staticmethod
     def compare_ports(p1, p2):
         return p1.name == p2.name and p1.port == p2.port and (p1.protocol or 'TCP') == (p2.protocol or 'TCP')
@@ -950,7 +1044,8 @@ class Kubernetes(AbstractDCS):
             if not self._api.create_namespaced_service(self._namespace, body):
                 return
         except Exception as e:
-            if not isinstance(e, k8s_client.rest.ApiException) or e.status != 409:  # Service already exists
+            # 409 - service already exists, 403 - creation forbidden
+            if not isinstance(e, k8s_client.rest.ApiException) or e.status not in (409, 403):
                 return logger.exception('create_config_service failed')
         self._should_create_config_service = False
 
@@ -960,6 +1055,9 @@ class Kubernetes(AbstractDCS):
     def _write_status(self, value):
         """Unused"""
 
+    def _write_failsafe(self, value):
+        """Unused"""
+
     def _update_leader(self):
         """Unused"""
 
@@ -978,16 +1076,19 @@ class Kubernetes(AbstractDCS):
             else:
                 logger.exception('Permission denied' if e.status == 403 else 'Unexpected error from Kubernetes API')
                 return False
-        except (RetryFailedError, K8sException):
-            return False
+        except (RetryFailedError, K8sException) as e:
+            raise KubernetesError(e)
 
+        # if we are here, that means update failed with 409
         retry.deadline = retry.stoptime - time.time()
         if retry.deadline < 1:
-            return False
+            return False  # No time for retry. Tell ha.py that we have to demote due to failed update.
 
         # Try to get the latest version directly from K8s API instead of relying on async cache
         try:
             kind = _retry(self._api.read_namespaced_kind, self.leader_path, self._namespace)
+        except (RetryFailedError, K8sException) as e:
+            raise KubernetesError(e)
         except Exception as e:
             logger.error('Failed to get the leader object "%s": %r', self.leader_path, e)
             return False
@@ -1005,9 +1106,10 @@ class Kubernetes(AbstractDCS):
         if kind and (kind_annotations.get(self._LEADER) != self._name or kind_resource_version == resource_version):
             return False
 
-        return self.patch_or_create(self.leader_path, annotations, kind_resource_version, ips=ips, retry=_retry)
+        return self._run_and_handle_exceptions(self._patch_or_create, self.leader_path, annotations,
+                                               kind_resource_version, ips=ips, retry=_retry)
 
-    def update_leader(self, last_lsn, slots=None):
+    def update_leader(self, last_lsn, slots=None, failsafe=None):
         kind = self._kinds.get(self.leader_path)
         kind_annotations = kind and kind.metadata.annotations or {}
 
@@ -1021,14 +1123,17 @@ class Kubernetes(AbstractDCS):
                        'transitions': leader_observed_record.get('transitions') or '0'}
         if last_lsn:
             annotations[self._OPTIME] = str(last_lsn)
-            annotations['slots'] = json.dumps(slots) if slots else None
+            annotations['slots'] = json.dumps(slots, separators=(',', ':')) if slots else None
+
+        if failsafe is not None:
+            annotations[self._FAILSAFE] = json.dumps(failsafe, separators=(',', ':')) if failsafe else None
 
         resource_version = kind and kind.metadata.resource_version
         return self._update_leader_with_retry(annotations, resource_version, self.__ips)
 
-    def attempt_to_acquire_leader(self, permanent=False):
+    def attempt_to_acquire_leader(self):
         now = datetime.datetime.now(tzutc).isoformat()
-        annotations = {self._LEADER: self._name, 'ttl': str(sys.maxsize if permanent else self._ttl),
+        annotations = {self._LEADER: self._name, 'ttl': str(self._ttl),
                        'renewTime': now, 'acquireTime': now, 'transitions': '0'}
         if self._leader_observed_record:
             try:
@@ -1042,7 +1147,19 @@ class Kubernetes(AbstractDCS):
                 annotations['acquireTime'] = self._leader_observed_record.get('acquireTime') or now
             annotations['transitions'] = str(transitions)
         ips = [] if self._api.use_endpoints else None
-        ret = self.patch_or_create(self.leader_path, annotations, self._leader_resource_version, ips=ips)
+
+        try:
+            ret = self._patch_or_create(self.leader_path, annotations,
+                                        self._leader_resource_version, retry=self.retry, ips=ips)
+        except k8s_client.rest.ApiException as e:
+            if e.status == 409 and self._leader_resource_version:  # Conflict in resource_version
+                # Terminate watchers, it could be a sign that K8s API is in a failed state
+                self._kinds.kill_stream()
+                self._pods.kill_stream()
+            ret = False
+        except (RetryFailedError, K8sException) as e:
+            raise KubernetesError(e)
+
         if not ret:
             logger.info('Could not take out TTL lock')
         return ret
@@ -1068,11 +1185,11 @@ class Kubernetes(AbstractDCS):
         return self.patch_or_create_config({self._CONFIG: value}, index, bool(self._config_resource_version), False)
 
     @catch_kubernetes_errors
-    def touch_member(self, data, permanent=False):
+    def touch_member(self, data):
         cluster = self.cluster
         if cluster and cluster.leader and cluster.leader.name == self._name:
             role = 'master'
-        elif data['state'] == 'running' and data['role'] != 'master':
+        elif data['state'] == 'running' and data['role'] not in ('master', 'primary'):
             role = data['role']
         else:
             role = None

patroni/dcs/raft.py

@@ -4,18 +4,24 @@ import os
 import threading
 import time
 
+from collections import defaultdict
 from pysyncobj import SyncObj, SyncObjConf, replicated, FAIL_REASON
 from pysyncobj.dns_resolver import globalDnsResolver
 from pysyncobj.node import TCPNode
 from pysyncobj.transport import TCPTransport, CONNECTION_STATE
 from pysyncobj.utility import TcpUtility
 
-from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
+from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory, citus_group_re
+from ..exceptions import DCSError
 from ..utils import validate_directory
 
 logger = logging.getLogger(__name__)
 
 
+class RaftError(DCSError):
+    pass
+
+
 class _TCPTransport(TCPTransport):
 
     def __init__(self, syncObj, selfNode, otherNodes):
@@ -39,9 +45,9 @@ setattr(TCPNode, 'ip', property(resolve_host))
 
 class SyncObjUtility(object):
 
-    def __init__(self, otherNodes, conf):
+    def __init__(self, otherNodes, conf, retry_timeout=10):
         self._nodes = otherNodes
-        self._utility = TcpUtility(conf.password)
+        self._utility = TcpUtility(conf.password, retry_timeout/max(1, len(otherNodes)))
 
     def executeCommand(self, command):
         try:
@@ -58,11 +64,11 @@ class SyncObjUtility(object):
 
 class DynMemberSyncObj(SyncObj):
 
-    def __init__(self, selfAddress, partnerAddrs, conf):
+    def __init__(self, selfAddress, partnerAddrs, conf, retry_timeout=10):
         self.__early_apply_local_log = selfAddress is not None
         self.applied_local_log = False
 
-        utility = SyncObjUtility(partnerAddrs, conf)
+        utility = SyncObjUtility(partnerAddrs, conf, retry_timeout)
         members = utility.getMembers()
         add_self = members and selfAddress not in members
 
@@ -97,7 +103,7 @@ class KVStoreTTL(DynMemberSyncObj):
         self.__on_set = on_set
         self.__on_delete = on_delete
         self.__limb = {}
-        self.__retry_timeout = None
+        self.set_retry_timeout(int(config.get('retry_timeout') or 10))
 
         self_addr = config.get('self_addr')
         partner_addrs = set(config.get('partner_addrs', []))
@@ -121,7 +127,7 @@ class KVStoreTTL(DynMemberSyncObj):
                            journalFile=(file_template + '.journal' if self_addr else None),
                            onReady=on_ready, dynamicMembershipChange=True)
 
-        super(KVStoreTTL, self).__init__(self_addr, partner_addrs, conf)
+        super(KVStoreTTL, self).__init__(self_addr, partner_addrs, conf, self.__retry_timeout)
         self.__data = {}
 
     @staticmethod
@@ -156,7 +162,7 @@ class KVStoreTTL(DynMemberSyncObj):
             elif deadline:
                 timeout = deadline - time.time()
                 if timeout <= 0:
-                    break
+                    raise RaftError('timeout')
             time.sleep(1)
         return False
 
@@ -175,7 +181,7 @@ class KVStoreTTL(DynMemberSyncObj):
             self.__on_set(key, value)
         return True
 
-    def set(self, key, value, ttl=None, **kwargs):
+    def set(self, key, value, ttl=None, handle_raft_error=True, **kwargs):
         old_value = self.__data.get(key, {})
         if not self.__check_requirements(old_value, **kwargs):
             return False
@@ -184,7 +190,12 @@ class KVStoreTTL(DynMemberSyncObj):
         value['created'] = old_value.get('created', value['updated'])
         if ttl:
             value['expire'] = value['updated'] + ttl
-        return self.retry(self._set, key, value, **kwargs)
+        try:
+            return self.retry(self._set, key, value, **kwargs)
+        except RaftError:
+            if not handle_raft_error:
+                raise
+            return False
 
     def __pop(self, key):
         self.__data.pop(key)
@@ -206,7 +217,10 @@ class KVStoreTTL(DynMemberSyncObj):
     def delete(self, key, recursive=False, **kwargs):
         if not recursive and not self.__check_requirements(self.__data.get(key, {}), **kwargs):
             return False
-        return self.retry(self._delete, key, recursive=recursive, **kwargs)
+        try:
+            return self.retry(self._delete, key, recursive=recursive, **kwargs)
+        except RaftError:
+            return False
 
     @staticmethod
     def __values_match(old, new):
@@ -275,7 +289,6 @@ class Raft(AbstractDCS):
                 break
             else:
                 logger.info('waiting on raft')
-        self.set_retry_timeout(int(config.get('retry_timeout') or 10))
 
     def _on_set(self, key, value):
         leader = (self._sync_obj.get(self.leader_path) or {}).get('value')
@@ -307,13 +320,7 @@ class Raft(AbstractDCS):
     def member(key, value):
         return Member.from_node(value['index'], os.path.basename(key), None, value['value'])
 
-    def _load_cluster(self):
-        prefix = self.client_path('')
-        response = self._sync_obj.get(prefix, recursive=True)
-        if not response:
-            return Cluster(None, None, None, None, [], None, None, None, None)
-        nodes = {os.path.relpath(key, prefix).replace('\\', '/'): value for key, value in response.items()}
-
+    def _cluster_from_nodes(self, nodes):
         # get initialize flag
         initialize = nodes.get(self._INITIALIZE)
         initialize = initialize and initialize['value']
@@ -364,7 +371,33 @@ class Raft(AbstractDCS):
         sync = nodes.get(self._SYNC)
         sync = SyncState.from_node(sync and sync['index'], sync and sync['value'])
 
-        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+        # get failsafe topology
+        failsafe = nodes.get(self._FAILSAFE)
+        try:
+            failsafe = json.loads(failsafe['value']) if failsafe else None
+        except Exception:
+            failsafe = None
+
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+
+    def _cluster_loader(self, path):
+        response = self._sync_obj.get(path, recursive=True)
+        if not response:
+            return Cluster(None, None, None, None, [], None, None, None, None, None)
+        nodes = {key[len(path):]: value for key, value in response.items()}
+        return self._cluster_from_nodes(nodes)
+
+    def _citus_cluster_loader(self, path):
+        clusters = defaultdict(dict)
+        response = self._sync_obj.get(path, recursive=True)
+        for key, value in response.items():
+            key = key[len(path):].split('/', 1)
+            if len(key) == 2 and citus_group_re.match(key[0]):
+                clusters[int(key[0])][key[1]] = value
+        return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()}
+
+    def _load_cluster(self, path, loader):
+        return loader(path)
 
     def _write_leader_optime(self, last_lsn):
         return self._sync_obj.set(self.leader_optime_path, last_lsn, timeout=1)
@@ -372,15 +405,18 @@ class Raft(AbstractDCS):
     def _write_status(self, value):
         return self._sync_obj.set(self.status_path, value, timeout=1)
 
+    def _write_failsafe(self, value):
+        return self._sync_obj.set(self.failsafe_path, value, timeout=1)
+
     def _update_leader(self):
-        ret = self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl, prevValue=self._name)
+        ret = self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl,
+                                 handle_raft_error=False, prevValue=self._name)
         if not ret and self._sync_obj.get(self.leader_path) is None:
             ret = self.attempt_to_acquire_leader()
         return ret
 
-    def attempt_to_acquire_leader(self, permanent=False):
-        return self._sync_obj.set(self.leader_path, self._name, prevExist=False,
-                                  ttl=None if permanent else self._ttl)
+    def attempt_to_acquire_leader(self):
+        return self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl, handle_raft_error=False, prevExist=False)
 
     def set_failover_value(self, value, index=None):
         return self._sync_obj.set(self.failover_path, value, prevIndex=index)
@@ -388,9 +424,9 @@ class Raft(AbstractDCS):
     def set_config_value(self, value, index=None):
         return self._sync_obj.set(self.config_path, value, prevIndex=index)
 
-    def touch_member(self, data, permanent=False):
+    def touch_member(self, data):
         data = json.dumps(data, separators=(',', ':'))
-        return self._sync_obj.set(self.member_path, data, None if permanent else self._ttl, timeout=2)
+        return self._sync_obj.set(self.member_path, data, self._ttl, timeout=2)
 
     def take_leader(self):
         return self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl)

patroni/dcs/zookeeper.py

@@ -1,15 +1,17 @@
 import json
 import logging
 import select
+import six
 import time
 
 from kazoo.client import KazooClient, KazooState, KazooRetry
-from kazoo.exceptions import NoNodeError, NodeExistsError, SessionExpiredError
+from kazoo.exceptions import ConnectionClosedError, NoNodeError, NodeExistsError, SessionExpiredError
 from kazoo.handlers.threading import SequentialThreadingHandler
 from kazoo.protocol.states import KeeperState
+from kazoo.retry import RetryFailedError
 from kazoo.security import make_acl
 
-from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
+from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory, citus_group_re
 from ..exceptions import DCSError
 from ..utils import deep_compare
 
@@ -50,11 +52,21 @@ class PatroniSequentialThreadingHandler(SequentialThreadingHandler):
         return super(PatroniSequentialThreadingHandler, self).create_connection(*args, **kwargs)
 
     def select(self, *args, **kwargs):
-        """Python3 raises `ValueError` if socket is closed, because fd == -1"""
+        """
+        Python 3.XY may raise following exceptions if select/poll are called with an invalid socket:
+        - `ValueError`: because fd == -1
+        - `TypeError`: Invalid file descriptor: -1 (starting from kazoo 2.9)
+        Python 2.7 may raise the `IOError` instead of `socket.error` (starting from kazoo 2.9)
+
+        When it is appropriate we map these exceptions to `socket.error`.
+        """
+
         try:
             return super(PatroniSequentialThreadingHandler, self).select(*args, **kwargs)
-        except ValueError as e:
-            raise select.error(9, str(e))
+        except IOError as e:
+            raise (select.error(e.errno, e.strerror) if six.PY2 else e)
+        except (TypeError, ValueError) as e:
+            raise (e if six.PY2 and isinstance(e, TypeError) else select.error(9, str(e)))
 
 
 class PatroniKazooClient(KazooClient):
@@ -137,7 +149,11 @@ class ZooKeeper(AbstractDCS):
 
     def cluster_watcher(self, event):
         self._fetch_cluster = True
-        self.status_watcher(event)
+        if not event or event.state != KazooState.CONNECTED or event.path.startswith(self.client_path('')):
+            self.status_watcher(event)
+
+    def members_watcher(self, event):
+        self._fetch_cluster = True
 
     def reload_config(self, config):
         self.set_retry_timeout(config['retry_timeout'])
@@ -182,10 +198,10 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return None
 
-    def get_status(self, leader):
+    def get_status(self, path, leader):
         watch = self.status_watcher if not leader or leader.name != self._name else None
 
-        status = self.get_node(self.status_path, watch)
+        status = self.get_node(path + self._STATUS, watch)
         if status:
             try:
                 status = json.loads(status[0])
@@ -194,7 +210,7 @@ class ZooKeeper(AbstractDCS):
             except Exception:
                 slots = last_lsn = None
         else:
-            last_lsn = self.get_node(self.leader_optime_path, watch)
+            last_lsn = self.get_node(path + self._LEADER_OPTIME, watch)
             last_lsn = last_lsn and last_lsn[0]
             slots = None
 
@@ -216,69 +232,78 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return []
 
-    def load_members(self):
+    def load_members(self, path):
         members = []
-        for member in self.get_children(self.members_path, self.cluster_watcher):
-            data = self.get_node(self.members_path + member)
+        for member in self.get_children(path + self._MEMBERS, self.cluster_watcher):
+            data = self.get_node(path + self._MEMBERS + member)
             if data is not None:
                 members.append(self.member(member, *data))
         return members
 
-    def _inner_load_cluster(self):
+    def _cluster_loader(self, path):
         self._fetch_cluster = False
         self.event.clear()
-        nodes = set(self.get_children(self.client_path(''), self.cluster_watcher))
+        nodes = set(self.get_children(path, self.cluster_watcher))
         if not nodes:
             self._fetch_cluster = True
 
         # get initialize flag
-        initialize = (self.get_node(self.initialize_path) or [None])[0] if self._INITIALIZE in nodes else None
+        initialize = (self.get_node(path + self._INITIALIZE) or [None])[0] if self._INITIALIZE in nodes else None
 
         # get global dynamic configuration
-        config = self.get_node(self.config_path, watch=self.cluster_watcher) if self._CONFIG in nodes else None
+        config = self.get_node(path + self._CONFIG, watch=self.cluster_watcher) if self._CONFIG in nodes else None
         config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid)
 
         # get timeline history
-        history = self.get_node(self.history_path, watch=self.cluster_watcher) if self._HISTORY in nodes else None
+        history = self.get_node(path + self._HISTORY, watch=self.cluster_watcher) if self._HISTORY in nodes else None
         history = history and TimelineHistory.from_node(history[1].mzxid, history[0])
 
         # get synchronization state
-        sync = self.get_node(self.sync_path, watch=self.cluster_watcher) if self._SYNC in nodes else None
+        sync = self.get_node(path + self._SYNC, watch=self.cluster_watcher) if self._SYNC in nodes else None
         sync = SyncState.from_node(sync and sync[1].version, sync and sync[0])
 
         # get list of members
-        members = self.load_members() if self._MEMBERS[:-1] in nodes else []
+        members = self.load_members(path) if self._MEMBERS[:-1] in nodes else []
 
         # get leader
-        leader = self.get_node(self.leader_path) if self._LEADER in nodes else None
+        leader = self.get_node(path + self._LEADER) if self._LEADER in nodes else None
         if leader:
-            client_id = self._client.client_id
-            if not self._ctl and leader[0] == self._name and client_id is not None \
-                    and client_id[0] != leader[1].ephemeralOwner:
-                logger.info('I am leader but not owner of the session. Removing leader node')
-                self._client.delete(self.leader_path)
-                leader = None
-
-            if leader:
-                member = Member(-1, leader[0], None, {})
-                member = ([m for m in members if m.name == leader[0]] or [member])[0]
-                leader = Leader(leader[1].version, leader[1].ephemeralOwner, member)
-                self._fetch_cluster = member.index == -1
+            member = Member(-1, leader[0], None, {})
+            member = ([m for m in members if m.name == leader[0]] or [member])[0]
+            leader = Leader(leader[1].version, leader[1].ephemeralOwner, member)
+            self._fetch_cluster = member.index == -1
 
         # get last known leader lsn and slots
-        last_lsn, slots = self.get_status(leader)
+        last_lsn, slots = self.get_status(path, leader)
 
         # failover key
-        failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
+        failover = self.get_node(path + self._FAILOVER, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
         failover = failover and Failover.from_node(failover[1].version, failover[0])
 
-        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
+        # get failsafe topology
+        failsafe = self.get_node(path + self._FAILSAFE, watch=self.cluster_watcher) if self._FAILSAFE in nodes else None
+        try:
+            failsafe = json.loads(failsafe[0]) if failsafe else None
+        except Exception:
+            failsafe = None
 
-    def _load_cluster(self):
-        cluster = self.cluster
+        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+
+    def _citus_cluster_loader(self, path):
+        fetch_cluster = False
+        ret = {}
+        for node in self.get_children(path, self.cluster_watcher):
+            if citus_group_re.match(node):
+                ret[int(node)] = self._cluster_loader(path + node + '/')
+                fetch_cluster = fetch_cluster or self._fetch_cluster
+        self._fetch_cluster = fetch_cluster
+        return ret
+
+    def _load_cluster(self, path, loader):
+        cluster = self.cluster if path == self._base_path + '/' else None
         if self._fetch_cluster or cluster is None:
             try:
-                cluster = self._client.retry(self._inner_load_cluster)
+                cluster = self._client.retry(loader, path)
             except Exception:
                 logger.exception('get_cluster')
                 self.cluster_watcher(None)
@@ -291,10 +316,12 @@ class ZooKeeper(AbstractDCS):
                 self.event.clear()
             else:
                 try:
-                    last_lsn, slots = self.get_status(cluster.leader)
+                    last_lsn, slots = self.get_status(self.client_path(''), cluster.leader)
                     self.event.clear()
-                    cluster = Cluster(cluster.initialize, cluster.config, cluster.leader, last_lsn,
-                                      cluster.members, cluster.failover, cluster.sync, cluster.history, slots)
+                    cluster = list(cluster)
+                    cluster[3] = last_lsn
+                    cluster[8] = slots
+                    cluster = Cluster(*cluster)
                 except Exception:
                     pass
         return cluster
@@ -313,11 +340,18 @@ class ZooKeeper(AbstractDCS):
             logger.exception('Failed to create %s', path)
         return False
 
-    def attempt_to_acquire_leader(self, permanent=False):
-        ret = self._create(self.leader_path, self._name.encode('utf-8'), retry=True, ephemeral=not permanent)
-        if not ret:
-            logger.info('Could not take out TTL lock')
-        return ret
+    def attempt_to_acquire_leader(self):
+        try:
+            self._client.retry(self._client.create, self.leader_path, self._name.encode('utf-8'),
+                               makepath=True, ephemeral=True)
+            return True
+        except (ConnectionClosedError, RetryFailedError) as e:
+            raise ZooKeeperError(e)
+        except Exception as e:
+            if not isinstance(e, NodeExistsError):
+                logger.error('Failed to create %s: %r', self.leader_path, e)
+        logger.info('Could not take out TTL lock')
+        return False
 
     def _set_or_create(self, key, value, index=None, retry=False, do_not_create_empty=False):
         value = value.encode('utf-8')
@@ -349,12 +383,15 @@ class ZooKeeper(AbstractDCS):
         return self._create(self.initialize_path, sysid, retry=True) if create_new \
             else self._client.retry(self._client.set, self.initialize_path, sysid)
 
-    def touch_member(self, data, permanent=False):
+    def touch_member(self, data):
         cluster = self.cluster
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
         member_data = self.__last_member_data or member and member.data
+        #  We want to notify leader if some important fields in the member key changed by removing ZNode
         if member and (self._client.client_id is not None and member.session != self._client.client_id[0] or
                        not (deep_compare(member_data.get('tags', {}), data.get('tags', {})) and
+                            (member_data.get('state') == data.get('state') or
+                                'running' not in (member_data.get('state'), data.get('state'))) and
                             member_data.get('version') == data.get('version') and
                             member_data.get('checkpoint_after_promote') == data.get('checkpoint_after_promote'))):
             try:
@@ -371,8 +408,7 @@ class ZooKeeper(AbstractDCS):
                 return True
         else:
             try:
-                self._client.create_async(self.member_path, encoded_data, makepath=True,
-                                          ephemeral=not permanent).get(timeout=1)
+                self._client.create_async(self.member_path, encoded_data, makepath=True, ephemeral=True).get(timeout=1)
                 self.__last_member_data = data
                 return True
             except Exception as e:
@@ -397,7 +433,32 @@ class ZooKeeper(AbstractDCS):
     def _write_status(self, value):
         return self._set_or_create(self.status_path, value)
 
+    def _write_failsafe(self, value):
+        return self._set_or_create(self.failsafe_path, value)
+
     def _update_leader(self):
+        cluster = self.cluster
+        session = cluster and isinstance(cluster.leader, Leader) and cluster.leader.session
+        if self._client.client_id and self._client.client_id[0] != session:
+            logger.warning('Recreating the leader ZNode due to ownership mismatch')
+            try:
+                self._client.retry(self._client.delete, self.leader_path)
+            except NoNodeError:
+                pass
+            except (ConnectionClosedError, RetryFailedError) as e:
+                raise ZooKeeperError(e)
+            except Exception as e:
+                logger.error('Failed to remove %s: %r', self.leader_path, e)
+                return False
+
+            try:
+                self._client.retry(self._client.create, self.leader_path,
+                                   self._name.encode('utf-8'), makepath=True, ephemeral=True)
+            except (ConnectionClosedError, RetryFailedError) as e:
+                raise ZooKeeperError(e)
+            except Exception as e:
+                logger.error('Failed to create %s: %r', self.leader_path, e)
+                return False
         return True
 
     def _delete_leader(self):

patroni/ha.py

@@ -18,7 +18,7 @@ from .postgresql import ACTION_ON_START, ACTION_ON_ROLE_CHANGE
 from .postgresql.misc import postgres_version_to_int
 from .postgresql.rewind import Rewind
 from .utils import polling_loop, tzutc, is_standby_cluster as _is_standby_cluster, parse_int
-from .dcs import RemoteMember
+from .dcs import Cluster, Leader, RemoteMember
 
 logger = logging.getLogger(__name__)
 
@@ -39,11 +39,20 @@ class _MemberStatus(namedtuple('_MemberStatus', ['member', 'reachable', 'in_reco
     """
     @classmethod
     def from_api_response(cls, member, json):
-        is_master = json['role'] == 'master'
+        """
+        :param member: dcs.Member object
+        :param json: RestApiHandler.get_postgresql_status() result
+        :returns: _MemberStatus object
+        """
+        # If one of those is not in a response we want to count the node as not healthy/reachable
+        assert 'wal' in json or 'xlog' in json
+
+        wal = json.get('wal', json.get('xlog'))
+        in_recovery = not bool(wal.get('location'))  # abuse difference in primary/replica response format
         timeline = json.get('timeline', 0)
         dcs_last_seen = json.get('dcs_last_seen', 0)
-        wal = not is_master and max(json['xlog'].get('received_location', 0), json['xlog'].get('replayed_location', 0))
-        return cls(member, True, not is_master, dcs_last_seen, timeline, wal,
+        wal = in_recovery and max(wal.get('received_location', 0), wal.get('replayed_location', 0))
+        return cls(member, True, in_recovery, dcs_last_seen, timeline, wal,
                    json.get('tags', {}), json.get('watchdog_failed', False))
 
     @classmethod
@@ -61,6 +70,64 @@ class _MemberStatus(namedtuple('_MemberStatus', ['member', 'reachable', 'in_reco
         return None
 
 
+class Failsafe(object):
+
+    def __init__(self, dcs):
+        self._lock = RLock()
+        self._dcs = dcs
+        self._last_update = 0
+        self._name = None
+        self._conn_url = None
+        self._api_url = None
+        self._slots = None
+
+    def update(self, data):
+        with self._lock:
+            self._last_update = time.time()
+            self._name = data['name']
+            self._conn_url = data['conn_url']
+            self._api_url = data['api_url']
+            self._slots = data.get('slots')
+
+    @property
+    def leader(self):
+        with self._lock:
+            if self._last_update + self._dcs.ttl > time.time():
+                return Leader(None, None,
+                              RemoteMember(self._name, {'api_url': self._api_url,
+                                                        'conn_url': self._conn_url,
+                                                        'slots': self._slots}))
+
+    def update_cluster(self, cluster):
+        # Enreach cluster with the real leader if there was a ping from it
+        leader = self.leader
+        if leader:
+            cluster = list(cluster)
+            # We rely on the strict order of fields in the namedtuple
+            cluster[2] = leader
+            cluster[8] = leader.member.data['slots']
+            cluster = Cluster(*cluster)
+        return cluster
+
+    def is_active(self):
+        """Is used to report in REST API whether the failsafe mode was activated.
+
+           On primary the self._last_update is set from the
+           set_is_active() method and always returns the correct value.
+
+           On replicas the self._last_update is set at the moment when
+           the primary performs POST /failsafe REST API calls.
+           The side-effect - it is possible that replicas will show
+           failsafe_is_active values different from the primary."""
+
+        with self._lock:
+            return self._last_update + self._dcs.ttl > time.time()
+
+    def set_is_active(self, value):
+        with self._lock:
+            self._last_update = value
+
+
 class Ha(object):
 
     def __init__(self, patroni):
@@ -72,6 +139,7 @@ class Ha(object):
         self.old_cluster = None
         self._is_leader = False
         self._is_leader_lock = RLock()
+        self._failsafe = Failsafe(patroni.dcs)
         self._was_paused = False
         self._leader_timeline = None
         self.recovering = False
@@ -88,6 +156,8 @@ class Ha(object):
         # Count of concurrent sync disabling requests. Value above zero means that we don't want to be synchronous
         # standby. Changes protected by _member_state_lock.
         self._disable_sync = 0
+        # Remember the last known member role and state written to the DCS in order to notify Citus coordinator
+        self._last_state = None
 
         # We need following property to avoid shutdown of postgres when join of Patroni to the postgres
         # already running as replica was aborted due to cluster not being initialized in DCS.
@@ -103,9 +173,9 @@ class Ha(object):
         else:
             return self.patroni.config.check_mode(mode)
 
-    def master_stop_timeout(self):
-        """ Master stop timeout """
-        ret = parse_int(self.patroni.config['master_stop_timeout'])
+    def primary_stop_timeout(self):
+        """ Primary stop timeout """
+        ret = parse_int(self.patroni.config['primary_stop_timeout'])
         return ret if ret and ret > 0 and self.is_synchronous_mode() else None
 
     def is_paused(self):
@@ -140,16 +210,33 @@ class Ha(object):
             self.old_cluster = cluster
         self.cluster = cluster
 
+        if self.cluster.is_unlocked() and self.is_failsafe_mode():
+            # If failsafe mode is enabled we want to inject the "real" leader to the cluster
+            self.cluster = cluster = self._failsafe.update_cluster(cluster)
+
         if not self.has_lock(False):
             self.set_is_leader(False)
 
         self._leader_timeline = None if cluster.is_unlocked() else cluster.leader.timeline
 
     def acquire_lock(self):
-        ret = self.dcs.attempt_to_acquire_leader()
+        try:
+            ret = self.dcs.attempt_to_acquire_leader()
+        except DCSError:
+            raise
+        except Exception:
+            logger.exception('Unexpected exception raised from attempt_to_acquire_leader, please report it as a BUG')
+            ret = False
         self.set_is_leader(ret)
         return ret
 
+    def _failsafe_config(self):
+        if self.is_failsafe_mode():
+            ret = {m.name: m.api_url for m in self.cluster.members}
+            if self.state_handler.name not in ret:
+                ret[self.state_handler.name] = self.patroni.api.connection_string
+            return ret
+
     def update_lock(self, write_leader_optime=False):
         last_lsn = slots = None
         if write_leader_optime:
@@ -159,7 +246,9 @@ class Ha(object):
             except Exception:
                 logger.exception('Exception when called state_handler.last_operation()')
         try:
-            ret = self.dcs.update_leader(last_lsn, slots)
+            ret = self.dcs.update_leader(last_lsn, slots, self._failsafe_config())
+        except DCSError:
+            raise
         except Exception:
             logger.exception('Unexpected exception raised from update_leader, please report it as a BUG')
             ret = False
@@ -182,6 +271,22 @@ class Ha(object):
             tags['nosync'] = True
         return tags
 
+    def notify_citus_coordinator(self, event):
+        if self.state_handler.citus_handler.is_worker():
+            coordinator = self.dcs.get_citus_coordinator()
+            if coordinator and coordinator.leader and coordinator.leader.conn_kwargs:
+                try:
+                    data = {'type': event,
+                            'group': self.state_handler.citus_handler.group(),
+                            'leader': self.state_handler.name,
+                            'timeout': self.dcs.ttl,
+                            'cooldown': self.patroni.config['retry_timeout']}
+                    timeout = self.dcs.ttl if event == 'before_demote' else 2
+                    self.patroni.request(coordinator.leader.member, 'post', 'citus', data, timeout=timeout, retries=0)
+                except Exception as e:
+                    logger.warning('Request to Citus coordinator leader %s %s failed: %r',
+                                   coordinator.leader.name, coordinator.leader.member.api_url, e)
+
     def touch_member(self):
         with self._member_state_lock:
             data = {
@@ -192,6 +297,10 @@ class Ha(object):
                 'version': self.patroni.version
             }
 
+            proxy_url = self.state_handler.proxy_url
+            if proxy_url:
+                data['proxy_url'] = proxy_url
+
             if self.is_leader() and not self._rewind.checkpoint_after_promote():
                 data['checkpoint_after_promote'] = False
             tags = self.get_effective_tags()
@@ -230,7 +339,13 @@ class Ha(object):
             if self.is_paused():
                 data['pause'] = True
 
-            return self.dcs.touch_member(data)
+            ret = self.dcs.touch_member(data)
+            if ret:
+                new_state = (data['state'], {'master': 'primary'}.get(data['role'], data['role']))
+                if self._last_state != new_state and new_state == ('running', 'primary'):
+                    self.notify_citus_coordinator('after_promote')
+                self._last_state = new_state
+            return ret
 
     def clone(self, clone_member=None, msg='(without leader)'):
         if self.is_standby_cluster() and not isinstance(clone_member, RemoteMember):
@@ -254,7 +369,7 @@ class Ha(object):
             ret = self._async_executor.try_run_async('bootstrap {0}'.format(msg), self.clone, args=(clone_member, msg))
             return ret or 'trying to bootstrap {0}'.format(msg)
 
-        # no initialize key and node is allowed to be master and has 'bootstrap' section in a configuration file
+        # no initialize key and node is allowed to be primary and has 'bootstrap' section in a configuration file
         elif self.cluster.initialize is None and not self.patroni.nofailover and 'bootstrap' in self.patroni.config:
             if self.dcs.initialize(create_new=True):  # race for initialization
                 self.state_handler.bootstrapping = True
@@ -273,18 +388,20 @@ class Ha(object):
         else:
             create_replica_methods = self.get_standby_cluster_config().get('create_replica_methods', []) \
                                      if self.is_standby_cluster() else None
-            if self.state_handler.can_create_replica_without_replication_connection(create_replica_methods):
+            can_bootstrap = self.state_handler.can_create_replica_without_replication_connection(create_replica_methods)
+            concurrent_bootstrap = self.cluster.initialize == ""
+            if can_bootstrap and not concurrent_bootstrap:
                 msg = 'bootstrap (without leader)'
                 return self._async_executor.try_run_async(msg, self.clone) or 'trying to ' + msg
             return 'waiting for {0}leader to bootstrap'.format('standby_' if self.is_standby_cluster() else '')
 
     def bootstrap_standby_leader(self):
         """ If we found 'standby' key in the configuration, we need to bootstrap
-            not a real master, but a 'standby leader', that will take base backup
-            from a remote master and start follow it.
+            not a real primary, but a 'standby leader', that will take base backup
+            from a remote member and start follow it.
         """
-        clone_source = self.get_remote_master()
-        msg = 'clone from remote master {0}'.format(clone_source.conn_url)
+        clone_source = self.get_remote_member()
+        msg = 'clone from remote member {0}'.format(clone_source.conn_url)
         result = self.clone(clone_source, msg)
         with self._async_response:  # pretend that post_bootstrap was already executed
             self._async_response.complete(result)
@@ -301,7 +418,7 @@ class Ha(object):
             return self._async_executor.try_run_async(msg, self._rewind.ensure_clean_shutdown) or msg
 
     def _handle_rewind_or_reinitialize(self):
-        leader = self.get_remote_master() if self.is_standby_cluster() else self.cluster.leader
+        leader = self.get_remote_member() if self.is_standby_cluster() else self.cluster.leader
         if not self._rewind.rewind_or_reinitialize_needed_and_possible(leader):
             return None
 
@@ -325,12 +442,12 @@ class Ha(object):
         self.watchdog.disable()
 
         if self.has_lock() and self.update_lock():
-            timeout = self.patroni.config['master_start_timeout']
+            timeout = self.patroni.config['primary_start_timeout']
             if timeout == 0:
-                # We are requested to prefer failing over to restarting master. But see first if there
+                # We are requested to prefer failing over to restarting primary. But see first if there
                 # is anyone to fail over to.
                 if self.is_failover_possible(self.cluster.members):
-                    logger.info("Master crashed. Failing over.")
+                    logger.info("Primary crashed. Failing over.")
                     self.demote('immediate')
                     return 'stopped PostgreSQL to fail over after a crash'
         else:
@@ -358,11 +475,14 @@ class Ha(object):
                 role = 'standby_leader'
                 node_to_follow = self._get_node_to_follow(self.cluster)
             elif self.is_standby_cluster() and self.cluster.is_unlocked():
-                msg = "trying to follow a remote master because standby cluster is unhealthy"
-                node_to_follow = self.get_remote_master()
+                msg = "trying to follow a remote member because standby cluster is unhealthy"
+                node_to_follow = self.get_remote_member()
             else:
                 msg = "starting as a secondary"
                 node_to_follow = self._get_node_to_follow(self.cluster)
+
+            if self.is_synchronous_mode():
+                self.state_handler.sync_handler.set_synchronous_standby_names([])
         elif self.has_lock():
             msg = "starting as readonly because i had the session lock"
             node_to_follow = None
@@ -378,7 +498,7 @@ class Ha(object):
         standby_config = self.get_standby_cluster_config()
         is_standby_cluster = _is_standby_cluster(standby_config)
         if is_standby_cluster and (self.cluster.is_unlocked() or self.has_lock(False)):
-            node_to_follow = self.get_remote_master()
+            node_to_follow = self.get_remote_member()
         elif self.patroni.replicatefrom and self.patroni.replicatefrom != self.state_handler.name:
             node_to_follow = cluster.get_member(self.patroni.replicatefrom)
         else:
@@ -409,7 +529,7 @@ class Ha(object):
                     or self.cluster.is_unlocked():
                 if is_leader:
                     self.state_handler.set_role('master')
-                    return 'continue to run as master without lock'
+                    return 'continue to run as primary without lock'
                 elif self.state_handler.role != 'standby_leader':
                     self.state_handler.set_role('replica')
 
@@ -457,11 +577,14 @@ class Ha(object):
     def is_synchronous_mode_strict(self):
         return self.check_mode('synchronous_mode_strict')
 
+    def is_failsafe_mode(self):
+        return self.check_mode('failsafe_mode')
+
     def process_sync_replication(self):
         """Process synchronous standby beahvior.
 
         Synchronous standbys are registered in two places postgresql.conf and DCS. The order of updating them must
-        be right. The invariant that should be kept is that if a node is master and sync_standby is set in DCS,
+        be right. The invariant that should be kept is that if a node is primary and sync_standby is set in DCS,
         then that node must have synchronous_standby set to that value. Or more simple, first set in postgresql.conf
         and then in DCS. When removing, first remove in DCS, then in postgresql.conf. This is so we only consider
         promoting standbys that were guaranteed to be replicating synchronously.
@@ -469,9 +592,9 @@ class Ha(object):
         if self.is_synchronous_mode():
             sync_node_count = self.patroni.config['synchronous_node_count']
             current = self.cluster.sync.leader and self.cluster.sync.members or []
-            picked, allow_promote = self.state_handler.pick_synchronous_standby(self.cluster, sync_node_count,
-                                                                                self.patroni.config[
-                                                                                    'maximum_lag_on_syncnode'])
+            picked, allow_promote = self.state_handler.sync_handler.current_state(self.cluster, sync_node_count,
+                                                                                  self.patroni.config[
+                                                                                      'maximum_lag_on_syncnode'])
             if set(picked) != set(current):
                 # update synchronous standby list in dcs temporarily to point to common nodes in current and picked
                 sync_common = list(set(current).intersection(set(allow_promote)))
@@ -489,15 +612,15 @@ class Ha(object):
                     logger.warning("No standbys available!")
 
                 logger.info("Assigning synchronous standby status to %s", picked)
-                self.state_handler.config.set_synchronous_standby(picked)
+                self.state_handler.sync_handler.set_synchronous_standby_names(picked)
 
                 if picked and picked[0] != '*' and set(allow_promote) != set(picked) and not allow_promote:
                     # Wait for PostgreSQL to enable synchronous mode and see if we can immediately set sync_standby
                     time.sleep(2)
-                    _, allow_promote = self.state_handler.pick_synchronous_standby(self.cluster,
-                                                                                   sync_node_count,
-                                                                                   self.patroni.config[
-                                                                                       'maximum_lag_on_syncnode'])
+                    _, allow_promote = self.state_handler.sync_handler.current_state(self.cluster,
+                                                                                     sync_node_count,
+                                                                                     self.patroni.config[
+                                                                                         'maximum_lag_on_syncnode'])
                 if allow_promote and set(allow_promote) != set(sync_common):
                     try:
                         cluster = self.dcs.get_cluster()
@@ -513,7 +636,7 @@ class Ha(object):
         else:
             if self.cluster.sync.leader and self.dcs.delete_sync_state(index=self.cluster.sync.index):
                 logger.info("Disabled synchronous replication")
-            self.state_handler.config.set_synchronous_standby([])
+            self.state_handler.sync_handler.set_synchronous_standby_names([])
 
     def is_sync_standby(self, cluster):
         return cluster.leader and cluster.sync.leader == cluster.leader.name \
@@ -527,7 +650,7 @@ class Ha(object):
 
         If the connection to DCS fails we run the action anyway, as this is only a hint.
 
-        There is a small race window where this function runs between a master picking us the sync standby and
+        There is a small race window where this function runs between a primary picking us the sync standby and
         publishing it to the DCS. As the window is rather tiny consequences are holding up commits for one cycle
         period we don't worry about it here."""
 
@@ -538,7 +661,7 @@ class Ha(object):
             self._disable_sync += 1
         try:
             if self.touch_member():
-                # Master should notice the updated value during the next cycle. We will wait double that, if master
+                # Primary should notice the updated value during the next cycle. We will wait double that, if primary
                 # hasn't noticed the value by then not disabling sync replication is not likely to matter.
                 for _ in polling_loop(timeout=self.dcs.loop_wait*2, interval=2):
                     try:
@@ -547,7 +670,7 @@ class Ha(object):
                     except DCSError:
                         logger.warning("Could not get cluster state, skipping synchronous standby disable")
                         break
-                    logger.info("Waiting for master to release us from synchronous standby")
+                    logger.info("Waiting for primary to release us from synchronous standby")
             else:
                 logger.warning("Updating member state failed, skipping synchronous standby disable")
 
@@ -557,14 +680,14 @@ class Ha(object):
                 self._disable_sync -= 1
 
     def update_cluster_history(self):
-        master_timeline = self.state_handler.get_master_timeline()
+        primary_timeline = self.state_handler.get_primary_timeline()
         cluster_history = self.cluster.history and self.cluster.history.lines
-        if master_timeline == 1:
+        if primary_timeline == 1:
             if cluster_history:
                 self.dcs.set_history_value('[]')
-        elif not cluster_history or cluster_history[-1][0] != master_timeline - 1 or len(cluster_history[-1]) != 5:
+        elif not cluster_history or cluster_history[-1][0] != primary_timeline - 1 or len(cluster_history[-1]) != 5:
             cluster_history = {line[0]: line for line in cluster_history or []}
-            history = self.state_handler.get_history(master_timeline)
+            history = self.state_handler.get_history(primary_timeline)
             if history and self.cluster.config:
                 history = history[-self.cluster.config.max_timelines_history:]
                 for line in history:
@@ -577,14 +700,14 @@ class Ha(object):
                             line.append(cluster_history[line[0]][4])
                 self.dcs.set_history_value(json.dumps(history, separators=(',', ':')))
 
-    def enforce_follow_remote_master(self, message):
-        demote_reason = 'cannot be a real master in standby cluster'
+    def enforce_follow_remote_member(self, message):
+        demote_reason = 'cannot be a real primary in standby cluster'
         return self.follow(demote_reason, message)
 
-    def enforce_master_role(self, message, promote_message):
+    def enforce_primary_role(self, message, promote_message):
         """
         Ensure the node that has won the race for the leader key meets criteria
-        for promoting its PG server to the 'master' role.
+        for promoting its PG server to the 'primary' role.
         """
         if not self.is_paused():
             if not self.watchdog.is_running and not self.watchdog.activate():
@@ -605,13 +728,14 @@ class Ha(object):
                     return 'Promotion cancelled because the pre-promote script failed'
 
         if self.state_handler.is_leader():
-            # Inform the state handler about its master role.
+            # Inform the state handler about its primary role.
             # It may be unaware of it if postgres is promoted manually.
             self.state_handler.set_role('master')
             self.process_sync_replication()
             self.update_cluster_history()
+            self.state_handler.citus_handler.sync_pg_dist_node(self.cluster)
             return message
-        elif self.state_handler.role == 'master':
+        elif self.state_handler.role in ('master', 'promoted', 'primary'):
             self.process_sync_replication()
             return message
         else:
@@ -622,16 +746,21 @@ class Ha(object):
                     # Somebody else updated sync state, it may be due to us losing the lock. To be safe, postpone
                     # promotion until next cycle. TODO: trigger immediate retry of run_cycle
                     return 'Postponing promotion because synchronous replication state was updated by somebody else'
-                self.state_handler.config.set_synchronous_standby(['*'] if self.is_synchronous_mode_strict() else [])
-            if self.state_handler.role != 'master':
+                self.state_handler.sync_handler.set_synchronous_standby_names(
+                    ['*'] if self.is_synchronous_mode_strict() else [])
+            if self.state_handler.role not in ('master', 'promoted', 'primary'):
                 def on_success():
                     self._rewind.reset_state()
                     logger.info("cleared rewind state after becoming the leader")
 
+                def before_promote():
+                    self.notify_citus_coordinator('before_promote')
+
                 with self._async_response:
                     self._async_response.reset()
                 self._async_executor.try_run_async('promote', self.state_handler.promote,
-                                                   args=(self.dcs.loop_wait, self._async_response, on_success))
+                                                   args=(self.dcs.loop_wait, self._async_response,
+                                                         before_promote, on_success))
             return promote_message
 
     def fetch_node_status(self, member):
@@ -655,6 +784,49 @@ class Ha(object):
         pool.join()
         return results
 
+    def update_failsafe(self, data):
+        if self.state_handler.state == 'running' and self.state_handler.role in ('master', 'primary'):
+            return 'Running as a leader'
+        self._failsafe.update(data)
+
+    def failsafe_is_active(self):
+        return self._failsafe.is_active()
+
+    def call_failsafe_member(self, data, member):
+        try:
+            response = self.patroni.request(member, 'post', 'failsafe', data, timeout=2, retries=1)
+            data = response.data.decode('utf-8')
+            logger.info('Got response from %s %s: %s', member.name, member.api_url, data)
+            return response.status == 200 and data == 'Accepted'
+        except Exception as e:
+            logger.warning("Request failed to %s: POST %s (%s)", member.name, member.api_url, e)
+        return False
+
+    def check_failsafe_topology(self):
+        failsafe = self.dcs.failsafe
+        if not isinstance(failsafe, dict) or self.state_handler.name not in failsafe:
+            return False
+        data = {
+            'name': self.state_handler.name,
+            'conn_url': self.state_handler.connection_string,
+            'api_url': self.patroni.api.connection_string,
+        }
+        try:
+            data['slots'] = self.state_handler.slots()
+        except Exception:
+            logger.exception('Exception when called state_handler.slots()')
+        members = [RemoteMember(name, {'api_url': url})
+                   for name, url in failsafe.items()
+                   if name != self.state_handler.name]
+        if not members:  # A sinlge node cluster
+            return True
+        pool = ThreadPool(len(members))
+        call_failsafe_member = functools.partial(self.call_failsafe_member, data)
+        results = pool.map(call_failsafe_member, members)
+        pool.close()
+        pool.join()
+        return all(results)
+
     def is_lagging(self, wal_position):
         """Returns if instance with an wal should consider itself unhealthy to be promoted due to replication lag.
 
@@ -670,7 +842,7 @@ class Ha(object):
         my_wal_position = self.state_handler.last_operation()
         if check_replication_lag and self.is_lagging(my_wal_position):
             logger.info('My wal position exceeds maximum replication lag')
-            return False  # Too far behind last reported wal position on master
+            return False  # Too far behind last reported wal position on primary
 
         if not self.is_standby_cluster() and self.check_timeline():
             cluster_timeline = self.cluster.timeline
@@ -686,7 +858,7 @@ class Ha(object):
             for st in self.fetch_nodes_statuses(members):
                 if st.failover_limitation() is None:
                     if not st.in_recovery:
-                        logger.warning('Master (%s) is still alive', st.member.name)
+                        logger.warning('Primary (%s) is still alive', st.member.name)
                         return False
                     if my_wal_position < st.wal_position:
                         logger.info('Wal position of %s is ahead of my wal position', st.member.name)
@@ -729,7 +901,7 @@ class Ha(object):
                 return True
             elif self.is_paused():
                 # Remove failover key if the node to failover has terminated to avoid waiting for it indefinitely
-                # In order to avoid attempts to delete this key from all nodes only the master is allowed to do it.
+                # In order to avoid attempts to delete this key from all nodes only the primary is allowed to do it.
                 if (not self.cluster.get_member(failover.candidate, fallback_to_leader=False) and
                    self.state_handler.is_leader()):
                     logger.warning("manual failover: removing failover key because failover candidate is not running")
@@ -737,6 +909,11 @@ class Ha(object):
                     return None
                 return False
 
+            # in synchronous mode when our name is not in the /sync key
+            # we shouldn't take any action even if the candidate is unhealthy
+            if self.is_synchronous_mode() and not self.cluster.sync.matches(self.state_handler.name):
+                return False
+
             # find specific node and check that it is healthy
             member = self.cluster.get_member(failover.candidate, fallback_to_leader=False)
             if member:
@@ -777,7 +954,7 @@ class Ha(object):
         if self.is_paused() and not self.patroni.nofailover and \
                 self.cluster.failover and not self.cluster.failover.scheduled_at:
             ret = self.manual_failover_process_no_leader()
-            if ret is not None:  # continue if we just deleted the stale failover key as a master
+            if ret is not None:  # continue if we just deleted the stale failover key as a leader
                 return ret
 
         if self.state_handler.is_starting():  # postgresql still starting up is unhealthy
@@ -797,7 +974,7 @@ class Ha(object):
         if self.cluster.failover:
             # When doing a switchover in synchronous mode only synchronous nodes and former leader are allowed to race
             if self.is_synchronous_mode() and self.cluster.failover.leader and \
-                    self.cluster.failover.candidate and not self.cluster.sync.matches(self.state_handler.name):
+                    not self.cluster.sync.matches(self.state_handler.name):
                 return False
             return self.manual_failover_process_no_leader()
 
@@ -805,8 +982,19 @@ class Ha(object):
             logger.warning('Watchdog device is not usable')
             return False
 
-        # When in sync mode, only last known master and sync standby are allowed to promote automatically.
-        all_known_members = self.cluster.members + self.old_cluster.members
+        all_known_members = self.old_cluster.members
+        if self.is_failsafe_mode():
+            failsafe_members = self.dcs.failsafe
+            # We want to discard failsafe_mode if the /failsafe key contains garbage or empty.
+            if isinstance(failsafe_members, dict):
+                # If current node is missing in the /failsafe key we immediately disqualify it from the race.
+                if failsafe_members and self.state_handler.name not in failsafe_members:
+                    return False
+                # Race among not only existing cluster members, but also all known members from the failsafe config
+                all_known_members += [RemoteMember(name, {'api_url': url}) for name, url in failsafe_members.items()]
+        all_known_members += self.cluster.members
+
+        # When in sync mode, only last known primary and sync standby are allowed to promote automatically.
         if self.is_synchronous_mode() and self.cluster.sync and self.cluster.sync.leader:
             if not self.cluster.sync.matches(self.state_handler.name):
                 return False
@@ -829,14 +1017,14 @@ class Ha(object):
         logger.info("Leader key released")
 
     def demote(self, mode):
-        """Demote PostgreSQL running as master.
+        """Demote PostgreSQL running as primary.
 
         :param mode: One of offline, graceful or immediate.
             offline is used when connection to DCS is not available.
             graceful is used when failing over to another node due to user request. May only be called running async.
-            immediate is used when we determine that we are not suitable for master and want to failover quickly
+            immediate is used when we determine that we are not suitable for primary and want to failover quickly
                 without regard for data durability. May only be called synchronously.
-            immediate-nolock is used when find out that we have lost the lock to be master. Need to bring down
+            immediate-nolock is used when find out that we have lost the lock to be primary. Need to bring down
                 PostgreSQL as quickly as possible without regard for data durability. May only be called synchronously.
         """
         mode_control = {
@@ -863,10 +1051,17 @@ class Ha(object):
                     self.release_leader_key_voluntarily(checkpoint_location)
                     status['released'] = True
 
+        def before_shutdown():
+            if self.state_handler.citus_handler.is_coordinator():
+                self.state_handler.citus_handler.on_demote()
+            else:
+                self.notify_citus_coordinator('before_demote')
+
         self.state_handler.stop(mode_control['stop'], checkpoint=mode_control['checkpoint'],
                                 on_safepoint=self.watchdog.disable if self.watchdog.is_running else None,
                                 on_shutdown=on_shutdown if mode_control['release'] else None,
-                                stop_timeout=self.master_stop_timeout())
+                                before_shutdown=before_shutdown if mode == 'graceful' else None,
+                                stop_timeout=self.primary_stop_timeout())
         self.state_handler.set_role('demoted')
         self.set_is_leader(False)
 
@@ -885,14 +1080,15 @@ class Ha(object):
             except Exception:
                 node_to_follow, leader = None, None
 
+        if self.is_synchronous_mode():
+            self.state_handler.sync_handler.set_synchronous_standby_names([])
+
         # FIXME: with mode offline called from DCS exception handler and handle_long_action_in_progress
         # there could be an async action already running, calling follow from here will lead
         # to racy state handler state updates.
         if mode_control['async_req']:
             self._async_executor.try_run_async('starting after demotion', self.state_handler.follow, (node_to_follow,))
         else:
-            if self.is_synchronous_mode():
-                self.state_handler.config.set_synchronous_standby([])
             if self._rewind.rewind_or_reinitialize_needed_and_possible(leader):
                 return False  # do not start postgres, but run pg_rewind on the next iteration
             self.state_handler.follow(node_to_follow)
@@ -993,11 +1189,11 @@ class Ha(object):
                 if self.is_standby_cluster():
                     # standby leader disappeared, and this is the healthiest
                     # replica, so it should become a new standby leader.
-                    # This implies we need to start following a remote master
+                    # This implies we need to start following a remote member
                     msg = 'promoted self to a standby leader by acquiring session lock'
-                    return self.enforce_follow_remote_master(msg)
+                    return self.enforce_follow_remote_member(msg)
                 else:
-                    return self.enforce_master_role(
+                    return self.enforce_primary_role(
                         'acquired session lock as a leader',
                         'promoted self to leader by acquiring session lock'
                     )
@@ -1012,7 +1208,7 @@ class Ha(object):
                 time.sleep(2)  # Give a time to somebody to take the leader lock
 
             if self.patroni.nofailover:
-                return self.follow('demoting self because I am not allowed to become master',
+                return self.follow('demoting self because I am not allowed to become primary',
                                    'following a different leader because I am not allowed to promote')
             return self.follow('demoting self because i am not the healthiest node',
                                'following a different leader because i am not the healthiest node')
@@ -1021,11 +1217,11 @@ class Ha(object):
         if self.has_lock():
             if self.is_paused() and not self.state_handler.is_leader():
                 if self.cluster.failover and self.cluster.failover.candidate == self.state_handler.name:
-                    return 'waiting to become master after promote...'
+                    return 'waiting to become primary after promote...'
 
                 if not self.is_standby_cluster():
                     self._delete_leader()
-                    return 'removed leader lock because postgres is not running as master'
+                    return 'removed leader lock because postgres is not running as primary'
 
             if self.update_lock(True):
                 msg = self.process_manual_failover_from_leader()
@@ -1037,14 +1233,14 @@ class Ha(object):
 
                 if self.is_standby_cluster():
                     # in case of standby cluster we don't really need to
-                    # enforce anything, since the leader is not a master.
+                    # enforce anything, since the leader is not a primary
                     # So just remind the role.
                     msg = 'no action. I am ({0}), the standby leader with the lock'.format(self.state_handler.name) \
                           if self.state_handler.role == 'standby_leader' else \
                           'promoted self to a standby leader because i had the session lock'
-                    return self.enforce_follow_remote_master(msg)
+                    return self.enforce_follow_remote_member(msg)
                 else:
-                    return self.enforce_master_role(
+                    return self.enforce_primary_role(
                         'no action. I am ({0}), the leader with the lock'.format(self.state_handler.name),
                         'promoted self to leader because I had the session lock'
                     )
@@ -1053,7 +1249,7 @@ class Ha(object):
                 logger.error('failed to update leader lock')
                 if self.state_handler.is_leader():
                     if self.is_paused():
-                        return 'continue to run as master after failing to update leader lock in DCS'
+                        return 'continue to run as primary after failing to update leader lock in DCS'
                     self.demote('immediate-nolock')
                     return 'demoted self because failed to update leader lock in DCS'
                 else:
@@ -1160,11 +1356,19 @@ class Ha(object):
 
         # Now that restart is scheduled we can set timeout for startup, it will get reset
         # once async executor runs and main loop notices PostgreSQL as up.
-        timeout = restart_data.get('timeout', self.patroni.config['master_start_timeout'])
+        timeout = restart_data.get('timeout', self.patroni.config['primary_start_timeout'])
         self.set_start_timeout(timeout)
 
+        def before_shutdown():
+            self.notify_citus_coordinator('before_demote')
+
+        def after_start():
+            self.notify_citus_coordinator('after_promote')
+
         # For non async cases we want to wait for restart to complete or timeout before returning.
-        do_restart = functools.partial(self.state_handler.restart, timeout, self._async_executor.critical_task)
+        do_restart = functools.partial(self.state_handler.restart, timeout, self._async_executor.critical_task,
+                                       before_shutdown=before_shutdown if self.has_lock() else None,
+                                       after_start=after_start if self.has_lock() else None)
         if self.is_synchronous_mode() and not self.has_lock():
             do_restart = functools.partial(self.while_not_sync_standby, do_restart)
 
@@ -1215,7 +1419,7 @@ class Ha(object):
         """
         if self.has_lock() and self.update_lock():
             if self._async_executor.scheduled_action == 'doing crash recovery in a single user mode':
-                time_left = self.patroni.config['master_start_timeout'] - (time.time() - self._crash_recovery_started)
+                time_left = self.patroni.config['primary_start_timeout'] - (time.time() - self._crash_recovery_started)
                 if time_left <= 0 and self.is_failover_possible(self.cluster.members):
                     logger.info("Demoting self because crash recovery is taking too long")
                     self.state_handler.cancellable.cancel(True)
@@ -1224,7 +1428,7 @@ class Ha(object):
 
             return 'updated leader lock during ' + self._async_executor.scheduled_action
         elif not self.state_handler.bootstrapping and not self.is_paused():
-            # Don't have lock, make sure we are not promoting or starting up a master in the background
+            # Don't have lock, make sure we are not promoting or starting up a primary in the background
             if self._async_executor.scheduled_action == 'promote':
                 with self._async_response:
                     cancel = self._async_response.cancel()
@@ -1232,8 +1436,8 @@ class Ha(object):
                     self.state_handler.cancellable.cancel()
                     return 'lost leader before promote'
 
-            if self.state_handler.role == 'master':
-                logger.info("Demoting master during " + self._async_executor.scheduled_action)
+            if self.state_handler.role in ('master', 'primary'):
+                logger.info("Demoting primary during " + self._async_executor.scheduled_action)
                 if self._async_executor.scheduled_action == 'restart':
                     # Restart needs a special interlocking cancel because postmaster may be just started in a
                     # background thread and has not even written a pid file yet.
@@ -1258,7 +1462,7 @@ class Ha(object):
         if not self.state_handler.is_running():
             self.watchdog.disable()
             if self.has_lock():
-                if self.state_handler.role in ('master', 'standby_leader'):
+                if self.state_handler.role in ('master', 'primary', 'standby_leader'):
                     self.state_handler.set_role('demoted')
                 self._delete_leader()
                 return 'removed leader key after trying and failing to start postgres'
@@ -1292,6 +1496,7 @@ class Ha(object):
         if not self.watchdog.activate():
             logger.error('Cancelling bootstrap because watchdog activation failed')
             self.cancel_initialization()
+
         self._rewind.ensure_checkpoint_after_promote(self.wakeup)
         self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
         self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':')))
@@ -1320,16 +1525,16 @@ class Ha(object):
                 self.demote('immediate-nolock')
                 return 'stopped PostgreSQL while starting up because leader key was lost'
 
-            timeout = self._start_timeout or self.patroni.config['master_start_timeout']
+            timeout = self._start_timeout or self.patroni.config['primary_start_timeout']
             time_left = timeout - self.state_handler.time_in_state()
 
             if time_left <= 0:
                 if self.is_failover_possible(self.cluster.members):
-                    logger.info("Demoting self because master startup is taking too long")
+                    logger.info("Demoting self because primary startup is taking too long")
                     self.demote('immediate')
                     return 'stopped PostgreSQL because of startup timeout'
                 else:
-                    return 'master start has timed out, but continuing to wait because failover is not possible'
+                    return 'primary start has timed out, but continuing to wait because failover is not possible'
             else:
                 msg = self.process_manual_failover_from_leader()
                 if msg is not None:
@@ -1342,7 +1547,7 @@ class Ha(object):
             return None
 
     def set_start_timeout(self, value):
-        """Sets timeout for starting as master before eligible for failover.
+        """Sets timeout for starting as primary before eligible for failover.
 
         Must be called when async_executor is busy or in the main thread."""
         self._start_timeout = value
@@ -1407,17 +1612,27 @@ class Ha(object):
                     return 'started as a secondary'
 
             # is data directory empty?
-            if self.state_handler.data_directory_empty():
+            try:
+                data_directory_is_empty = self.state_handler.data_directory_empty()
+                data_directory_is_accessible = True
+            except OSError as e:
+                data_directory_is_accessible = False
+                data_directory_error = e
+
+            if not data_directory_is_accessible or data_directory_is_empty:
                 self.state_handler.set_role('uninitialized')
                 self.state_handler.stop('immediate', stop_timeout=self.patroni.config['retry_timeout'])
-                # In case datadir went away while we were master.
+                # In case datadir went away while we were primary
                 self.watchdog.disable()
 
                 # is this instance the leader?
                 if self.has_lock():
                     self.release_leader_key_voluntarily()
-                    return 'released leader key voluntarily as data dir empty and currently leader'
+                    return 'released leader key voluntarily as data dir {0} and currently leader'.format(
+                                'empty' if data_directory_is_accessible else 'not accessible')
 
+                if not data_directory_is_accessible:
+                    return 'data directory is not accessible: {0}'.format(data_directory_error)
                 if self.is_paused():
                     return 'running with empty data directory'
                 return self.bootstrap()  # new node
@@ -1447,7 +1662,7 @@ class Ha(object):
                             and not self.state_handler.is_leader():
                         self._join_aborted = True
                         logger.error('No initialize key in DCS and PostgreSQL is running as replica, aborting start')
-                        logger.error('Please first start Patroni on the node running as master')
+                        logger.error('Please first start Patroni on the node running as primary')
                         sys.exit(1)
                     self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=data_sysid)
 
@@ -1471,43 +1686,85 @@ class Ha(object):
                 # try to start dead postgres
                 return self.recover()
 
-            try:
-                if self.cluster.is_unlocked():
-                    ret = self.process_unhealthy_cluster()
-                else:
-                    msg = self.process_healthy_cluster()
-                    ret = self.evaluate_scheduled_restart() or msg
-            finally:
-                # we might not have a valid PostgreSQL connection here if another thread
-                # stops PostgreSQL, therefore, we only reload replication slots if no
-                # asynchronous processes are running (should be always the case for the master)
-                if not self._async_executor.busy and not self.state_handler.is_starting():
-                    create_slots = self.state_handler.slots_handler.sync_replication_slots(self.cluster,
-                                                                                           self.patroni.nofailover)
-                    if not self.state_handler.cb_called:
-                        if not self.state_handler.is_leader():
-                            self._rewind.trigger_check_diverged_lsn()
-                        self.state_handler.call_nowait(ACTION_ON_START)
-                    if create_slots and self.cluster.leader:
-                        err = self._async_executor.try_run_async('copy_logical_slots',
-                                                                 self.state_handler.slots_handler.copy_logical_slots,
-                                                                 args=(self.cluster, create_slots))
-                        if not err:
-                            ret = 'Copying logical slots {0} from the primary'.format(create_slots)
+            if self.cluster.is_unlocked():
+                ret = self.process_unhealthy_cluster()
+            else:
+                msg = self.process_healthy_cluster()
+                ret = self.evaluate_scheduled_restart() or msg
+
+            # we might not have a valid PostgreSQL connection here if another thread
+            # stops PostgreSQL, therefore, we only reload replication slots if no
+            # asynchronous processes are running (should be always the case for the primary)
+            if not self._async_executor.busy and not self.state_handler.is_starting():
+                create_slots = self._sync_replication_slots(False)
+                if not self.state_handler.cb_called:
+                    if not self.state_handler.is_leader():
+                        self._rewind.trigger_check_diverged_lsn()
+                    self.state_handler.call_nowait(ACTION_ON_START)
+                if create_slots and self.cluster.leader:
+                    err = self._async_executor.try_run_async('copy_logical_slots',
+                                                             self.state_handler.slots_handler.copy_logical_slots,
+                                                             args=(self.cluster, create_slots))
+                    if not err:
+                        ret = 'Copying logical slots {0} from the primary'.format(create_slots)
             return ret
         except DCSError:
             dcs_failed = True
             logger.error('Error communicating with DCS')
-            if not self.is_paused() and self.state_handler.is_running() and self.state_handler.is_leader():
-                self.demote('offline')
-                return 'demoted self because DCS is not accessible and i was a leader'
-            return 'DCS is not accessible'
+            return self._handle_dcs_error()
         except (psycopg.Error, PostgresConnectionException):
             return 'Error communicating with PostgreSQL. Will try again later'
         finally:
             if not dcs_failed:
+                if self.is_leader():
+                    self._failsafe.set_is_active(0)
                 self.touch_member()
 
+    def _handle_dcs_error(self):
+        if not self.is_paused() and self.state_handler.is_running():
+            if self.state_handler.is_leader():
+                if self.is_failsafe_mode() and self.check_failsafe_topology():
+                    self.set_is_leader(True)
+                    self._failsafe.set_is_active(time.time())
+                    self.watchdog.keepalive()
+                    return 'continue to run as a leader because failsafe mode is enabled and all members are accessible'
+                self._failsafe.set_is_active(0)
+                msg = 'demoting self because DCS is not accessible and I was a leader'
+                if not self._async_executor.try_run_async(msg, self.demote, ('offline',)):
+                    return msg
+                logger.warning('AsyncExecutor is busy, demoting from the main thread')
+                self.demote('offline')
+                return 'demoted self because DCS is not accessible and I was a leader'
+            else:
+                self._sync_replication_slots(True)
+        return 'DCS is not accessible'
+
+    def _sync_replication_slots(self, dcs_failed):
+        """Handles replication slots.
+
+        :param dcs_failed: bool, indicates that communication with DCS failed (get_cluster() or update_leader())
+        :returns: list[str], replication slots names that should be copied from the primary"""
+
+        slots = []
+
+        # If dcs_failed we don't want to touch replication slots on a leader or replicas if failsafe_mode isn't enabled.
+        if not self.cluster or dcs_failed and (self.is_leader() or not self.is_failsafe_mode()):
+            return slots
+
+        # It could be that DCS is read-only, or only the leader can't access it.
+        # Only the second one could be handled by `load_cluster_from_dcs()`.
+        # The first one affects advancing logical replication slots on replicas, therefore we rely on
+        # Failsafe.update_cluster(), that will return "modified" Cluster if failsafe mode is active.
+        cluster = self._failsafe.update_cluster(self.cluster)\
+            if self.is_failsafe_mode() and not self.is_leader() else self.cluster
+        if cluster:
+            slots = self.state_handler.slots_handler.sync_replication_slots(cluster,
+                                                                            self.patroni.nofailover,
+                                                                            self.patroni.replicatefrom,
+                                                                            self.is_paused())
+        # Don't copy replication slots if failsafe_mode is active
+        return [] if self.failsafe_is_active() else slots
+
     def run_cycle(self):
         with self._async_executor:
             try:
@@ -1544,10 +1801,15 @@ class Ha(object):
                     else:
                         self.dcs.write_leader_optime(checkpoint_location)
 
+            def _before_shutdown():
+                self.notify_citus_coordinator('before_demote')
+
             on_shutdown = _on_shutdown if self.is_leader() else None
+            before_shutdown = _before_shutdown if self.is_leader() else None
             self.while_not_sync_standby(lambda: self.state_handler.stop(checkpoint=False, on_safepoint=disable_wd,
                                                                         on_shutdown=on_shutdown,
-                                                                        stop_timeout=self.master_stop_timeout()))
+                                                                        before_shutdown=before_shutdown,
+                                                                        stop_timeout=self.primary_stop_timeout()))
             if not self.state_handler.is_running():
                 if self.is_leader() and not status['deleted']:
                     checkpoint_location = self.state_handler.latest_checkpoint_location()
@@ -1572,18 +1834,18 @@ class Ha(object):
     def wakeup(self):
         """Call of this method will trigger the next run of HA loop if there is
         no "active" leader watch request in progress.
-        This usually happens on the master or if the node is running async action"""
+        This usually happens on the leader or if the node is running async action"""
         self.dcs.event.set()
 
     def get_remote_member(self, member=None):
         """ In case of standby cluster this will tel us from which remote
-            master to stream. Config can be both patroni config or
+            member to stream. Config can be both patroni config or
             cluster.config.data
         """
         cluster_params = self.get_standby_cluster_config()
 
         if cluster_params:
-            name = member.name if member else 'remote_master:{}'.format(uuid.uuid1())
+            name = member.name if member else 'remote_member:{}'.format(uuid.uuid1())
 
             data = {k: v for k, v in cluster_params.items() if k in RemoteMember.allowed_keys()}
             data['no_replication_slot'] = 'primary_slot_name' not in cluster_params
@@ -1593,6 +1855,3 @@ class Ha(object):
                 data['conn_kwargs'] = conn_kwargs
 
             return RemoteMember(name, data)
-
-    def get_remote_master(self):
-        return self.get_remote_member()

patroni/postgresql/__init__.py

@@ -19,9 +19,11 @@ from .callback_executor import CallbackExecutor
 from .cancellable import CancellableSubprocess
 from .config import ConfigHandler, mtime
 from .connection import Connection, get_connection_cursor
+from .citus import CitusHandler
 from .misc import parse_history, parse_lsn, postgres_major_version_to_int
 from .postmaster import PostmasterProcess
 from .slots import SlotsHandler
+from .sync import SyncHandler
 from .. import psycopg
 from ..exceptions import PostgresConnectionException
 from ..utils import Retry, RetryFailedError, polling_loop, data_directory_is_empty, parse_int
@@ -54,7 +56,7 @@ class Postgresql(object):
     POSTMASTER_START_TIME = "pg_catalog.pg_postmaster_start_time()"
     TL_LSN = ("CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 "
               "ELSE ('x' || pg_catalog.substr(pg_catalog.pg_{0}file_name("
-              "pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END, "  # master timeline
+              "pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END, "  # primary timeline
               "CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 "
               "ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), '0/0')::bigint END, "  # write_lsn
               "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), '0/0')::bigint, "
@@ -75,6 +77,7 @@ class Postgresql(object):
 
         self._pending_restart = False
         self._connection = Connection()
+        self.citus_handler = CitusHandler(self, config.get('citus'))
         self.config = ConfigHandler(self, config)
         self.config.check_directories()
 
@@ -84,6 +87,7 @@ class Postgresql(object):
         self.__thread_ident = current_thread().ident
 
         self.slots_handler = SlotsHandler(self)
+        self.sync_handler = SyncHandler(self)
 
         self._callback_executor = CallbackExecutor()
         self.__cb_called = False
@@ -106,6 +110,7 @@ class Postgresql(object):
         self._cluster_info_state = {}
         self._has_permanent_logical_slots = True
         self._enforce_hot_standby_feedback = False
+        self._is_synchronous_mode = True
         self._cached_replica_timeline = None
 
         # Last known running process
@@ -121,7 +126,7 @@ class Postgresql(object):
             ident_saved = self.config.replace_pg_ident()
             if hba_saved or ident_saved:
                 self.reload()
-        elif self.role == 'master':
+        elif self.role in ('master', 'primary'):
             self.set_role('demoted')
 
     @property
@@ -158,11 +163,36 @@ class Postgresql(object):
 
     @property
     def cluster_info_query(self):
+        """Returns the monitoring query with a fixed number of fields.
+
+        The query text is constructed based on current state in DCS and PostgreSQL version:
+        1. function names depend on version. wal/lsn for v10+ and xlog/location for pre v10.
+        2. for primary we query timeline_id (extracted from pg_walfile_name()) and pg_current_wal_lsn()
+        3. for replicas we query pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn(), and  pg_is_wal_replay_paused()
+        4. for v9.6+ we query primary_slot_name and primary_conninfo from pg_stat_get_wal_receiver()
+        5. for v11+ with permanent logical slots we query from pg_replication_slots and aggregate the result
+        6. for standby_leader node running v9.6+ we also query pg_control_checkpoint to fetch timeline_id
+        7. if sync replication is enabled we query pg_stat_replication and aggregate the result.
+           In addition to that we get current values of synchronous_commit and synchronous_standby_names GUCs.
+
+        If some conditions are not satisfied we simply put static values instead. E.g., NULL, 0, '', and so on."""
+
+        extra = ", " + (("pg_catalog.current_setting('synchronous_commit'), " +
+                         "pg_catalog.current_setting('synchronous_standby_names'), "
+                         "(SELECT pg_catalog.json_agg(r.*) FROM (SELECT w.pid as pid, application_name, sync_state," +
+                         " pg_catalog.pg_{0}_{1}_diff(write_{1}, '0/0')::bigint AS write_lsn," +
+                         " pg_catalog.pg_{0}_{1}_diff(flush_{1}, '0/0')::bigint AS flush_lsn," +
+                         " pg_catalog.pg_{0}_{1}_diff(replay_{1}, '0/0')::bigint AS replay_lsn " +
+                         "FROM pg_catalog.pg_stat_get_wal_senders() w," +
+                         " pg_catalog.pg_stat_get_activity(w.pid)" +
+                         " WHERE w.state = 'streaming') r)").format(self.wal_name, self.lsn_name)
+                        if self._is_synchronous_mode and self.role in ('master', 'primary') else "'on', '', NULL")
+
         if self._major_version >= 90600:
-            extra = "(SELECT pg_catalog.json_agg(s.*) FROM (SELECT slot_name, slot_type as type, datoid::bigint, " +\
-                    "plugin, catalog_xmin, pg_catalog.pg_wal_lsn_diff(confirmed_flush_lsn, '0/0')::bigint" + \
-                    " AS confirmed_flush_lsn FROM pg_catalog.pg_get_replication_slots()) AS s)"\
-                            if self._has_permanent_logical_slots and self._major_version >= 110000 else "NULL"
+            extra = ("(SELECT pg_catalog.json_agg(s.*) FROM (SELECT slot_name, slot_type as type, datoid::bigint, " +
+                     "plugin, catalog_xmin, pg_catalog.pg_wal_lsn_diff(confirmed_flush_lsn, '0/0')::bigint" +
+                     " AS confirmed_flush_lsn FROM pg_catalog.pg_get_replication_slots()) AS s)"
+                     if self._has_permanent_logical_slots and self._major_version >= 110000 else "NULL") + extra
             extra = (", CASE WHEN latest_end_lsn IS NULL THEN NULL ELSE received_tli END,"
                      " slot_name, conninfo, {0} FROM pg_catalog.pg_stat_get_wal_receiver()").format(extra)
             if self.role == 'standby_leader':
@@ -170,7 +200,7 @@ class Postgresql(object):
             else:
                 extra = "0" + extra
         else:
-            extra = "0, NULL, NULL, NULL, NULL"
+            extra = "0, NULL, NULL, NULL, NULL" + extra
 
         return ("SELECT " + self.TL_LSN + ", {2}").format(self.wal_name, self.lsn_name, extra)
 
@@ -255,7 +285,8 @@ class Postgresql(object):
         return self._connection.get()
 
     def set_connection_kwargs(self, kwargs):
-        self._connection.set_conn_kwargs(kwargs)
+        self._connection.set_conn_kwargs(kwargs.copy())
+        self.citus_handler.set_conn_kwargs(kwargs.copy())
 
     def _query(self, sql, *params):
         """We are always using the same cursor, therefore this method is not thread-safe!!!
@@ -300,7 +331,8 @@ class Postgresql(object):
         return deepcopy(self.config.get(method, {}))
 
     def replica_method_can_work_without_replication_connection(self, method):
-        return method != 'basebackup' and self.replica_method_options(method).get('no_master')
+        return method != 'basebackup' and (self.replica_method_options(method).get('no_master') or
+                                           self.replica_method_options(method).get('no_leader'))
 
     def can_create_replica_without_replication_connection(self, replica_methods=None):
         """ go through the replication methods to see if there are ones
@@ -334,13 +366,16 @@ class Postgresql(object):
                 self._has_permanent_logical_slots or
                 cluster.should_enforce_hot_standby_feedback(self.name, nofailover, self.major_version))
 
+            self._is_synchronous_mode = cluster.is_synchronous_mode()
+
     def _cluster_info_state_get(self, name):
         if not self._cluster_info_state:
             try:
                 result = self._is_leader_retry(self._query, self.cluster_info_query).fetchone()
                 cluster_info_state = dict(zip(['timeline', 'wal_position', 'replayed_location',
                                                'received_location', 'replay_paused', 'pg_control_timeline',
-                                               'received_tli', 'slot_name', 'conninfo', 'slots'], result))
+                                               'received_tli', 'slot_name', 'conninfo', 'slots', 'synchronous_commit',
+                                               'synchronous_standby_names', 'pg_stat_replication'], result))
                 if self._has_permanent_logical_slots:
                     cluster_info_state['slots'] =\
                         self.slots_handler.process_permanent_slots(cluster_info_state['slots'])
@@ -373,12 +408,21 @@ class Postgresql(object):
     def received_timeline(self):
         return self._cluster_info_state_get('received_tli')
 
+    def synchronous_commit(self):
+        return self._cluster_info_state_get('synchronous_commit')
+
+    def synchronous_standby_names(self):
+        return self._cluster_info_state_get('synchronous_standby_names')
+
+    def pg_stat_replication(self):
+        return self._cluster_info_state_get('pg_stat_replication') or []
+
     def is_leader(self):
         try:
             return bool(self._cluster_info_state_get('timeline'))
         except PostgresConnectionException:
             logger.warning('Failed to determine PostgreSQL state from the connection, falling back to cached role')
-            return bool(self.is_running() and self.role == 'master')
+            return bool(self.is_running() and self.role in ('master', 'primary'))
 
     def replay_paused(self):
         return self._cluster_info_state_get('replay_paused')
@@ -428,11 +472,11 @@ class Postgresql(object):
                     # If the cluster is shutdown with archive_mode=on, WAL is switched before writing the checkpoint.
                     # In this case we want to take the LSN of previous record (switch) as the last known WAL location.
                     if parse_lsn(lsn) == prev and desc.strip() in ('xlog switch', 'SWITCH'):
-                        return str(prev)
+                        return prev
             except Exception as e:
                 logger.error('Exception when parsing WAL pg_%sdump output: %r', self.wal_name, e)
             if isinstance(checkpoint_lsn, six.integer_types):
-                return str(checkpoint_lsn)
+                return checkpoint_lsn
 
     def is_running(self):
         """Returns PostmasterProcess if one is running on the data directory or None. If most recently seen process
@@ -512,7 +556,7 @@ class Postgresql(object):
         logger.warning("Timed out waiting for PostgreSQL to start")
         return False
 
-    def start(self, timeout=None, task=None, block_callbacks=False, role=None):
+    def start(self, timeout=None, task=None, block_callbacks=False, role=None, after_start=None):
         """Start PostgreSQL
 
         Waits for postmaster to open ports or terminate so pg_isready can be used to check startup completion
@@ -583,6 +627,8 @@ class Postgresql(object):
 
         ret = self.wait_for_startup(start_timeout)
         if ret is not None:
+            if ret and after_start:
+                after_start()
             return ret
         elif timeout is not None:
             return False
@@ -609,7 +655,7 @@ class Postgresql(object):
             return 'not accessible or not healty'
 
     def stop(self, mode='fast', block_callbacks=False, checkpoint=None,
-             on_safepoint=None, on_shutdown=None, stop_timeout=None):
+             on_safepoint=None, on_shutdown=None, before_shutdown=None, stop_timeout=None):
         """Stop PostgreSQL
 
         Supports a callback when a safepoint is reached. A safepoint is when no user backend can return a successful
@@ -618,11 +664,13 @@ class Postgresql(object):
 
         :param on_safepoint: This callback is called when no user backends are running.
         :param on_shutdown: is called when pg_controldata starts reporting `Database cluster state: shut down`
+        :param before_shutdown: is called after running optional CHECKPOINT and before running pg_ctl stop
         """
         if checkpoint is None:
             checkpoint = False if mode == 'immediate' else True
 
-        success, pg_signaled = self._do_stop(mode, block_callbacks, checkpoint, on_safepoint, on_shutdown, stop_timeout)
+        success, pg_signaled = self._do_stop(mode, block_callbacks, checkpoint, on_safepoint,
+                                             on_shutdown, before_shutdown, stop_timeout)
         if success:
             # block_callbacks is used during restart to avoid
             # running start/stop callbacks in addition to restart ones
@@ -635,7 +683,7 @@ class Postgresql(object):
             self.set_state('stop failed')
         return success
 
-    def _do_stop(self, mode, block_callbacks, checkpoint, on_safepoint, on_shutdown, stop_timeout):
+    def _do_stop(self, mode, block_callbacks, checkpoint, on_safepoint, on_shutdown, before_shutdown, stop_timeout):
         postmaster = self.is_running()
         if not postmaster:
             if on_safepoint:
@@ -648,6 +696,9 @@ class Postgresql(object):
         if not block_callbacks:
             self.set_state('stopping')
 
+        if before_shutdown:
+            before_shutdown()
+
         # Send signal to postmaster to stop
         success = postmaster.signal_stop(mode, self.pgcommand('pg_ctl'))
         if success is not None:
@@ -659,7 +710,7 @@ class Postgresql(object):
         if on_safepoint:
             # Wait for our connection to terminate so we can be sure that no new connections are being initiated
             self._wait_for_connection_close(postmaster)
-            postmaster.wait_for_user_backends_to_close()
+            postmaster.wait_for_user_backends_to_close(stop_timeout)
             on_safepoint()
 
         if on_shutdown and mode in ('fast', 'smart'):
@@ -668,7 +719,7 @@ class Postgresql(object):
             while postmaster.is_running():
                 data = self.controldata()
                 if data.get('Database cluster state', '') == 'shut down':
-                    on_shutdown(int(self.latest_checkpoint_location()))
+                    on_shutdown(self.latest_checkpoint_location())
                     break
                 elif data.get('Database cluster state', '').startswith('shut down'):  # shut down in recovery
                     break
@@ -775,7 +826,8 @@ class Postgresql(object):
 
         return self.state == 'running'
 
-    def restart(self, timeout=None, task=None, block_callbacks=False, role=None):
+    def restart(self, timeout=None, task=None, block_callbacks=False,
+                role=None, before_shutdown=None, after_start=None):
         """Restarts PostgreSQL.
 
         When timeout parameter is set the call will block either until PostgreSQL has started, failed to start or
@@ -786,7 +838,8 @@ class Postgresql(object):
         self.set_state('restarting')
         if not block_callbacks:
             self.__cb_pending = ACTION_ON_RESTART
-        ret = self.stop(block_callbacks=True) and self.start(timeout, task, True, role)
+        ret = self.stop(block_callbacks=True, before_shutdown=before_shutdown)\
+            and self.start(timeout, task, True, role, after_start)
         if not ret and not self.is_starting():
             self.set_state('restart failed ({0})'.format(self.state))
         return ret
@@ -853,12 +906,13 @@ class Postgresql(object):
         except Exception:
             logger.exception('Can not fetch local timeline and lsn from replication connection')
 
-    def replica_cached_timeline(self, master_timeline):
-        if not self._cached_replica_timeline or not master_timeline or self._cached_replica_timeline != master_timeline:
+    def replica_cached_timeline(self, primary_timeline):
+        if not self._cached_replica_timeline or not primary_timeline\
+                or self._cached_replica_timeline != primary_timeline:
             self._cached_replica_timeline = self.get_replica_timeline()
         return self._cached_replica_timeline
 
-    def get_master_timeline(self):
+    def get_primary_timeline(self):
         return self._cluster_info_state_get('timeline')
 
     def get_history(self, timeline):
@@ -881,11 +935,11 @@ class Postgresql(object):
         recovery_params = self.config.build_recovery_params(member)
         self.config.write_recovery_conf(recovery_params)
 
-        # When we demoting the master or standby_leader to replica or promoting replica to a standby_leader
+        # When we demoting the primary or standby_leader to replica or promoting replica to a standby_leader
         # and we know for sure that postgres was already running before, we will only execute on_role_change
         # callback and prevent execution of on_restart/on_start callback.
         # If the role remains the same (replica or standby_leader), we will execute on_start or on_restart
-        change_role = self.cb_called and (self.role in ('master', 'demoted') or
+        change_role = self.cb_called and (self.role in ('master', 'primary', 'demoted') or
                                           not {'standby_leader', 'replica'} - {self.role, role})
         if change_role:
             self.__cb_pending = ACTION_NOOP
@@ -911,6 +965,7 @@ class Postgresql(object):
         for _ in polling_loop(wait_seconds):
             data = self.controldata()
             if data.get('Database cluster state') == 'in production':
+                self.set_role('master')
                 return True
 
     def _pre_promote(self):
@@ -928,8 +983,8 @@ class Postgresql(object):
             logger.info('pre_promote script `%s` exited with %s', cmd, ret)
         return ret == 0
 
-    def promote(self, wait_seconds, task, on_success=None):
-        if self.role == 'master':
+    def promote(self, wait_seconds, task, before_promote=None, on_success=None):
+        if self.role in ('promoted', 'master', 'primary'):
             return True
 
         ret = self._pre_promote()
@@ -945,11 +1000,15 @@ class Postgresql(object):
             logger.info("PostgreSQL promote cancelled.")
             return False
 
+        if before_promote is not None:
+            before_promote()
+
         self.slots_handler.on_promote()
+        self.citus_handler.schedule_cache_rebuild()
 
         ret = self.pg_ctl('promote', '-W')
         if ret:
-            self.set_role('master')
+            self.set_role('promoted')
             if on_success is not None:
                 on_success()
             self.call_nowait(ACTION_ON_ROLE_CHANGE)
@@ -1023,13 +1082,15 @@ class Postgresql(object):
     def move_data_directory(self):
         if os.path.isdir(self._data_dir) and not self.is_running():
             try:
-                postfix = time.strftime('%Y-%m-%d-%H-%M-%S')
+                postfix = 'failed'
 
                 # let's see if the wal directory is a symlink, in this case we
                 # should move the target
                 for (source, pg_wal_realpath) in self.pg_wal_realpath().items():
                     logger.info('renaming WAL directory and updating symlink: %s', pg_wal_realpath)
-                    new_name = '{0}_{1}'.format(pg_wal_realpath, postfix)
+                    new_name = '{0}.{1}'.format(pg_wal_realpath, postfix)
+                    if os.path.exists(new_name):
+                        shutil.rmtree(new_name)
                     os.rename(pg_wal_realpath, new_name)
                     os.unlink(source)
                     os.symlink(new_name, source)
@@ -1037,13 +1098,17 @@ class Postgresql(object):
                 # Move user defined tablespace directory
                 for (source, pg_tsp_rpath) in self.pg_tblspc_realpaths().items():
                     logger.info('renaming user defined tablespace directory and updating symlink: %s', pg_tsp_rpath)
-                    new_name = '{0}_{1}'.format(pg_tsp_rpath, postfix)
+                    new_name = '{0}.{1}'.format(pg_tsp_rpath, postfix)
+                    if os.path.exists(new_name):
+                        shutil.rmtree(new_name)
                     os.rename(pg_tsp_rpath, new_name)
                     os.unlink(source)
                     os.symlink(new_name, source)
 
-                new_name = '{0}_{1}'.format(self._data_dir, postfix)
+                new_name = '{0}.{1}'.format(self._data_dir, postfix)
                 logger.info('renaming data directory to %s', new_name)
+                if os.path.exists(new_name):
+                    shutil.rmtree(new_name)
                 os.rename(self._data_dir, new_name)
             except OSError:
                 logger.exception("Could not rename data directory %s", self._data_dir)
@@ -1076,58 +1141,6 @@ class Postgresql(object):
             logger.exception('Could not remove data directory %s', self._data_dir)
             self.move_data_directory()
 
-    def _get_synchronous_commit_param(self):
-        return self.query("SHOW synchronous_commit").fetchone()[0]
-
-    def pick_synchronous_standby(self, cluster, sync_node_count=1, sync_node_maxlag=-1):
-        """Finds the best candidate to be the synchronous standby.
-
-        Current synchronous standby is always preferred, unless it has disconnected or does not want to be a
-        synchronous standby any longer.
-        Parameter sync_node_maxlag(maximum_lag_on_syncnode) would help swapping unhealthy sync replica in case
-        if it stops responding (or hung). Please set the value high enough so it won't unncessarily swap sync
-        standbys during high loads. Any less or equal of 0 value keep the behavior backward compatible and
-        will not swap. Please note that it will not also swap sync standbys in case where all replicas are hung.
-
-        :returns tuple of candidates list and synchronous standby list.
-        """
-        if self._major_version < 90600:
-            sync_node_count = 1
-        members = {m.name.lower(): m for m in cluster.members}
-        candidates = []
-        sync_nodes = []
-        replica_list = []
-        # Pick candidates based on who has higher replay/remote_write/flush lsn.
-        sync_commit_par = self._get_synchronous_commit_param()
-        sort_col = {'remote_apply': 'replay', 'remote_write': 'write'}.get(sync_commit_par, 'flush')
-        # pg_stat_replication.sync_state has 4 possible states - async, potential, quorum, sync.
-        # Sort clause "ORDER BY sync_state DESC" is to get the result in required order and to keep
-        # the result consistent in case if a synchronous standby member is slowed down OR async node
-        # receiving changes faster than the sync member (very rare but possible). Such cases would
-        # trigger sync standby member swapping frequently and the sort on sync_state desc should
-        # help in keeping the query result consistent.
-        for app_name, sync_state, replica_lsn in self.query(
-                "SELECT pg_catalog.lower(application_name), sync_state, pg_{2}_{1}_diff({0}_{1}, '0/0')::bigint"
-                " FROM pg_catalog.pg_stat_replication"
-                " WHERE state = 'streaming' AND {0}_{1} IS NOT NULL"
-                " ORDER BY sync_state DESC, {0}_{1} DESC".format(sort_col, self.lsn_name, self.wal_name)):
-            member = members.get(app_name)
-            if member and not member.tags.get('nosync', False):
-                replica_list.append((member.name, sync_state, replica_lsn, bool(member.nofailover)))
-
-        max_lsn = max(replica_list, key=lambda x: x[2])[2] if len(replica_list) > 1 else int(str(self.last_operation()))
-
-        # Prefer members without nofailover tag. We are relying on the fact that sorts are guaranteed to be stable.
-        for app_name, sync_state, replica_lsn, _ in sorted(replica_list, key=lambda x: x[3]):
-            if sync_node_maxlag <= 0 or max_lsn - replica_lsn <= sync_node_maxlag:
-                candidates.append(app_name)
-                if sync_state == 'sync':
-                    sync_nodes.append(app_name)
-            if len(candidates) >= sync_node_count:
-                break
-
-        return candidates, sync_nodes
-
     def schedule_sanity_checks_after_pause(self):
         """
             After coming out of pause we have to:
@@ -1138,4 +1151,5 @@ class Postgresql(object):
         if not self._major_version:
             self.configure_server_parameters()
         self.slots_handler.schedule()
+        self.citus_handler.schedule_cache_rebuild()
         self._sysid = None

patroni/postgresql/bootstrap.py

@@ -155,12 +155,12 @@ class Bootstrap(object):
         self._postgresql.set_state('creating replica')
         self._postgresql.schedule_sanity_checks_after_pause()
 
-        is_remote_master = isinstance(clone_member, RemoteMember)
+        is_remote_member = isinstance(clone_member, RemoteMember)
 
         # get list of replica methods either from clone member or from
         # the config. If there is no configuration key, or no value is
         # specified, use basebackup
-        replica_methods = (clone_member.create_replica_methods if is_remote_master
+        replica_methods = (clone_member.create_replica_methods if is_remote_member
                            else self._postgresql.create_replica_methods) or ['basebackup']
 
         if clone_member and clone_member.conn_url:
@@ -212,7 +212,7 @@ class Bootstrap(object):
                                           "datadir": self._postgresql.data_dir,
                                           "connstring": connstring})
                 else:
-                    for param in ('no_params', 'no_master', 'keep_data'):
+                    for param in ('no_params', 'no_master', 'no_leader', 'keep_data'):
                         method_config.pop(param, None)
                 params = ["--{0}={1}".format(arg, val) for arg, val in method_config.items()]
                 try:
@@ -269,7 +269,7 @@ class Bootstrap(object):
 
     def clone(self, clone_member):
         """
-             - initialize the replica from an existing member (master or replica)
+             - initialize the replica from an existing member (primary or replica)
              - initialize the replica using the replica creation method that
                works without the replication connection (i.e. restore from on-disk
                base backup)
@@ -315,12 +315,14 @@ END;$$""".format(quote_literal(name), quote_ident(name, self._postgresql.connect
         self._postgresql.query('SET log_statement TO none')
         self._postgresql.query('SET log_min_duration_statement TO -1')
         self._postgresql.query("SET log_min_error_statement TO 'log'")
+        self._postgresql.query("SET pg_stat_statements.track_utility to 'off'")
         try:
             self._postgresql.query(sql)
         finally:
             self._postgresql.query('RESET log_min_error_statement')
             self._postgresql.query('RESET log_min_duration_statement')
             self._postgresql.query('RESET log_statement')
+            self._postgresql.query('RESET pg_stat_statements.track_utility')
 
     def post_bootstrap(self, config, task):
         try:
@@ -343,7 +345,7 @@ END;$$""".format(quote_literal(name), quote_ident(name, self._postgresql.connect
 BEGIN
     SET local synchronous_commit = 'local';
     GRANT EXECUTE ON function pg_catalog.{0} TO {1};
-END;$$""".format(f, quote_ident(rewind['username'], self._postgresql.connection()))
+END;$$""".format(f, quote_ident(rewind['username'], postgresql.connection()))
                         postgresql.query(sql)
 
                 for name, value in (config.get('users') or {}).items():
@@ -375,6 +377,9 @@ END;$$""".format(f, quote_ident(rewind['username'], self._postgresql.connection(
                             postgresql.reload()
                             time.sleep(1)  # give a time to postgres to "reload" configuration files
                             postgresql.connection().close()  # close connection to reconnect with a new password
+                else:  # initdb
+                    # We may want create database and extension for citus
+                    self._postgresql.citus_handler.bootstrap()
         except Exception:
             logger.exception('post_bootstrap')
             task.complete(False)

patroni/postgresql/citus.py

@@ -0,0 +1,389 @@
+import logging
+import re
+import time
+
+from six.moves.urllib_parse import urlparse
+from threading import Condition, Event, Thread
+
+from .connection import Connection
+from ..dcs import CITUS_COORDINATOR_GROUP_ID
+from ..psycopg import connect, quote_ident
+
+CITUS_SLOT_NAME_RE = re.compile(r'^citus_shard_(move|split)_slot(_[1-9][0-9]*){2,3}$')
+logger = logging.getLogger(__name__)
+
+
+class PgDistNode(object):
+    """Represents a single row in the `pg_dist_node` table"""
+
+    def __init__(self, group, host, port, event, nodeid=None, timeout=None, cooldown=None):
+        self.group = group
+        # A weird way of pausing client connections by adding the `-demoted` suffix to the hostname
+        self.host = host + ('-demoted' if event == 'before_demote' else '')
+        self.port = port
+        # Event that is trying to change or changed the given row.
+        # Possible values: before_demote, before_promote, after_promote.
+        self.event = event
+        self.nodeid = nodeid
+
+        # If transaction was started, we need to COMMIT/ROLLBACK before the deadline
+        self.timeout = timeout
+        self.cooldown = cooldown or 10000  # 10s by default
+        self.deadline = 0
+
+        # All changes in the pg_dist_node are serialized on the Patroni
+        # side by performing them from a thread. The thread, that is
+        # requested a change, sometimes needs to wait for a result.
+        # For example, we want to pause client connections before demoting
+        # the worker, and once it is done notify the calling thread.
+        self._event = Event()
+
+    def wait(self):
+        self._event.wait()
+
+    def wakeup(self):
+        self._event.set()
+
+    def __eq__(self, other):
+        return isinstance(other, PgDistNode) and self.event == other.event\
+            and self.host == other.host and self.port == other.port
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __str__(self):
+        return ('PgDistNode(nodeid={0},group={1},host={2},port={3},event={4})'
+                .format(self.nodeid, self.group, self.host, self.port, self.event))
+
+    def __repr__(self):
+        return str(self)
+
+
+class CitusHandler(Thread):
+
+    def __init__(self, postgresql, config):
+        super(CitusHandler, self).__init__()
+        self.daemon = True
+        self._postgresql = postgresql
+        self._config = config
+        self._connection = Connection()
+        self._pg_dist_node = {}  # Cache of pg_dist_node: {groupid: PgDistNode()}
+        self._tasks = []  # Requests to change pg_dist_node, every task is a `PgDistNode`
+        self._condition = Condition()  # protects _pg_dist_node, _tasks, and _schedule_load_pg_dist_node
+        self._in_flight = None  # Reference to the `PgDistNode` if there is a transaction in progress changing it
+        self.schedule_cache_rebuild()
+
+    def is_enabled(self):
+        return isinstance(self._config, dict)
+
+    def group(self):
+        return self._config['group']
+
+    def is_coordinator(self):
+        return self.is_enabled() and self.group() == CITUS_COORDINATOR_GROUP_ID
+
+    def is_worker(self):
+        return self.is_enabled() and not self.is_coordinator()
+
+    def set_conn_kwargs(self, kwargs):
+        if self.is_enabled():
+            kwargs.update({'dbname': self._config['database'],
+                           'options': '-c statement_timeout=0 -c idle_in_transaction_session_timeout=0'})
+            self._connection.set_conn_kwargs(kwargs)
+
+    def schedule_cache_rebuild(self):
+        with self._condition:
+            self._schedule_load_pg_dist_node = True
+
+    def on_demote(self):
+        with self._condition:
+            self._pg_dist_node.clear()
+            self._tasks[:] = []
+            self._in_flight = None
+
+    def query(self, sql, *params):
+        try:
+            logger.debug('query(%s, %s)', sql, params)
+            cursor = self._connection.cursor()
+            cursor.execute(sql, params or None)
+            return cursor
+        except Exception as e:
+            logger.error('Exception when executing query "%s", (%s): %r', sql, params, e)
+            self._connection.close()
+            self._in_flight = None
+            self.schedule_cache_rebuild()
+            raise e
+
+    def load_pg_dist_node(self):
+        """Read from the `pg_dist_node` table and put it into the local cache"""
+
+        with self._condition:
+            if not self._schedule_load_pg_dist_node:
+                return True
+            self._schedule_load_pg_dist_node = False
+
+        try:
+            cursor = self.query("SELECT nodeid, groupid, nodename, nodeport, noderole"
+                                " FROM pg_catalog.pg_dist_node WHERE noderole = 'primary'")
+        except Exception:
+            return False
+
+        with self._condition:
+            self._pg_dist_node = {r[1]: PgDistNode(r[1], r[2], r[3], 'after_promote', r[0]) for r in cursor}
+        return True
+
+    def sync_pg_dist_node(self, cluster):
+        """Maintain the `pg_dist_node` from the coordinator leader every heartbeat loop.
+
+        We can't always rely on REST API calls from worker nodes in order
+        to maintain `pg_dist_node`, therefore at least once per heartbeat
+        loop we make sure that workes registered in `self._pg_dist_node`
+        cache are matching the cluster view from DCS by creating tasks
+        the same way as it is done from the REST API."""
+
+        if not self.is_coordinator():
+            return
+
+        with self._condition:
+            if not self.is_alive():
+                self.start()
+
+        self.add_task('after_promote', CITUS_COORDINATOR_GROUP_ID, self._postgresql.connection_string)
+
+        for group, worker in cluster.workers.items():
+            leader = worker.leader
+            if leader and leader.conn_url\
+                    and leader.data.get('role') in ('master', 'primary') and leader.data.get('state') == 'running':
+                self.add_task('after_promote', group, leader.conn_url)
+
+    def find_task_by_group(self, group):
+        for i, task in enumerate(self._tasks):
+            if task.group == group:
+                return i
+
+    def pick_task(self):
+        """Returns the tuple(i, task), where `i` - is the task index in the self._tasks list
+
+        Tasks are picked by following priorities:
+        1. If there is already a transaction in progress, pick a task
+           that that will change already affected worker primary.
+        2. If the coordinator address should be changed - pick a task
+           with group=0 (coordinators are always in group 0).
+        3. Pick a task that is the oldest (first from the self._tasks)"""
+
+        with self._condition:
+            if self._in_flight:
+                i = self.find_task_by_group(self._in_flight.group)
+            else:
+                while True:
+                    i = self.find_task_by_group(CITUS_COORDINATOR_GROUP_ID)  # set_coordinator
+                    if i is None and self._tasks:
+                        i = 0
+                    if i is None:
+                        break
+                    task = self._tasks[i]
+                    if task == self._pg_dist_node.get(task.group):
+                        self._tasks.pop(i)  # nothing to do because cached version of pg_dist_node already matches
+                    else:
+                        break
+            task = self._tasks[i] if i is not None else None
+
+            # When tasks are added it could happen that self._pg_dist_node
+            # wasn't ready (self._schedule_load_pg_dist_node is False)
+            # and hence the nodeid wasn't filled.
+            if task and task.group in self._pg_dist_node:
+                task.nodeid = self._pg_dist_node[task.group].nodeid
+            return i, task
+
+    def update_node(self, task):
+        if task.group == CITUS_COORDINATOR_GROUP_ID:
+            return self.query("SELECT pg_catalog.citus_set_coordinator_host(%s, %s, 'primary', 'default')",
+                              task.host, task.port)
+
+        if task.nodeid is None and task.event != 'before_demote':
+            task.nodeid = self.query("SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default')",
+                                     task.host, task.port, task.group).fetchone()[0]
+        elif task.nodeid is not None:
+            # XXX: statement_timeout?
+            self.query('SELECT pg_catalog.citus_update_node(%s, %s, %s, true, %s)',
+                       task.nodeid, task.host, task.port, task.cooldown)
+
+    def process_task(self, task):
+        """Updates a single row in `pg_dist_node` table, optionally in a transaction.
+
+        The transaction is started if we do a demote of the worker node
+        or before promoting the other worker if there is not transaction
+        in progress. And, the transaction it is committed when the
+        switchover/failover completed.
+
+        This method returns `True` if node was updated (optionally,
+        transaction was committed) as an indicator that
+        the `self._pg_dist_node` cache should be updated.
+
+        The maximum lifetime of the transaction in progress
+        is controlled outside of this method."""
+
+        if task.event == 'after_promote':
+            # The after_promote may happen without previous before_demote and/or
+            # before_promore.  In this case we just call self.update_node() method.
+            # If there is a transaction in progress, it could be that it already did
+            # required changes and we can simply COMMIT.
+            if not self._in_flight or self._in_flight.host != task.host or self._in_flight.port != task.port:
+                self.update_node(task)
+            if self._in_flight:
+                self.query('COMMIT')
+                self._in_flight = None
+            return True
+        else:  # before_demote, before_promote
+            if task.timeout:
+                task.deadline = time.time() + task.timeout
+            if not self._in_flight:
+                self.query('BEGIN')
+            self.update_node(task)
+            self._in_flight = task
+        return False
+
+    def process_tasks(self):
+        while True:
+            if not self._in_flight and not self.load_pg_dist_node():
+                break
+
+            i, task = self.pick_task()
+            if not task:
+                break
+            try:
+                update_cache = self.process_task(task)
+            except Exception as e:
+                logger.error('Exception when working with pg_dist_node: %r', e)
+                update_cache = False
+            with self._condition:
+                if self._tasks:
+                    if update_cache:
+                        self._pg_dist_node[task.group] = task
+                    if id(self._tasks[i]) == id(task):
+                        self._tasks.pop(i)
+            task.wakeup()
+
+    def run(self):
+        while True:
+            try:
+                with self._condition:
+                    if self._schedule_load_pg_dist_node:
+                        timeout = -1
+                    elif self._in_flight:
+                        timeout = self._in_flight.deadline - time.time() if self._tasks else None
+                    else:
+                        timeout = -1 if self._tasks else None
+
+                    if timeout is None or timeout > 0:
+                        self._condition.wait(timeout)
+                    elif self._in_flight:
+                        logger.warning('Rolling back transaction. Last known status: %s', self._in_flight)
+                        self.query('ROLLBACK')
+                        self._in_flight = None
+                self.process_tasks()
+            except Exception:
+                logger.exception('run')
+
+    def _add_task(self, task):
+        with self._condition:
+            i = self.find_task_by_group(task.group)
+
+            # task.timeout is None is an indicator that it was scheduled
+            # from the sync_pg_dist_node() and we don't want to override
+            # already existing task created from REST API.
+            if task.timeout is None and (i is not None or self._in_flight and self._in_flight.group == task.group):
+                return False
+
+            # Override already existing task for the same worker group
+            if i is not None:
+                if task != self._tasks[i]:
+                    logger.debug('Overriding existing task: %s != %s', self._tasks[i], task)
+                    self._tasks[i] = task
+                    self._condition.notify()
+                    return True
+            # Add the task to the list if Worker node state is different from the cached `pg_dist_node`
+            elif self._schedule_load_pg_dist_node or task != self._pg_dist_node.get(task.group)\
+                    or self._in_flight and task.group == self._in_flight.group:
+                logger.debug('Adding the new task: %s', task)
+                self._tasks.append(task)
+                self._condition.notify()
+                return True
+        return False
+
+    def add_task(self, event, group, conn_url, timeout=None, cooldown=None):
+        try:
+            r = urlparse(conn_url)
+        except Exception as e:
+            return logger.error('Failed to parse connection url %s: %r', conn_url, e)
+        host = r.hostname
+        port = r.port or 5432
+        task = PgDistNode(group, host, port, event, timeout=timeout, cooldown=cooldown)
+        return task if self._add_task(task) else None
+
+    def handle_event(self, cluster, event):
+        if not self.is_alive():
+            return
+
+        cluster = cluster.workers.get(event['group'])
+        if not (cluster and cluster.leader and cluster.leader.name == event['leader'] and cluster.leader.conn_url):
+            return
+
+        task = self.add_task(event['type'], event['group'],
+                             cluster.leader.conn_url,
+                             event['timeout'], event['cooldown']*1000)
+        if task and event['type'] == 'before_demote':
+            task.wait()
+
+    def bootstrap(self):
+        if not self.is_enabled():
+            return
+
+        conn_kwargs = self._postgresql.config.local_connect_kwargs
+        conn_kwargs['options'] = '-c synchronous_commit=local -c statement_timeout=0'
+        if self._config['database'] != self._postgresql.database:
+            conn = connect(**conn_kwargs)
+            try:
+                with conn.cursor() as cur:
+                    cur.execute('CREATE DATABASE {0}'.format(quote_ident(self._config['database'], conn)))
+            finally:
+                conn.close()
+
+        conn_kwargs['dbname'] = self._config['database']
+        conn = connect(**conn_kwargs)
+        try:
+            with conn.cursor() as cur:
+                cur.execute('CREATE EXTENSION citus')
+
+                superuser = self._postgresql.config.superuser
+                params = {k: superuser[k] for k in ('password', 'sslcert', 'sslkey') if k in superuser}
+                if params:
+                    cur.execute("INSERT INTO pg_catalog.pg_dist_authinfo VALUES"
+                                "(0, pg_catalog.current_user(), %s)",
+                                (self._postgresql.config.format_dsn(params),))
+        finally:
+            conn.close()
+
+    def adjust_postgres_gucs(self, parameters):
+        if not self.is_enabled():
+            return
+
+        # citus extension must be on the first place in shared_preload_libraries
+        shared_preload_libraries = list(filter(
+            lambda el: el and el != 'citus',
+            [p.strip() for p in parameters.get('shared_preload_libraries', '').split(',')]))
+        parameters['shared_preload_libraries'] = ','.join(['citus'] + shared_preload_libraries)
+
+        # if not explicitly set Citus overrides max_prepared_transactions to max_connections*2
+        if parameters.get('max_prepared_transactions') == 0:
+            parameters['max_prepared_transactions'] = parameters['max_connections'] * 2
+
+        # Resharding in Citus implemented using logical replication
+        parameters['wal_level'] = 'logical'
+
+    def ignore_replication_slot(self, slot):
+        if self.is_enabled() and self._postgresql.is_leader() and\
+                slot['type'] == 'logical' and slot['database'] == self._config['database']:
+            m = CITUS_SLOT_NAME_RE.match(slot['name'])
+            return m and {'move': 'pgoutput', 'split': 'citus'}.get(m.group(1)) == slot['plugin']
+        return False

patroni/postgresql/config.py

@@ -12,21 +12,14 @@ from .validator import CaseInsensitiveDict, recovery_parameters,\
         transform_postgresql_parameter_value, transform_recovery_parameter_value
 from ..dcs import slot_name_from_member_name, RemoteMember
 from ..exceptions import PatroniFatalException
-from ..psycopg import quote_ident as _quote_ident
 from ..utils import compare_values, parse_bool, parse_int, split_host_port, uri, \
         validate_directory, is_subpath
 
 logger = logging.getLogger(__name__)
 
-SYNC_STANDBY_NAME_RE = re.compile(r'^[A-Za-z_][A-Za-z_0-9\$]*$')
 PARAMETER_RE = re.compile(r'([a-z_]+)\s*=\s*')
 
 
-def quote_ident(value):
-    """Very simplified version of quote_ident"""
-    return value if SYNC_STANDBY_NAME_RE.match(value) else _quote_ident(value)
-
-
 def conninfo_uri_parse(dsn):
     ret = {}
     r = urlparse(dsn)
@@ -536,24 +529,24 @@ class ConfigHandler(object):
             recovery_params.update({'recovery_target': '', 'recovery_target_name': '', 'recovery_target_time': '',
                                     'recovery_target_xid': '', 'recovery_target_lsn': ''})
 
-        is_remote_master = isinstance(member, RemoteMember)
+        is_remote_member = isinstance(member, RemoteMember)
         primary_conninfo = self.primary_conninfo_params(member)
         if primary_conninfo:
             use_slots = self.get('use_slots', True) and self._postgresql.major_version >= 90400
-            if use_slots and not (is_remote_master and member.no_replication_slot):
-                primary_slot_name = member.primary_slot_name if is_remote_master else self._postgresql.name
+            if use_slots and not (is_remote_member and member.no_replication_slot):
+                primary_slot_name = member.primary_slot_name if is_remote_member else self._postgresql.name
                 recovery_params['primary_slot_name'] = slot_name_from_member_name(primary_slot_name)
                 # We are a standby leader and are using a replication slot. Make sure we connect to
                 # the leader of the main cluster (in case more than one host is specified in the
                 # connstr) by adding 'target_session_attrs=read-write' to primary_conninfo.
-                if is_remote_master and 'target_sesions_attrs' not in primary_conninfo and\
+                if is_remote_member and 'target_sesions_attrs' not in primary_conninfo and\
                         self._postgresql.major_version >= 100000:
                     primary_conninfo['target_session_attrs'] = 'read-write'
             recovery_params['primary_conninfo'] = primary_conninfo
 
         # standby_cluster config might have different parameters, we want to override them
         standby_cluster_params = ['restore_command', 'archive_cleanup_command']\
-            + (['recovery_min_apply_delay'] if is_remote_master else [])
+            + (['recovery_min_apply_delay'] if is_remote_member else [])
         recovery_params.update({p: member.data.get(p) for p in standby_cluster_params if member and member.data.get(p)})
         return recovery_params
 
@@ -869,6 +862,9 @@ class ConfigHandler(object):
         elif self._postgresql.major_version:
             wal_keep_size = parse_int(parameters.pop('wal_keep_size', self.CMDLINE_OPTIONS['wal_keep_size'][0]), 'MB')
             parameters.setdefault('wal_keep_segments', int((wal_keep_size + 8) / 16))
+
+        self._postgresql.citus_handler.adjust_postgres_gucs(parameters)
+
         ret = CaseInsensitiveDict({k: v for k, v in parameters.items() if not self._postgresql.major_version or
                                    self._postgresql.major_version >= self.CMDLINE_OPTIONS.get(k, (0, 1, 90100))[2]})
         ret.update({k: os.path.join(self._config_dir, ret[k]) for k in ('hba_file', 'ident_file') if k in ret})
@@ -1017,6 +1013,9 @@ class ConfigHandler(object):
         if not local_connection_address_changed:
             self.resolve_connection_addresses()
 
+        proxy_addr = config.get('proxy_address')
+        self._postgresql.proxy_url = uri('postgres', proxy_addr, self._postgresql.database) if proxy_addr else None
+
         if conf_changed:
             self.write_postgresql_conf()
 
@@ -1041,23 +1040,19 @@ class ConfigHandler(object):
         else:
             logger.info('No PostgreSQL configuration items changed, nothing to reload.')
 
-    def set_synchronous_standby(self, sync_members):
-        """Sets a node to be synchronous standby and if changed does a reload for PostgreSQL."""
-        if sync_members and sync_members != ['*']:
-            sync_members = [quote_ident(x) for x in sync_members]
-        if self._postgresql.major_version >= 90600 and len(sync_members) > 1:
-            sync_param = '{0} ({1})'.format(len(sync_members), ','.join(sync_members))
-        else:
-            sync_param = next(iter(sync_members), None)
-        if sync_param != self._synchronous_standby_names:
-            if sync_param is None:
+    def set_synchronous_standby_names(self, value):
+        """Updates synchronous_standby_names and reloads if necessary.
+        :returns: True if value was updated."""
+        if value != self._synchronous_standby_names:
+            if value is None:
                 self._server_parameters.pop('synchronous_standby_names', None)
             else:
-                self._server_parameters['synchronous_standby_names'] = sync_param
-            self._synchronous_standby_names = sync_param
+                self._server_parameters['synchronous_standby_names'] = value
+            self._synchronous_standby_names = value
             if self._postgresql.state == 'running':
                 self.write_postgresql_conf()
                 self._postgresql.reload()
+            return True
 
     @property
     def effective_configuration(self):
@@ -1070,7 +1065,7 @@ class ConfigHandler(object):
         As a workaround we will start it with the values from controldata and set `pending_restart`
         to true as an indicator that current values of parameters are not matching expectations."""
 
-        if self._postgresql.role == 'master':
+        if self._postgresql.role in ('master', 'primary'):
             return self._server_parameters
 
         options_mapping = {

patroni/postgresql/connection.py

@@ -22,7 +22,6 @@ class Connection(object):
         with self._lock:
             if not self._connection or self._connection.closed != 0:
                 self._connection = psycopg.connect(**self._conn_kwargs)
-                self._connection.autocommit = True
                 self.server_version = self._connection.server_version
         return self._connection
 
@@ -42,7 +41,6 @@ class Connection(object):
 @contextmanager
 def get_connection_cursor(**kwargs):
     conn = psycopg.connect(**kwargs)
-    conn.autocommit = True
     with conn.cursor() as cur:
         yield cur
     conn.close()

patroni/postgresql/misc.py

@@ -1,4 +1,6 @@
+import errno
 import logging
+import os
 
 from patroni.exceptions import PostgresException
 
@@ -73,3 +75,16 @@ def parse_history(data):
 def format_lsn(lsn, full=False):
     template = '{0:X}/{1:08X}' if full else '{0:X}/{1:X}'
     return template.format(lsn >> 32, lsn & 0xFFFFFFFF)
+
+
+def fsync_dir(path):
+    if os.name != 'nt':
+        fd = os.open(path, os.O_DIRECTORY)
+        try:
+            os.fsync(fd)
+        except OSError as e:
+            # Some filesystems don't like fsyncing directories and raise EINVAL. Ignoring it is usually safe.
+            if e.errno != errno.EINVAL:
+                raise
+        finally:
+            os.close(fd)

patroni/postgresql/postmaster.py

@@ -171,8 +171,8 @@ class PostmasterProcess(psutil.Process):
         else:
             return not self.is_running()
 
-    def wait_for_user_backends_to_close(self):
-        # These regexps are cross checked against versions PostgreSQL 9.1 .. 11
+    def wait_for_user_backends_to_close(self, stop_timeout):
+        # These regexps are cross checked against versions PostgreSQL 9.1 .. 15
         aux_proc_re = re.compile("(?:postgres:)( .*:)? (?:(?:archiver|startup|autovacuum launcher|autovacuum worker|"
                                  "checkpointer|logger|stats collector|wal receiver|wal writer|writer)(?: process  )?|"
                                  "walreceiver|wal sender process|walsender|walwriter|background writer|"
@@ -184,19 +184,23 @@ class PostmasterProcess(psutil.Process):
             return logger.debug('Failed to get list of postmaster children')
 
         user_backends = []
-        user_backends_cmdlines = []
+        user_backends_cmdlines = {}
         for child in children:
             try:
                 cmdline = child.cmdline()
                 if cmdline and not aux_proc_re.match(cmdline[0]):
                     user_backends.append(child)
-                    user_backends_cmdlines.append(cmdline[0])
+                    user_backends_cmdlines[child.pid] = cmdline[0]
             except psutil.NoSuchProcess:
                 pass
         if user_backends:
-            logger.debug('Waiting for user backends %s to close', ', '.join(user_backends_cmdlines))
-            psutil.wait_procs(user_backends)
-        logger.debug("Backends closed")
+            logger.debug('Waiting for user backends %s to close', ', '.join(user_backends_cmdlines.values()))
+            gone, live = psutil.wait_procs(user_backends, stop_timeout)
+            if stop_timeout and live:
+                live = [user_backends_cmdlines[b.pid] for b in live]
+                logger.warning('Backends still alive after %s: %s', stop_timeout, ', '.join(live))
+            else:
+                logger.debug("Backends closed")
 
     @staticmethod
     def start(pgcommand, data_dir, conf, options):

patroni/postgresql/rewind.py

@@ -1,13 +1,15 @@
 import logging
 import os
+import re
 import shlex
+import shutil
 import six
 import subprocess
 
 from threading import Lock, Thread
 
 from .connection import get_connection_cursor
-from .misc import format_lsn, parse_history, parse_lsn
+from .misc import format_lsn, fsync_dir, parse_history, parse_lsn
 from ..async_executor import CriticalTask
 from ..dcs import Leader
 
@@ -124,7 +126,7 @@ class Rewind(object):
                 in_recovery = True
                 lsn = data.get('Minimum recovery ending location')
                 timeline = int(data.get("Min recovery ending loc's timeline"))
-                if lsn == '0/0' or timeline == 0:  # it was a master when it crashed
+                if lsn == '0/0' or timeline == 0:  # it was a primary when it crashed
                     data['Database cluster state'] = 'shut down'
             if data.get('Database cluster state') == 'shut down':
                 in_recovery = False
@@ -155,7 +157,7 @@ class Rewind(object):
         return in_recovery, timeline, lsn
 
     @staticmethod
-    def _log_master_history(history, i):
+    def _log_primary_history(history, i):
         start = max(0, i - 3)
         end = None if i + 4 >= len(history) else i + 2
         history_show = []
@@ -170,7 +172,7 @@ class Rewind(object):
             history_show.append('...')
             history_show.append(format_history_line(history[-1]))
 
-        logger.info('master: history=%s', '\n'.join(history_show))
+        logger.info('primary: history=%s', '\n'.join(history_show))
 
     def _conn_kwargs(self, member, auth):
         ret = member.conn_kwargs(auth)
@@ -187,7 +189,7 @@ class Rewind(object):
         if local_timeline is None or local_lsn is None:
             return
 
-        if isinstance(leader, Leader) and leader.member.data.get('role') != 'master':
+        if isinstance(leader, Leader) and leader.member.data.get('role') not in ('master', 'primary'):
             return
 
         # We want to use replication credentials when connecting to the "postgres" database in case if
@@ -204,20 +206,20 @@ class Rewind(object):
         try:
             with self._postgresql.get_replication_connection_cursor(**leader.conn_kwargs()) as cur:
                 cur.execute('IDENTIFY_SYSTEM')
-                master_timeline = cur.fetchone()[1]
-                logger.info('master_timeline=%s', master_timeline)
-                if local_timeline > master_timeline:  # Not always supported by pg_rewind
+                primary_timeline = cur.fetchone()[1]
+                logger.info('primary_timeline=%s', primary_timeline)
+                if local_timeline > primary_timeline:  # Not always supported by pg_rewind
                     need_rewind = True
-                elif local_timeline == master_timeline:
+                elif local_timeline == primary_timeline:
                     need_rewind = False
-                elif master_timeline > 1:
-                    cur.execute('TIMELINE_HISTORY {0}'.format(master_timeline))
+                elif primary_timeline > 1:
+                    cur.execute('TIMELINE_HISTORY {0}'.format(primary_timeline))
                     history = cur.fetchone()[1]
                     if not isinstance(history, six.string_types):
                         history = bytes(history).decode('utf-8')
-                    logger.debug('master: history=%s', history)
+                    logger.debug('primary: history=%s', history)
         except Exception:
-            return logger.exception('Exception when working with master via replication connection')
+            return logger.exception('Exception when working with primary via replication connection')
 
         if history is not None:
             history = list(parse_history(history))
@@ -238,7 +240,7 @@ class Rewind(object):
                     break
             else:
                 need_rewind = True
-            self._log_master_history(history, i)
+            self._log_primary_history(history, i)
 
         self._state = need_rewind and REWIND_STATUS.NEED or REWIND_STATUS.NOT_NEED
 
@@ -268,7 +270,7 @@ class Rewind(object):
                         if self._checkpoint_task.result is not None:
                             self._state = REWIND_STATUS.CHECKPOINT
                             self._checkpoint_task = None
-                elif self._postgresql.get_master_timeline() == self._postgresql.pg_control_timeline():
+                elif self._postgresql.get_primary_timeline() == self._postgresql.pg_control_timeline():
                     self._state = REWIND_STATUS.CHECKPOINT
                 else:
                     self._checkpoint_task = CriticalTask()
@@ -277,28 +279,37 @@ class Rewind(object):
     def checkpoint_after_promote(self):
         return self._state == REWIND_STATUS.CHECKPOINT
 
-    def _fetch_missing_wal(self, restore_command, wal_filename):
+    def _buid_archiver_command(self, command, wal_filename):
+        """Replace placeholders in the given archiver command's template.
+        Applicable for archive_command and restore_command.
+        Can also be used for archive_cleanup_command and recovery_end_command,
+        however %r value is always set to 000000010000000000000001."""
         cmd = ''
-        length = len(restore_command)
+        length = len(command)
         i = 0
         while i < length:
-            if restore_command[i] == '%' and i + 1 < length:
+            if command[i] == '%' and i + 1 < length:
                 i += 1
-                if restore_command[i] == 'p':
+                if command[i] == 'p':
                     cmd += os.path.join(self._postgresql.wal_dir, wal_filename)
-                elif restore_command[i] == 'f':
+                elif command[i] == 'f':
                     cmd += wal_filename
-                elif restore_command[i] == 'r':
+                elif command[i] == 'r':
                     cmd += '000000010000000000000001'
-                elif restore_command[i] == '%':
+                elif command[i] == '%':
                     cmd += '%'
                 else:
                     cmd += '%'
                     i -= 1
             else:
-                cmd += restore_command[i]
+                cmd += command[i]
             i += 1
 
+        return cmd
+
+    def _fetch_missing_wal(self, restore_command, wal_filename):
+        cmd = self._buid_archiver_command(restore_command, wal_filename)
+
         logger.info('Trying to fetch the missing wal: %s', cmd)
         return self._postgresql.cancellable.call(shlex.split(cmd)) == 0
 
@@ -315,6 +326,54 @@ class Rewind(object):
                     if waldir.endswith('/pg_' + self._postgresql.wal_name) and len(wal_filename) == 24:
                         return wal_filename
 
+    def _archive_ready_wals(self):
+        """Try to archive WALs that have .ready files just in case
+        archive_mode was not set to 'always' before promote, while
+        after it the WALs were recycled on the promoted replica.
+        With this we prevent the entire loss of such WALs and the
+        consequent old leader's start failure."""
+        archive_mode = self._postgresql.get_guc_value('archive_mode')
+        archive_cmd = self._postgresql.get_guc_value('archive_command')
+        if archive_mode not in ('on', 'always') or not archive_cmd:
+            return
+
+        walseg_regex = re.compile(r'^[0-9A-F]{24}(\.partial){0,1}\.ready$')
+        status_dir = os.path.join(self._postgresql.wal_dir, 'archive_status')
+        try:
+            wals_to_archive = [f[:-6] for f in os.listdir(status_dir) if walseg_regex.match(f)]
+        except OSError as e:
+            return logger.error('Unable to list %s: %r', status_dir, e)
+
+        # skip fsync, as postgres --single or pg_rewind will anyway run it
+        for wal in sorted(wals_to_archive):
+            old_name = os.path.join(status_dir, wal + '.ready')
+            # wal file might have alredy been archived
+            if os.path.isfile(old_name) and os.path.isfile(os.path.join(self._postgresql.wal_dir, wal)):
+                cmd = self._buid_archiver_command(archive_cmd, wal)
+                # it is the author of archive_command, who is responsible
+                # for not overriding the WALs already present in archive
+                logger.info('Trying to archive %s: %s', wal, cmd)
+                if self._postgresql.cancellable.call(shlex.split(cmd)) == 0:
+                    new_name = os.path.join(status_dir, wal + '.done')
+                    try:
+                        shutil.move(old_name, new_name)
+                    except Exception as e:
+                        logger.error('Unable to rename %s to %s: %r', old_name, new_name, e)
+                else:
+                    logger.info('Failed to archive WAL segment %s', wal)
+
+    def _maybe_clean_pg_replslot(self):
+        """Clean pg_replslot directory if pg version is less then 11
+        (pg_rewind deletes $PGDATA/pg_replslot content only since pg11)."""
+        if self._postgresql.major_version < 110000:
+            replslot_dir = self._postgresql.slots_handler.pg_replslot_dir
+            try:
+                for f in os.listdir(replslot_dir):
+                    shutil.rmtree(os.path.join(replslot_dir, f))
+                fsync_dir(replslot_dir)
+            except Exception as e:
+                logger.warning('Unable to clean %s: %r', replslot_dir, e)
+
     def pg_rewind(self, r):
         # prepare pg_rewind connection
         env = self._postgresql.config.write_pgpass(r)
@@ -367,15 +426,17 @@ class Rewind(object):
         if self._postgresql.is_running() and not self._postgresql.stop(checkpoint=False):
             return logger.warning('Can not run pg_rewind because postgres is still running')
 
+        self._archive_ready_wals()
+
         # prepare pg_rewind connection
         r = self._conn_kwargs(leader, self._postgresql.config.rewind_credentials)
 
-        # 1. make sure that we are really trying to rewind from the master
+        # 1. make sure that we are really trying to rewind from the primary
         # 2. make sure that pg_control contains the new timeline by:
         #   running a checkpoint or
-        #   waiting until Patroni on the master will expose checkpoint_after_promote=True
+        #   waiting until Patroni on the primary will expose checkpoint_after_promote=True
         checkpoint_status = leader.checkpoint_after_promote if isinstance(leader, Leader) else None
-        if checkpoint_status is None:  # we are the standby-cluster leader or master still runs the old Patroni
+        if checkpoint_status is None:  # we are the standby-cluster leader or primary still runs the old Patroni
             # superuser credentials match rewind_credentials if the latter are not provided or we run 10 or older
             if self._postgresql.config.superuser == self._postgresql.config.rewind_credentials:
                 leader_status = self._postgresql.checkpoint(
@@ -390,14 +451,15 @@ class Rewind(object):
             return
 
         if self.pg_rewind(r):
+            self._maybe_clean_pg_replslot()
             self._state = REWIND_STATUS.SUCCESS
         else:
             if not self.check_leader_is_not_in_recovery(r):
-                logger.warning('Failed to rewind because master %s become unreachable', leader.name)
+                logger.warning('Failed to rewind because primary %s become unreachable', leader.name)
                 if not self.can_rewind:  # It is possible that the previous attempt damaged pg_control file!
                     self._state = REWIND_STATUS.FAILED
             else:
-                logger.error('Failed to rewind from healty master: %s', leader.name)
+                logger.error('Failed to rewind from healty primary: %s', leader.name)
                 self._state = REWIND_STATUS.FAILED
 
             if self.failed:
@@ -465,6 +527,7 @@ class Rewind(object):
             logger.exception('Unable to list %s', status_dir)
 
     def ensure_clean_shutdown(self):
+        self._archive_ready_wals()
         self.cleanup_archive_status()
 
         # Start in a single user mode and stop to produce a clean shutdown

patroni/postgresql/slots.py

@@ -1,13 +1,13 @@
-import errno
 import logging
 import os
 import shutil
 
 from collections import defaultdict
 from contextlib import contextmanager
+from threading import Condition, Thread
 
 from .connection import get_connection_cursor
-from .misc import format_lsn
+from .misc import format_lsn, fsync_dir
 from ..psycopg import OperationalError
 
 logger = logging.getLogger(__name__)
@@ -18,25 +18,97 @@ def compare_slots(s1, s2, dbid='database'):
                                          s1.get(dbid) == s2.get(dbid) and s1['plugin'] == s2['plugin'])
 
 
-def fsync_dir(path):
-    if os.name != 'nt':
-        fd = os.open(path, os.O_DIRECTORY)
+class SlotsAdvanceThread(Thread):
+
+    def __init__(self, slots_handler):
+        super(SlotsAdvanceThread, self).__init__()
+        self.daemon = True
+        self._slots_handler = slots_handler
+
+        # _copy_slots and _failed are used to asynchronously give some feedback to the main thread
+        self._copy_slots = []
+        self._failed = False
+
+        self._scheduled = defaultdict(dict)  # {'dbname1': {'slot1': 100, 'slot2': 100}, 'dbname2': {'slot3': 100}}
+        self._condition = Condition()  # protect self._scheduled from concurrent access and to wakeup the run() method
+
+        self.start()
+
+    def sync_slot(self, cur, database, slot, lsn):
+        failed = copy = False
         try:
-            os.fsync(fd)
-        except OSError as e:
-            # Some filesystems don't like fsyncing directories and raise EINVAL. Ignoring it is usually safe.
-            if e.errno != errno.EINVAL:
-                raise
-        finally:
-            os.close(fd)
+            cur.execute("SELECT pg_catalog.pg_replication_slot_advance(%s, %s)", (slot, format_lsn(lsn)))
+        except Exception as e:
+            logger.error("Failed to advance logical replication slot '%s': %r", slot, e)
+            failed = True
+            copy = isinstance(e, OperationalError) and e.diag.sqlstate == '58P01'  # WAL file is gone
+        with self._condition:
+            if self._scheduled and failed:
+                if copy and slot not in self._copy_slots:
+                    self._copy_slots.append(slot)
+                self._failed = True
+
+            new_lsn = self._scheduled.get(database, {}).get(slot, 0)
+            # remove slot from the self._scheduled structure only if it wasn't changed
+            if new_lsn == lsn and database in self._scheduled:
+                self._scheduled[database].pop(slot)
+                if not self._scheduled[database]:
+                    self._scheduled.pop(database)
+
+    def sync_slots_in_database(self, database, slots):
+        with self._slots_handler.get_local_connection_cursor(dbname=database, options='-c statement_timeout=0') as cur:
+            for slot in slots:
+                with self._condition:
+                    lsn = self._scheduled.get(database, {}).get(slot, 0)
+                if lsn:
+                    self.sync_slot(cur, database, slot, lsn)
+
+    def sync_slots(self):
+        with self._condition:
+            databases = list(self._scheduled.keys())
+        for database in databases:
+            with self._condition:
+                slots = list(self._scheduled.get(database, {}).keys())
+            if slots:
+                try:
+                    self.sync_slots_in_database(database, slots)
+                except Exception as e:
+                    logger.error('Failed to advance replication slots in database %s: %r', database, e)
+
+    def run(self):
+        while True:
+            with self._condition:
+                if not self._scheduled:
+                    self._condition.wait()
+
+            self.sync_slots()
+
+    def schedule(self, advance_slots):
+        with self._condition:
+            for database, values in advance_slots.items():
+                self._scheduled[database].update(values)
+            ret = (self._failed, self._copy_slots)
+            self._copy_slots = []
+            self._failed = False
+            self._condition.notify()
+
+        return ret
+
+    def on_promote(self):
+        with self._condition:
+            self._scheduled.clear()
+            self._failed = False
+            self._copy_slots = []
 
 
 class SlotsHandler(object):
 
     def __init__(self, postgresql):
         self._postgresql = postgresql
+        self._advance = None
         self._replication_slots = {}  # already existing replication slots
         self._unready_logical_slots = {}
+        self.pg_replslot_dir = os.path.join(self._postgresql.data_dir, 'pg_replslot')
         self.schedule()
 
     def _query(self, sql, *params):
@@ -104,26 +176,36 @@ class SlotsHandler(object):
             if ((matcher.get("name") is None or matcher["name"] == name)
                and all(not matcher.get(a) or matcher[a] == slot.get(a) for a in ('database', 'plugin', 'type'))):
                 return True
-        return False
+        return self._postgresql.citus_handler.ignore_replication_slot(slot)
 
     def drop_replication_slot(self, name):
-        cursor = self._query(('SELECT pg_catalog.pg_drop_replication_slot(%s) WHERE EXISTS (SELECT 1 ' +
-                              'FROM pg_catalog.pg_replication_slots WHERE slot_name = %s AND NOT active)'), name, name)
-        # In normal situation rowcount should be 1, otherwise either slot doesn't exists or it is still active
-        return cursor.rowcount == 1
+        """Returns a tuple(active, dropped)"""
+        cursor = self._query(('WITH slots AS (SELECT slot_name, active' +
+                              ' FROM pg_catalog.pg_replication_slots WHERE slot_name = %s),' +
+                              ' dropped AS (SELECT pg_catalog.pg_drop_replication_slot(slot_name),' +
+                              ' true AS dropped FROM slots WHERE not active) ' +
+                              'SELECT active, COALESCE(dropped, false) FROM slots' +
+                              ' FULL OUTER JOIN dropped ON true'), name)
+        return cursor.fetchone() if cursor.rowcount == 1 else (False, False)
 
-    def _drop_incorrect_slots(self, cluster, slots):
+    def _drop_incorrect_slots(self, cluster, slots, paused):
         # drop old replication slots which are not presented in desired slots
         for name in set(self._replication_slots) - set(slots):
-            if not self.ignore_replication_slot(cluster, name) and not self.drop_replication_slot(name):
-                logger.error("Failed to drop replication slot '%s'", name)
-                self._schedule_load_slots = True
-
+            if not paused and not self.ignore_replication_slot(cluster, name):
+                active, dropped = self.drop_replication_slot(name)
+                if dropped:
+                    logger.info("Dropped unknown replication slot '%s'", name)
+                else:
+                    self._schedule_load_slots = True
+                    if active:
+                        logger.debug("Unable to drop unknown replication slot '%s', slot is still active", name)
+                    else:
+                        logger.error("Failed to drop replication slot '%s'", name)
         for name, value in slots.items():
             if name in self._replication_slots and not compare_slots(value, self._replication_slots[name]):
                 logger.info("Trying to drop replication slot '%s' because value is changing from %s to %s",
                             name, self._replication_slots[name], value)
-                if self.drop_replication_slot(name):
+                if self.drop_replication_slot(name) == (False, True):
                     self._replication_slots.pop(name)
                 else:
                     logger.error("Failed to drop replication slot '%s'", name)
@@ -143,7 +225,7 @@ class SlotsHandler(object):
                 self._schedule_load_slots = True
 
     @contextmanager
-    def _get_local_connection_cursor(self, **kwargs):
+    def get_local_connection_cursor(self, **kwargs):
         conn_kwargs = self._postgresql.config.local_connect_kwargs
         conn_kwargs.update(kwargs)
         with get_connection_cursor(**conn_kwargs) as cur:
@@ -162,7 +244,7 @@ class SlotsHandler(object):
 
         # Create new logical slots
         for database, values in logical_slots.items():
-            with self._get_local_connection_cursor(dbname=database) as cur:
+            with self.get_local_connection_cursor(dbname=database) as cur:
                 for name, value in values.items():
                     try:
                         cur.execute("SELECT pg_catalog.pg_create_logical_replication_slot(%s, %s)" +
@@ -175,6 +257,11 @@ class SlotsHandler(object):
                         slots.pop(name)
                     self._schedule_load_slots = True
 
+    def schedule_advance_slots(self, slots):
+        if not self._advance:
+            self._advance = SlotsAdvanceThread(self)
+        return self._advance.schedule(slots)
+
     def _ensure_logical_slots_replica(self, cluster, slots):
         advance_slots = defaultdict(dict)  # Group logical slots to be advanced by database name
         create_slots = []  # And collect logical slots to be created on the replica
@@ -186,27 +273,18 @@ class SlotsHandler(object):
                     if name in cluster.slots:
                         try:  # Skip slots that doesn't need to be advanced
                             if value['confirmed_flush_lsn'] < int(cluster.slots[name]):
-                                advance_slots[value['database']][name] = value
+                                advance_slots[value['database']][name] = int(cluster.slots[name])
                         except Exception as e:
                             logger.error('Failed to parse "%s": %r', cluster.slots[name], e)
                 elif name in cluster.slots:  # We want to copy only slots with feedback in a DCS
                     create_slots.append(name)
 
-        # Advance logical slots
-        for database, values in advance_slots.items():
-            with self._get_local_connection_cursor(dbname=database, options='-c statement_timeout=0') as cur:
-                for name, value in values.items():
-                    try:
-                        cur.execute("SELECT pg_catalog.pg_replication_slot_advance(%s, %s)",
-                                    (name, format_lsn(int(cluster.slots[name]))))
-                    except Exception as e:
-                        logger.error("Failed to advance logical replication slot '%s': %r", name, e)
-                        if isinstance(e, OperationalError) and e.diag.sqlstate == '58P01':  # WAL file is gone
-                            create_slots.append(name)
-                        self._schedule_load_slots = True
-        return create_slots
+        error, copy_slots = self.schedule_advance_slots(advance_slots)
+        if error:
+            self._schedule_load_slots = True
+        return create_slots + copy_slots
 
-    def sync_replication_slots(self, cluster, nofailover, replicatefrom=None):
+    def sync_replication_slots(self, cluster, nofailover, replicatefrom=None, paused=False):
         ret = None
         if self._postgresql.major_version >= 90400 and cluster.config:
             try:
@@ -215,7 +293,7 @@ class SlotsHandler(object):
                 slots = cluster.get_replication_slots(self._postgresql.name, self._postgresql.role,
                                                       nofailover, self._postgresql.major_version, True)
 
-                self._drop_incorrect_slots(cluster, slots)
+                self._drop_incorrect_slots(cluster, slots, paused)
 
                 self._ensure_physical_slots(slots)
 
@@ -264,10 +342,11 @@ class SlotsHandler(object):
                 try:
                     cur = self._query("SELECT pg_catalog.current_setting('hot_standby_feedback')::boolean")
                     if not cur.fetchone()[0]:
-                        return logger.error('Logical slot failover requires "hot_standby_feedback".'
-                                            ' Please check postgresql.auto.conf')
+                        logger.error('Logical slot failover requires "hot_standby_feedback".'
+                                     ' Please check postgresql.auto.conf')
                 except Exception as e:
-                    return logger.error('Failed to check the hot_standby_feedback setting: %r', e)
+                    logger.error('Failed to check the hot_standby_feedback setting: %r', e)
+                return  # since `catalog_xmin` isn't valid further checks don't make any sense
 
         for name in list(self._unready_logical_slots):
             value = self._replication_slots.get(name)
@@ -305,9 +384,8 @@ class SlotsHandler(object):
                 logger.error("Failed to copy logical slots from the %s via postgresql connection: %r", leader.name, e)
 
         if isinstance(create_slots, dict) and create_slots and self._postgresql.stop():
-            pg_replslot_dir = os.path.join(self._postgresql.data_dir, 'pg_replslot')
             for name, value in create_slots.items():
-                slot_dir = os.path.join(pg_replslot_dir, name)
+                slot_dir = os.path.join(self._postgresql.slots_handler.pg_replslot_dir, name)
                 slot_tmp_dir = slot_dir + '.tmp'
                 if os.path.exists(slot_tmp_dir):
                     shutil.rmtree(slot_tmp_dir)
@@ -322,7 +400,7 @@ class SlotsHandler(object):
                 os.rename(slot_tmp_dir, slot_dir)
                 fsync_dir(slot_dir)
                 self._unready_logical_slots[name] = None
-            fsync_dir(pg_replslot_dir)
+            fsync_dir(self._postgresql.slots_handler.pg_replslot_dir)
             self._postgresql.start()
 
     def schedule(self, value=None):
@@ -331,6 +409,9 @@ class SlotsHandler(object):
         self._schedule_load_slots = self._force_readiness_check = value
 
     def on_promote(self):
+        if self._advance:
+            self._advance.on_promote()
+
         if self._unready_logical_slots:
             logger.warning('Logical replication slots that might be unsafe to use after promote: %s',
                            set(self._unready_logical_slots))

patroni/postgresql/sync.py

@@ -0,0 +1,258 @@
+import logging
+import re
+import time
+
+from copy import deepcopy
+
+from .validator import CaseInsensitiveDict
+from ..psycopg import quote_ident as _quote_ident
+
+logger = logging.getLogger(__name__)
+
+SYNC_STANDBY_NAME_RE = re.compile(r'^[A-Za-z_][A-Za-z_0-9\$]*$')
+SYNC_REP_PARSER_RE = re.compile(r"""
+           (?P<first> [fF][iI][rR][sS][tT] )
+         | (?P<any> [aA][nN][yY] )
+         | (?P<space> \s+ )
+         | (?P<ident> [A-Za-z_][A-Za-z_0-9\$]* )
+         | (?P<dquot> " (?: [^"]+ | "" )* " )
+         | (?P<star> [*] )
+         | (?P<num> \d+ )
+         | (?P<comma> , )
+         | (?P<parenstart> \( )
+         | (?P<parenend> \) )
+         | (?P<JUNK> . )
+        """, re.X)
+_EMPTY_SSN = {'type': 'off', 'num': 0, 'members': CaseInsensitiveDict({})}
+
+
+def quote_ident(value):
+    """Very simplified version of quote_ident"""
+    return value if SYNC_STANDBY_NAME_RE.match(value) else _quote_ident(value)
+
+
+def parse_sync_standby_names(value):
+    """Parse postgresql synchronous_standby_names to constituent parts.
+    Returns dict with the following keys:
+    * type: 'quorum'|'priority'
+    * num: int
+    * members: CaseInsensitiveDict, with names as keys
+    * has_star: bool - Present if true
+    If the configuration value can not be parsed, raises a ValueError.
+
+    >>> parse_sync_standby_names('')['type']
+    'off'
+
+    >>> parse_sync_standby_names('FiRsT')['type']
+    'priority'
+
+    >>> parse_sync_standby_names('FiRsT')['members']
+    {'FiRsT': True}
+
+    >>> parse_sync_standby_names('"1"')['members']
+    {'1': True}
+
+    >>> parse_sync_standby_names(' a , b ')['members']
+    {'a': True, 'b': True}
+
+    >>> parse_sync_standby_names(' a , b ')['num']
+    1
+
+    >>> parse_sync_standby_names('ANY 4("a",*,b)')['has_star']
+    True
+
+    >>> parse_sync_standby_names('ANY 4("a",*,b)')['num']
+    4
+
+    >>> parse_sync_standby_names('1')  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValueError: Unparseable synchronous_standby_names value
+
+    >>> parse_sync_standby_names('a,')  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValueError: Unparseable synchronous_standby_names value
+
+    >>> parse_sync_standby_names('ANY 4("a" b,"c c")')  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValueError: Unparseable synchronous_standby_names value
+
+    >>> parse_sync_standby_names('FIRST 4("a",)')  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValueError: Unparseable synchronous_standby_names value
+
+    >>> parse_sync_standby_names('2 (,)')  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValueError: Unparseable synchronous_standby_names value
+    """
+    tokens = [(m.lastgroup, m.group(0), m.start())
+              for m in SYNC_REP_PARSER_RE.finditer(value)
+              if m.lastgroup != 'space']
+    if not tokens:
+        return deepcopy(_EMPTY_SSN)
+
+    if [t[0] for t in tokens[0:3]] == ['any', 'num', 'parenstart'] and tokens[-1][0] == 'parenend':
+        result = {'type': 'quorum', 'num': int(tokens[1][1])}
+        synclist = tokens[3:-1]
+    elif [t[0] for t in tokens[0:3]] == ['first', 'num', 'parenstart'] and tokens[-1][0] == 'parenend':
+        result = {'type': 'priority', 'num': int(tokens[1][1])}
+        synclist = tokens[3:-1]
+    elif [t[0] for t in tokens[0:2]] == ['num', 'parenstart'] and tokens[-1][0] == 'parenend':
+        result = {'type': 'priority', 'num': int(tokens[0][1])}
+        synclist = tokens[2:-1]
+    else:
+        result = {'type': 'priority', 'num': 1}
+        synclist = tokens
+    result['members'] = CaseInsensitiveDict({})
+    for i, (a_type, a_value, a_pos) in enumerate(synclist):
+        if i % 2 == 1:  # odd elements are supposed to be commas
+            if len(synclist) == i + 1:  # except the last token
+                raise ValueError("Unparseable synchronous_standby_names value %r: Unexpected token %s %r at %d" %
+                                 (value, a_type, a_value, a_pos))
+            elif a_type != 'comma':
+                raise ValueError("Unparseable synchronous_standby_names value %r: ""Got token %s %r while"
+                                 " expecting comma at %d" % (value, a_type, a_value, a_pos))
+        elif a_type in {'ident', 'first', 'any'}:
+            result['members'][a_value] = True
+        elif a_type == 'star':
+            result['members'][a_value] = True
+            result['has_star'] = True
+        elif a_type == 'dquot':
+            result['members'][a_value[1:-1].replace('""', '"')] = True
+        else:
+            raise ValueError("Unparseable synchronous_standby_names value %r: Unexpected token %s %r at %d" %
+                             (value, a_type, a_value, a_pos))
+    return result
+
+
+class SyncHandler(object):
+    """Class responsible for working with the `synchronous_standby_names`.
+
+    Sync standbys are chosen based on their state in `pg_stat_replication`.
+    When `synchronous_standby_names` is changed we memorize the `_primary_flush_lsn`
+    and the `current_state()` method will count newly added names as "sync" only when
+    they reached memorized LSN and also reported as "sync" by `pg_stat_replication`"""
+
+    def __init__(self, postgresql):
+        self._postgresql = postgresql
+        self._synchronous_standby_names = ''  # last known value of synchronous_standby_names
+        self._ssn_data = deepcopy(_EMPTY_SSN)
+        self._primary_flush_lsn = 0
+        # "sync" replication connections, that were verified to reach self._primary_flush_lsn at some point
+        self._ready_replicas = CaseInsensitiveDict({})  # keys: member names, values: connection pids
+
+    def _handle_synchronous_standby_names_change(self):
+        """If synchronous_standby_names has changed we need to check that newly added replicas
+        have reached self._primary_flush_lsn. Only after that they could be counted as sync."""
+        synchronous_standby_names = self._postgresql.synchronous_standby_names()
+        if synchronous_standby_names == self._synchronous_standby_names:
+            return False
+
+        self._synchronous_standby_names = synchronous_standby_names
+        try:
+            self._ssn_data = parse_sync_standby_names(synchronous_standby_names)
+        except ValueError as e:
+            logger.warning('%s', e)
+            self._ssn_data = deepcopy(_EMPTY_SSN)
+
+        # Invalidate cache of "sync" connections
+        for app_name in list(self._ready_replicas.keys()):
+            if app_name not in self._ssn_data['members']:
+                del self._ready_replicas[app_name]
+
+        # Newly connected replicas will be counted as sync only when reached self._primary_flush_lsn
+        self._primary_flush_lsn = self._postgresql.last_operation()
+        self._postgresql.query('SELECT pg_catalog.txid_current()')  # Ensure some WAL traffic to move replication
+        self._postgresql.reset_cluster_info_state(None)  # Reset internal cache to query fresh values
+
+    def current_state(self, cluster, sync_node_count=1, sync_node_maxlag=-1):
+        """Finds best candidates to be the synchronous standbys.
+
+        Current synchronous standby is always preferred, unless it has disconnected or does not want to be a
+        synchronous standby any longer.
+        Parameter sync_node_maxlag(maximum_lag_on_syncnode) would help swapping unhealthy sync replica in case
+        if it stops responding (or hung). Please set the value high enough so it won't unncessarily swap sync
+        standbys during high loads. Any less or equal of 0 value keep the behavior backward compatible and
+        will not swap. Please note that it will not also swap sync standbys in case where all replicas are hung.
+
+        :returns: tuple of candidates list and synchronous standby list."""
+
+        self._handle_synchronous_standby_names_change()
+
+        # Pick candidates based on who has higher replay/remote_write/flush lsn.
+        sort_col = {
+            'remote_apply': 'replay',
+            'remote_write': 'write'
+        }.get(self._postgresql.synchronous_commit(), 'flush') + '_lsn'
+
+        pg_stat_replication = [(r['pid'], r['application_name'], r['sync_state'], r[sort_col])
+                               for r in self._postgresql.pg_stat_replication()
+                               if r[sort_col] is not None]
+
+        members = CaseInsensitiveDict({m.name: m for m in cluster.members})
+        replica_list = []
+        # pg_stat_replication.sync_state has 4 possible states - async, potential, quorum, sync.
+        # That is, alphabetically they are in the reversed order of priority.
+        # Since we are doing reversed sort on (sync_state, lsn) tuples, it helps to keep the result
+        # consistent in case if a synchronous standby member is slowed down OR async node receiving
+        # changes faster than the sync member (very rare but possible).
+        # Such cases would trigger sync standby member swapping, but only if lag on a sync node exceeding a threshold.
+        for pid, app_name, sync_state, replica_lsn in sorted(pg_stat_replication, key=lambda r: r[2:4], reverse=True):
+            member = members.get(app_name)
+            if member and member.is_running and not member.tags.get('nosync', False):
+                replica_list.append((pid, member.name, sync_state, replica_lsn, bool(member.nofailover)))
+
+        max_lsn = max(replica_list, key=lambda x: x[3])[3]\
+            if len(replica_list) > 1 else self._postgresql.last_operation()
+
+        if self._postgresql.major_version < 90600:
+            sync_node_count = 1
+
+        candidates = []
+        sync_nodes = []
+        # Prefer members without nofailover tag. We are relying on the fact that sorts are guaranteed to be stable.
+        for pid, app_name, sync_state, replica_lsn, _ in sorted(replica_list, key=lambda x: x[4]):
+            # if standby name is listed in the /sync key we can count it as synchronous, otherwice
+            # ig becomes really synchronous when sync_state = 'sync' and it is known that it managed to catch up
+            if app_name not in self._ready_replicas and app_name in self._ssn_data['members'] and\
+                    (cluster.sync and app_name in cluster.sync.members or
+                     sync_state == 'sync' and replica_lsn >= self._primary_flush_lsn):
+                self._ready_replicas[app_name] = pid
+
+            if sync_node_maxlag <= 0 or max_lsn - replica_lsn <= sync_node_maxlag:
+                candidates.append(app_name)
+                if sync_state == 'sync' and app_name in self._ready_replicas:
+                    sync_nodes.append(app_name)
+            if len(candidates) >= sync_node_count:
+                break
+
+        return candidates, sync_nodes
+
+    def set_synchronous_standby_names(self, value):
+        """Constructs and sets `synchronous_standby_names` value.
+
+        :param value: list[str] - the list of wanted sync members"""
+        if value and value != ['*']:
+            value = [quote_ident(x) for x in value]
+
+        if self._postgresql.major_version >= 90600 and len(value) > 1:
+            sync_param = '{0} ({1})'.format(len(value), ','.join(value))
+        else:
+            sync_param = next(iter(value), None)
+
+        if not (self._postgresql.config.set_synchronous_standby_names(sync_param) and
+                self._postgresql.state == 'running' and self._postgresql.is_leader()) or value == ['*']:
+            return
+
+        time.sleep(0.1)  # Usualy it takes 1ms to reload postgresql.conf, but we will give it 100ms
+
+        # Reset internal cache to query fresh values
+        self._postgresql.reset_cluster_info_state(None)
+
+        # timeline == 0 -- indicates that this is the replica, shoudn't ever happen
+        if self._postgresql.get_primary_timeline() > 0:
+            self._handle_synchronous_standby_names_change()

patroni/postgresql/validator.py

@@ -200,7 +200,6 @@ parameters = CaseInsensitiveDict({
     'enable_async_append': Bool(140000, None),
     'enable_bitmapscan': Bool(90300, None),
     'enable_gathermerge': Bool(100000, None),
-    'enable_group_by_reordering': Bool(150000, None),
     'enable_hashagg': Bool(90300, None),
     'enable_hashjoin': Bool(90300, None),
     'enable_incremental_sort': Bool(130000, None),

patroni/psycopg.py

@@ -6,7 +6,7 @@ try:
     from . import MIN_PSYCOPG2, parse_version
     if parse_version(__version__) < MIN_PSYCOPG2:
         raise ImportError
-    from psycopg2 import connect, Error, DatabaseError, OperationalError, ProgrammingError
+    from psycopg2 import connect as _connect, Error, DatabaseError, OperationalError, ProgrammingError
     from psycopg2.extensions import adapt
 
     try:
@@ -20,10 +20,10 @@ try:
             value.prepare(conn)
         return value.getquoted().decode('utf-8')
 except ImportError:
-    from psycopg import connect as _connect, sql, Error, DatabaseError, OperationalError, ProgrammingError
+    from psycopg import connect as __connect, sql, Error, DatabaseError, OperationalError, ProgrammingError
 
-    def connect(*args, **kwargs):
-        ret = _connect(*args, **kwargs)
+    def _connect(*args, **kwargs):
+        ret = __connect(*args, **kwargs)
         ret.server_version = ret.pgconn.server_version  # compatibility with psycopg2
         return ret
 
@@ -34,6 +34,16 @@ except ImportError:
         return sql.Literal(value).as_string(conn)
 
 
+def connect(*args, **kwargs):
+    if kwargs and 'replication' not in kwargs and kwargs.get('fallback_application_name') != 'Patroni ctl':
+        options = [kwargs['options']] if 'options' in kwargs else []
+        options.append('-c search_path=pg_catalog')
+        kwargs['options'] = ' '.join(options)
+    ret = _connect(*args, **kwargs)
+    ret.autocommit = True
+    return ret
+
+
 def quote_ident(value, conn=None):
     if _legacy or conn is None:
         return '"{0}"'.format(value.replace('"', '""'))

patroni/request.py

@@ -9,9 +9,9 @@ from .utils import USER_AGENT
 
 class PatroniRequest(object):
 
-    def __init__(self, config, insecure=False):
-        cert_reqs = 'CERT_NONE' if insecure or config.get('ctl', {}).get('insecure', False) else 'CERT_REQUIRED'
-        self._pool = urllib3.PoolManager(num_pools=10, maxsize=10, cert_reqs=cert_reqs)
+    def __init__(self, config, insecure=None):
+        self._insecure = insecure
+        self._pool = urllib3.PoolManager(num_pools=10, maxsize=10)
         self.reload_config(config)
 
     @staticmethod
@@ -32,12 +32,19 @@ class PatroniRequest(object):
     def reload_config(self, config):
         self._pool.headers = urllib3.make_headers(basic_auth=self._get_cfg_value(config, 'auth'), user_agent=USER_AGENT)
 
+        insecure = self._insecure if isinstance(self._insecure, bool) else config.get('ctl', {}).get('insecure', False)
         if self._apply_ssl_file_param(config, 'cert'):
+            #  With client certificate the cert_reqs must be set to CERT_REQUIRED even if insecure option is used
+            self._pool.connection_pool_kw['cert_reqs'] = 'CERT_REQUIRED'
+            #  The assert_hostname = False helps to silence warnings
+            self._pool.connection_pool_kw['assert_hostname'] = False if insecure else None
+
             self._apply_ssl_file_param(config, 'key')
 
             password = self._get_cfg_value(config, 'keyfile_password')
             self._apply_pool_param('key_password', password)
         else:
+            self._pool.connection_pool_kw['cert_reqs'] = 'CERT_NONE' if insecure else 'CERT_REQUIRED'
             self._pool.connection_pool_kw.pop('key_file', None)
 
         cacert = config.get('ctl', {}).get('cacert') or config.get('restapi', {}).get('cafile')

patroni/scripts/wale_restore.py

@@ -11,7 +11,7 @@
 # arguments are:
 #   - cluster scope
 #   - cluster role
-#   - master connection string
+#   - leader connection string
 #   - number of retries
 #   - envdir for the WALE env
 # - WALE_BACKUP_THRESHOLD_MEGABYTES if WAL amount is above that - use pg_basebackup
@@ -104,11 +104,11 @@ WALEConfig = namedtuple(
 
 class WALERestore(object):
     def __init__(self, scope, datadir, connstring, env_dir, threshold_mb,
-                 threshold_pct, use_iam, no_master, retries):
+                 threshold_pct, use_iam, no_leader, retries):
         self.scope = scope
-        self.master_connection = connstring
+        self.leader_connection = connstring
         self.data_dir = datadir
-        self.no_master = no_master
+        self.no_leader = no_leader
 
         wale_cmd = [
             'envdir',
@@ -213,41 +213,44 @@ class WALERestore(object):
         diff_in_bytes = backup_size
         attempts_no = 0
         while True:
-            if self.master_connection:
+            if self.leader_connection:
+                con = None
                 try:
                     # get the difference in bytes between the current WAL location and the backup start offset
-                    with psycopg.connect(self.master_connection) as con:
-                        if con.server_version >= 100000:
-                            wal_name = 'wal'
-                            lsn_name = 'lsn'
-                        else:
-                            wal_name = 'xlog'
-                            lsn_name = 'location'
-                        con.autocommit = True
-                        with con.cursor() as cur:
-                            cur.execute(("SELECT CASE WHEN pg_catalog.pg_is_in_recovery()"
-                                         " THEN GREATEST(pg_catalog.pg_{0}_{1}_diff(COALESCE("
-                                         "pg_last_{0}_receive_{1}(), '0/0'), %s)::bigint, "
-                                         "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), %s)::bigint)"
-                                         " ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), %s)::bigint"
-                                         " END").format(wal_name, lsn_name),
-                                        (backup_start_lsn, backup_start_lsn, backup_start_lsn))
+                    con = psycopg.connect(self.leader_connection)
+                    if con.server_version >= 100000:
+                        wal_name = 'wal'
+                        lsn_name = 'lsn'
+                    else:
+                        wal_name = 'xlog'
+                        lsn_name = 'location'
+                    with con.cursor() as cur:
+                        cur.execute(("SELECT CASE WHEN pg_catalog.pg_is_in_recovery()"
+                                     " THEN GREATEST(pg_catalog.pg_{0}_{1}_diff(COALESCE("
+                                     "pg_last_{0}_receive_{1}(), '0/0'), %s)::bigint, "
+                                     "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), %s)::bigint)"
+                                     " ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), %s)::bigint"
+                                     " END").format(wal_name, lsn_name),
+                                    (backup_start_lsn, backup_start_lsn, backup_start_lsn))
 
-                            diff_in_bytes = int(cur.fetchone()[0])
+                        diff_in_bytes = int(cur.fetchone()[0])
                 except psycopg.Error:
-                    logger.exception('could not determine difference with the master location')
+                    logger.exception('could not determine difference with the leader location')
                     if attempts_no < self.retries:  # retry in case of a temporarily connection issue
                         attempts_no = attempts_no + 1
                         time.sleep(RETRY_SLEEP_INTERVAL)
                         continue
                     else:
-                        if not self.no_master:
+                        if not self.no_leader:
                             return False  # do no more retries on the outer level
-                        logger.info("continue with base backup from S3 since master is not available")
+                        logger.info("continue with base backup from S3 since leader is not available")
                         diff_in_bytes = 0
                         break
+                finally:
+                    if con:
+                        con.close()
             else:
-                # always try to use WAL-E if master connection string is not available
+                # always try to use WAL-E if leader connection string is not available
                 diff_in_bytes = 0
             break
 
@@ -343,22 +346,22 @@ def main():
     parser.add_argument('--threshold_megabytes', type=int, default=10240)
     parser.add_argument('--threshold_backup_size_percentage', type=int, default=30)
     parser.add_argument('--use_iam', type=int, default=0)
-    parser.add_argument('--no_master', type=int, default=0)
+    parser.add_argument('--no_leader', '--no_master', type=int, default=0)
     args = parser.parse_args()
 
     exit_code = None
     assert args.retries >= 0
 
-    # Retry cloning in a loop. We do separate retries for the master
+    # Retry cloning in a loop. We do separate retries for the leader
     # connection attempt inside should_use_s3_to_create_replica,
     # because we need to differentiate between the last attempt and
     # the rest and make a decision when the last attempt fails on
-    # whether to use WAL-E or not depending on the no_master flag.
+    # whether to use WAL-E or not depending on the no_leader flag.
     for _ in range(0, args.retries + 1):
         restore = WALERestore(scope=args.scope, datadir=args.datadir, connstring=args.connstring,
                               env_dir=args.envdir, threshold_mb=args.threshold_megabytes,
                               threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam,
-                              no_master=args.no_master, retries=args.retries)
+                              no_leader=args.no_leader, retries=args.retries)
         exit_code = restore.run()
         if not exit_code == ExitCode.RETRY_LATER:  # only WAL-E failures lead to the retry
             logger.debug('exit_code is %r, not retrying', exit_code)

patroni/utils.py

@@ -519,8 +519,8 @@ def enable_keepalive(sock, timeout, idle, cnt=3):
 def find_executable(executable, path=None):
     _, ext = os.path.splitext(executable)
 
-    if (sys.platform == 'win32') and (ext != '.exe'):
-        executable = executable + '.exe'
+    if (sys.platform == 'win32') and (ext == ''):
+        executable = executable + '.exe'  # Set default WIN extension
 
     if os.path.isfile(executable):
         return executable

patroni/validator.py

@@ -37,6 +37,10 @@ def validate_host_port(host_port, listen=False, multiple_hosts=False):
             hosts = hosts.split(",")
         else:
             hosts = [hosts]
+        if "*" in hosts:
+            if len(hosts) != 1:
+                raise ConfigParseError("expecting '*' alone")
+            hosts = [p[-1][0] for p in socket.getaddrinfo(None, port, 0, socket.SOCK_STREAM, 0, socket.AI_PASSIVE)]
         for host in hosts:
             proto = socket.getaddrinfo(host, "", 0, socket.SOCK_STREAM, 0, socket.AI_PASSIVE)
             s = socket.socket(proto[0][0], socket.SOCK_STREAM)
@@ -178,9 +182,11 @@ class Schema(object):
         self.validator = validator
 
     def __call__(self, data):
+        errors = []
         for i in self.validate(data):
             if not i.status:
-                print(i)
+                errors.append(str(i))
+        return errors
 
     def validate(self, data):
         self.data = data
@@ -361,9 +367,14 @@ schema = Schema({
           Optional("ports"): [{"name": str, "port": int}],
           },
       }),
+  Optional("citus"): {
+    "database": str,
+    "group": int
+  },
   "postgresql": {
     "listen": validate_host_port_listen_multiple_hosts,
     "connect_address": validate_connect_address,
+    Optional("proxy_address"): validate_connect_address,
     "authentication": {
       "replication": userattributes,
       "superuser": userattributes,

patroni/version.py

@@ -1 +1 @@
-__version__ = '2.1.4'
+__version__ = '3.0.0'

patroni/watchdog/base.py

@@ -215,6 +215,10 @@ class Watchdog(object):
                     self._activate()
                 if self.config.timeout != self.active_config.timeout:
                     self.impl.set_timeout(self.config.timeout)
+                    if self.is_running:
+                        logger.info("{0} updated with {1} second timeout, timing slack {2} seconds"
+                                    .format(self.impl.describe(), self.impl.get_timeout(), self.config.timing_slack))
+                self.active_config = self.config
         except WatchdogError as e:
             logger.error("Error while sending keepalive: %s", e)
 

postgres0.yml

@@ -5,16 +5,22 @@ name: postgresql0
 restapi:
   listen: 127.0.0.1:8008
   connect_address: 127.0.0.1:8008
+#  cafile: /etc/ssl/certs/ssl-cacert-snakeoil.pem
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
 #  authentication:
 #    username: username
 #    password: password
 
-# ctl:
-#   insecure: false # Allow connections to SSL sites without certs
-#   certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
-#   cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem
+#ctl:
+#  insecure: false # Allow connections to Patroni REST API without verifying certificates
+#  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
+#  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
+#  cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem
+
+#citus:
+#  database: citus
+#  group: 0  # coordinator
 
 etcd:
   #Provide host to do the initial discovery of the cluster topology:
@@ -45,7 +51,7 @@ bootstrap:
     loop_wait: 10
     retry_timeout: 10
     maximum_lag_on_failover: 1048576
-#    master_start_timeout: 300
+#    primary_start_timeout: 300
 #    synchronous_mode: false
     #standby_cluster:
       #host: 127.0.0.1
@@ -99,6 +105,8 @@ bootstrap:
 postgresql:
   listen: 127.0.0.1:5432
   connect_address: 127.0.0.1:5432
+
+#  proxy_address: 127.0.0.1:5433  # The address of connection pool (e.g., pgbouncer) running next to Patroni/Postgres. Only for service discovery.
   data_dir: data/postgresql0
 #  bin_dir:
 #  config_dir:

postgres1.yml

@@ -5,16 +5,22 @@ name: postgresql1
 restapi:
   listen: 127.0.0.1:8009
   connect_address: 127.0.0.1:8009
+#  cafile: /etc/ssl/certs/ssl-cacert-snakeoil.pem
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
 #  authentication:
 #    username: username
 #    password: password
 
-# ctl:
-#   insecure: false # Allow connections to SSL sites without certs
-#   certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
-#   cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem
+#ctl:
+#  insecure: false # Allow connections to Patroni REST API without verifying certificates
+#  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
+#  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
+#  cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem
+
+#citus:
+#  database: citus
+#  group: 1  # worker
 
 etcd:
   #Provide host to do the initial discovery of the cluster topology:
@@ -93,6 +99,7 @@ bootstrap:
 postgresql:
   listen: 127.0.0.1:5433
   connect_address: 127.0.0.1:5433
+#  proxy_address: 127.0.0.1:5434  # The address of connection pool (e.g., pgbouncer) running next to Patroni/Postgres. Only for service discovery.
   data_dir: data/postgresql1
 #  bin_dir:
 #  config_dir:

postgres2.yml

@@ -5,16 +5,22 @@ name: postgresql2
 restapi:
   listen: 127.0.0.1:8010
   connect_address: 127.0.0.1:8010
+#  cafile: /etc/ssl/certs/ssl-cacert-snakeoil.pem
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
   authentication:
     username: username
     password: password
 
-# ctl:
-#   insecure: false # Allow connections to SSL sites without certs
-#   certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
-#   cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem
+#ctl:
+#  insecure: false # Allow connections to Patroni REST API without verifying certificates
+#  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
+#  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
+#  cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem
+
+#citus:
+#  database: citus
+#  group: 1  # worker
 
 etcd:
   #Provide host to do the initial discovery of the cluster topology:
@@ -90,6 +96,7 @@ bootstrap:
 postgresql:
   listen: 127.0.0.1:5434
   connect_address: 127.0.0.1:5434
+#  proxy_address: 127.0.0.1:5435  # The address of connection pool (e.g., pgbouncer) running next to Patroni/Postgres. Only for service discovery.
   data_dir: data/postgresql2
 #  bin_dir:
 #  config_dir:

release.sh

@@ -1,31 +1,28 @@
-#!/bin/sh
+#!/bin/bash
 
-if [ $# -ne 1 ]; then
-    >&2 echo "usage: $0 <version>"
-    exit 1
-fi
-
-readonly VERSIONFILE="patroni/version.py"
+# Release process:
+# 1. Open a PR that updates release notes and Patroni version
+# 2. Merge it
+# 3. Run release.sh
+# 4. After the new tag is pushed, the .github/workflows/release.yaml will run tests and upload the new package to test.pypi.org
+# 5. Once the release is created, the .github/workflows/release.yaml will run tests and upload the new package to pypi.org
 
 ## Bail out on any non-zero exitcode from the called processes
 set -xe
 
-python3 --version
+if python3 --version &> /dev/null; then
+    alias python=python3
+    shopt -s expand_aliases
+fi
+
+python --version
 git --version
 
-version=$1
+version=$(python -c 'from patroni.version import __version__; print(__version__)')
 
-sed -i "s/__version__ = .*/__version__ = '${version}'/"  "${VERSIONFILE}"
-python3 setup.py clean
-python3 setup.py test
-python3 setup.py flake8
+python setup.py clean
+python setup.py test
+python setup.py flake8
 
-git add "${VERSIONFILE}"
-
-git commit -m "Bumped version to $version"
-git push
-
-python3 setup.py sdist bdist_wheel upload
-
-git tag v${version}
+git tag "v$version"
 git push --tags

requirements.dev.txt

@@ -1,7 +1,7 @@
 psycopg2-binary
 behave
 coverage
-flake8
+flake8>=3.0.0
 mock
 pytest-cov
 pytest

setup.py

@@ -18,8 +18,8 @@ MAIN_PACKAGE = NAME
 DESCRIPTION = 'PostgreSQL High-Available orchestrator and CLI'
 LICENSE = 'The MIT License'
 URL = 'https://github.com/zalando/patroni'
-AUTHOR = 'Alexander Kukushkin, Dmitrii Dolgov, Oleksii Kliukin'
-AUTHOR_EMAIL = 'alexander.kukushkin@zalando.de, [email protected], [email protected]'
+AUTHOR = 'Alexander Kukushkin, Polina Bungina'
+AUTHOR_EMAIL = 'akukushkin@microsoft.com, [email protected]'
 KEYWORDS = 'etcd governor patroni postgresql postgres ha haproxy confd' +\
     ' zookeeper exhibitor consul streaming replication kubernetes k8s'
 
@@ -41,15 +41,12 @@ CLASSIFIERS = [
     'Operating System :: POSIX :: BSD :: FreeBSD',
     'Operating System :: Microsoft :: Windows',
     'Programming Language :: Python',
-    'Programming Language :: Python :: 2.7',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.4',
-    'Programming Language :: Python :: 3.5',
-    'Programming Language :: Python :: 3.6',
     'Programming Language :: Python :: 3.7',
     'Programming Language :: Python :: 3.8',
     'Programming Language :: Python :: 3.9',
     'Programming Language :: Python :: 3.10',
+    'Programming Language :: Python :: 3.11',
     'Programming Language :: Python :: Implementation :: CPython',
 ]
 
@@ -93,12 +90,10 @@ class Flake8(_Command):
         return [package for package in self.package_files()] + ['tests', 'setup.py']
 
     def run(self):
-        from flake8.main import application
+        from flake8.main.cli import main
 
         logging.getLogger().setLevel(logging.ERROR)
-        flake8 = application.Application()
-        flake8.run(self.targets())
-        flake8.exit()
+        main(self.targets())
 
 
 class PyTest(_Command):
@@ -162,7 +157,6 @@ def setup_package(version):
         classifiers=CLASSIFIERS,
         packages=find_packages(exclude=['tests', 'tests.*']),
         package_data={MAIN_PACKAGE: ["*.json"]},
-        python_requires='>=2.7',
         install_requires=install_requires,
         extras_require=EXTRAS_REQUIRE,
         cmdclass=cmdclass,
@@ -173,14 +167,12 @@ def setup_package(version):
 if __name__ == '__main__':
     old_modules = sys.modules.copy()
     try:
-        from patroni import check_psycopg, fatal
+        from patroni import check_psycopg
         from patroni.version import __version__
     finally:
         sys.modules.clear()
         sys.modules.update(old_modules)
 
-    if sys.version_info < (2, 7, 0):
-        fatal('Patroni needs to be run with Python 2.7+')
     check_psycopg()
 
     setup_package(__version__)

tests/__init__.py

@@ -43,20 +43,21 @@ class MockResponse(object):
         return {'content-type': 'json'}
 
 
-def requests_get(url, **kwargs):
+def requests_get(url, method='GET', endpoint=None, data='', **kwargs):
     members = '[{"id":14855829450254237642,"peerURLs":["http://localhost:2380","http://localhost:7001"],' +\
               '"name":"default","clientURLs":["http://localhost:2379","http://localhost:4001"]}]'
     response = MockResponse()
-    if url.startswith('http://local'):
+    if endpoint == 'failsafe':
+        response.content = 'Accepted'
+    elif url.startswith('http://local'):
         raise urllib3.exceptions.HTTPError()
     elif ':8011/patroni' in url:
-        response.content = '{"role": "replica", "xlog": {"received_location": 0}, "tags": {}}'
+        response.content = '{"role": "replica", "wal": {"received_location": 0}, "tags": {}}'
     elif url.endswith('/members'):
         response.content = '[{}]' if url.startswith('http://error') else members
     elif url.startswith('http://exhibitor'):
         response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'
     elif url.endswith(':8011/reinitialize'):
-        data = kwargs.get('data', '')
         if ' false}' in data:
             response.status_code = 503
             response.content = 'restarting after failure already in progress'
@@ -66,9 +67,8 @@ def requests_get(url, **kwargs):
 
 
 class MockPostmaster(object):
-    def __init__(self, is_running=True, is_single_master=False):
-        self.is_running = Mock(return_value=is_running)
-        self.is_single_master = Mock(return_value=is_single_master)
+    def __init__(self, pid=1):
+        self.is_running = Mock(return_value=self)
         self.wait_for_user_backends_to_close = Mock()
         self.signal_stop = Mock(return_value=None)
         self.wait = Mock()
@@ -97,8 +97,12 @@ class MockCursor(object):
             self.results = [('ls', 'logical', 'a', 'b', 100, 500, b'123456')]
         elif sql.startswith('SELECT slot_name'):
             self.results = [('blabla', 'physical'), ('foobar', 'physical'), ('ls', 'logical', 'a', 'b', 5, 100, 500)]
+        elif sql.startswith('WITH slots AS (SELECT slot_name, active'):
+            self.results = [(False, True)]
         elif sql.startswith('SELECT CASE WHEN pg_catalog.pg_is_in_recovery()'):
-            self.results = [(1, 2, 1, 0, False, 1, 1, None, None, [{"slot_name": "ls", "confirmed_flush_lsn": 12345}])]
+            self.results = [(1, 2, 1, 0, False, 1, 1, None, None,
+                             [{"slot_name": "ls", "confirmed_flush_lsn": 12345}],
+                             'on', 'n1', None)]
         elif sql.startswith('SELECT pg_catalog.pg_is_in_recovery()'):
             self.results = [(False, 2)]
         elif sql.startswith('SELECT pg_catalog.pg_postmaster_start_time'):
@@ -123,6 +127,10 @@ class MockCursor(object):
                                  b'1\t0/40159C0\tno recovery target specified\n\n'
                                  b'2\t0/402DD98\tno recovery target specified\n\n'
                                  b'3\t0/403DD98\tno recovery target specified\n')]
+        elif sql.startswith('SELECT pg_catalog.citus_add_node'):
+            self.results = [(2,)]
+        elif sql.startswith('SELECT nodeid, groupid'):
+            self.results = [(1, 0, 'host1', 5432, 'primary'), (2, 1, 'host2', 5432, 'primary')]
         else:
             self.results = [(None, None, None, None, None, None, None, None, None, None)]
 
@@ -177,18 +185,19 @@ class PostgresInit(unittest.TestCase):
                    'force_parallel_mode': '1', 'constraint_exclusion': '',
                    'max_stack_depth': 'Z', 'vacuum_cost_limit': -1, 'vacuum_cost_delay': 200}
 
-    @patch('patroni.psycopg.connect', psycopg_connect)
+    @patch('patroni.psycopg._connect', psycopg_connect)
     @patch('patroni.postgresql.CallbackExecutor', Mock())
     @patch.object(ConfigHandler, 'write_postgresql_conf', Mock())
     @patch.object(ConfigHandler, 'replace_pg_hba', Mock())
     @patch.object(ConfigHandler, 'replace_pg_ident', Mock())
-    @patch.object(Postgresql, 'get_postgres_role_from_data_directory', Mock(return_value='master'))
+    @patch.object(Postgresql, 'get_postgres_role_from_data_directory', Mock(return_value='primary'))
     def setUp(self):
         data_dir = os.path.join('data', 'test0')
         self.p = Postgresql({'name': 'postgresql0', 'scope': 'batman', 'data_dir': data_dir,
                              'config_dir': data_dir, 'retry_timeout': 10,
                              'krbsrvname': 'postgres', 'pgpass': os.path.join(data_dir, 'pgpass0'),
-                             'listen': '127.0.0.2, 127.0.0.3:5432', 'connect_address': '127.0.0.2:5432',
+                             'listen': '127.0.0.2, 127.0.0.3:5432',
+                             'connect_address': '127.0.0.2:5432', 'proxy_address': '127.0.0.2:5433',
                              'authentication': {'superuser': {'username': 'foo', 'password': 'test'},
                                                 'replication': {'username': '', 'password': 'rep-pass'},
                                                 'rewind': {'username': 'rewind', 'password': 'test'}},
@@ -199,7 +208,8 @@ class PostgresInit(unittest.TestCase):
                              'pg_hba': ['host all all 0.0.0.0/0 md5'],
                              'pg_ident': ['krb realm postgres'],
                              'callbacks': {'on_start': 'true', 'on_stop': 'true', 'on_reload': 'true',
-                                           'on_restart': 'true', 'on_role_change': 'true'}})
+                                           'on_restart': 'true', 'on_role_change': 'true'},
+                             'citus': {'group': 0, 'database': 'citus'}})
 
 
 class BaseTestPostgresql(PostgresInit):
@@ -210,11 +220,13 @@ class BaseTestPostgresql(PostgresInit):
         if not os.path.exists(self.p.data_dir):
             os.makedirs(self.p.data_dir)
 
-        self.leadermem = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
+        self.leadermem = Member(0, 'leader', 28, {
+            'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
         self.leader = Leader(-1, 28, self.leadermem)
         self.other = Member(0, 'test-1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
-                                              'tags': {'replicatefrom': 'leader'}})
-        self.me = Member(0, 'test0', 28, {'conn_url': 'postgres://replicator:[email protected]:5434/postgres'})
+                                              'state': 'running', 'tags': {'replicatefrom': 'leader'}})
+        self.me = Member(0, 'test0', 28, {
+            'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5434/postgres'})
 
     def tearDown(self):
         if os.path.exists(self.p.data_dir):