Add postgres-9.6 support (#357)

starting from 9.6 we need wal_level = 'replica' which is alias for 'hot_standby'. It was working before without problems, but if somebody change wal_level to replica, Patroni will expose pending_restart flag, although restart in this case is not necessary.

* bump versions of consul and etcd to the latest for travis integration-tests

.travis.yml

@@ -9,7 +9,7 @@ addons:
   postgresql: "9.5"
 env:
   global:
-  - ETCDVERSION=3.0.10 ZKVERSION=3.4.9 CONSULVERSION=0.7.0
+  - ETCDVERSION=3.0.15 ZKVERSION=3.4.9 CONSULVERSION=0.7.1
   matrix:
   - TEST_SUITE="python setup.py"
   - DCS="etcd" TEST_SUITE="behave"

Dockerfile

@@ -1,26 +1,41 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
 FROM ubuntu:16.04
-MAINTAINER Feike Steenbergen <feike.steenberge[email protected]>
+MAINTAINER Alexander Kukushkin <alexander.kukushki[email protected]>
 
 RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend \
     && echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend
 
-ENV PGVERSION 9.5
+ENV PGVERSION 9.6
 ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH
 RUN apt-get update -y \
     && apt-get upgrade -y \
-    && apt-get install -y curl jq haproxy zookeeper postgresql-${PGVERSION} python-psycopg2 python-yaml \
-        python-requests python-six python-click python-dateutil python-tzlocal python-urllib3 \
-        python-dnspython python-pip python-setuptools python-kazoo python-prettytable python-wheel python \
-    && pip install python-etcd==0.4.3 python-consul==0.6.0 --upgrade \
+    && apt-get install -y curl jq haproxy zookeeper python-psycopg2 python-yaml python-requests \
+        python-six python-click python-dateutil python-tzlocal python-urllib3 python-dnspython \
+        python-pip python-setuptools python-kazoo python-prettytable python-wheel python \
+
+    && export DISTRIB_CODENAME=$(sed -n 's/DISTRIB_CODENAME=//p' /etc/lsb-release) \
+    && echo "deb http://apt.postgresql.org/pub/repos/apt/ ${DISTRIB_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
+    && echo "deb-src http://apt.postgresql.org/pub/repos/apt/ ${DISTRIB_CODENAME}-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \
+    && curl -s -o - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+
+    ## Make sure we have a en_US.UTF-8 locale available
+    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
+
+    && apt-get update -y \
+    && apt-get install -y postgresql-contrib-${PGVERSION} \
+
+    # Remove the default cluster, which Debian stupidly starts right after installation of the packages
+    && pg_dropcluster --stop ${PGVERSION} main \
+    && pip install python-etcd==0.4.3 python-consul==0.6.1 --upgrade \
+
+    # Clean up
     && apt-get remove -y python-pip python-setuptools \
     && apt-get autoremove -y \
-        # Clean up
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/* /root/.cache
 
-ENV ETCDVERSION 3.0.8
+ENV ETCDVERSION 3.0.15
 RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
     | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
 

patroni/postgresql.py

@@ -59,7 +59,7 @@ class Postgresql(object):
         'listen_addresses': (None, lambda _: False, 9.1),
         'port': (None, lambda _: False, 9.1),
         'cluster_name': (None, lambda _: False, 9.5),
-        'wal_level': ('hot_standby', lambda v: v.lower() in ('hot_standby', 'logical'), 9.1),
+        'wal_level': ('hot_standby', lambda v: v.lower() in ('hot_standby', 'replica', 'logical'), 9.1),
         'hot_standby': ('on', lambda _: False, 9.1),
         'max_connections': (100, lambda v: int(v) >= 100, 9.1),
         'max_wal_senders': (5, lambda v: int(v) >= 5, 9.1),
@@ -213,6 +213,8 @@ class Postgresql(object):
                     elif r[0] in changes:
                         unit = changes['wal_segment_size'] if r[0] in ('min_wal_size', 'max_wal_size') else r[2]
                         new_value = changes.pop(r[0])
+                        if self._major_version >= 9.6 and r[0] == 'wal_level' and new_value == 'hot_standby':
+                            new_value = 'replica'
                         if new_value is None or not compare_values(r[3], unit, r[1], new_value):
                             if r[4] == 'postmaster':
                                 pending_restart = True
@@ -597,8 +599,10 @@ class Postgresql(object):
         self._write_postgresql_conf()
         self.resolve_connection_addresses()
 
-        options = ' '.join("--{0}='{1}'".format(p, self._server_parameters[p]) for p, v in self.CMDLINE_OPTIONS.items()
-                           if self._major_version >= v[2])
+        opts = {p: self._server_parameters[p] for p, v in self.CMDLINE_OPTIONS.items() if self._major_version >= v[2]}
+        if self._major_version >= 9.6 and opts['wal_level'] == 'hot_standby':
+            opts['wal_level'] = 'replica'
+        options = ' '.join("--{0}='{1}'".format(p, v) for p, v in opts.items())
 
         ret = self.pg_ctl('start', '-o', options, env=env, preexec_fn=os.setsid)
         self._pending_restart = False

tests/test_postgresql.py

@@ -40,7 +40,8 @@ class MockCursor(object):
                             ('search_path', 'public', None, 'string', 'user'),
                             ('port', '5433', None, 'integer', 'postmaster'),
                             ('listen_addresses', '*', None, 'string', 'postmaster'),
-                            ('autovacuum', 'on', None, 'bool', 'sighup')]
+                            ('autovacuum', 'on', None, 'bool', 'sighup'),
+                            ('wal_level', 'replica', None, 'enum', 'postmaster')]
         else:
             self.results = [(None, None, None, None, None, None, None, None, None, None)]
 
@@ -157,12 +158,13 @@ class TestPostgresql(unittest.TestCase):
     _PARAMETERS = {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'f.oo': 'bar',
                    'search_path': 'public', 'hot_standby': 'on', 'max_wal_senders': 5,
                    'wal_keep_segments': 8, 'wal_log_hints': 'on', 'max_locks_per_transaction': 64,
-                   'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0}
+                   'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0,
+                   'track_commit_timestamp': 'off'}
 
     @patch('subprocess.call', Mock(return_value=0))
     @patch('psycopg2.connect', psycopg2_connect)
     @patch('os.rename', Mock())
-    @patch.object(Postgresql, 'get_major_version', Mock(return_value=9.4))
+    @patch.object(Postgresql, 'get_major_version', Mock(return_value=9.6))
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def setUp(self):
         self.data_dir = 'data/test0'