Merge branch 'master' into feature/clusterid

.gitignore

@@ -1,3 +1,11 @@
 data/*
 *.pyc
-helpers/*.pyc
+*.egg/
+*.egg-info/
+.cache/
+.coverage
+.eggs/
+build/
+coverage.xml
+junit.xml
+pgpass

patroni/postgresql.py

@@ -135,9 +135,15 @@ class Postgresql:
 
     def _cursor(self):
         if not self._cursor_holder or self._cursor_holder.closed or self._cursor_holder.connection.closed != 0:
+            logger.info("established a new patroni connection to the postgres cluster")
             self._cursor_holder = self.connection().cursor()
         return self._cursor_holder
 
+    def close_connection(self):
+        if self._cursor_holder and self._cursor_holder.connection and self._cursor_holder.connection.closed == 0:
+            self._cursor_holder.connection.close()
+            logger.info("closed patroni connection to the postgresql cluster")
+
     def _query(self, sql, *params):
         cursor = None
         try:
@@ -277,6 +283,12 @@ class Postgresql:
             logging.exception('Exception during CHECKPOINT')
 
     def stop(self, mode='fast', block_callbacks=False):
+        # make sure we close all connections established against
+        # the former node, otherwise, we might get a stalled one
+        # after kill -9, which would report incorrect data to
+        # patroni.
+
+        self.close_connection()
         if not self.is_running():
             if not block_callbacks:
                 self.set_state('stopped')
@@ -324,7 +336,8 @@ class Postgresql:
         return True
 
     def check_replication_lag(self, last_leader_operation):
-        return last_leader_operation - self.xlog_position() <= self.config.get('maximum_lag_on_failover', 0)
+        return (last_leader_operation if last_leader_operation else 0) - self.xlog_position() <=\
+            self.config.get('maximum_lag_on_failover', 0)
 
     def write_pg_hba(self):
         with open(os.path.join(self.data_dir, 'pg_hba.conf'), 'a') as f:
@@ -387,7 +400,7 @@ recovery_target_timeline = 'latest'
             data = subprocess.check_output(['pg_controldata', self.data_dir])
             if data:
                 data = data.splitlines()
-                result = {l.split(':')[0]: l.split(':')[1].strip() for l in data if l}
+                result = {l.split(':')[0].replace('Current ', '', 1): l.split(':')[1].strip() for l in data if l}
         except subprocess.CalledProcessError:
             logger.exception("Error when calling pg_controldata")
         finally:

requirements-py2.txt

@@ -1,7 +1,7 @@
 boto
 dnspython
 mock
-psycopg2
+psycopg2>=2.6.1
 PyYAML
 requests
 six >= 1.7

requirements-py3.txt

@@ -1,7 +1,7 @@
 boto
 mock
 dnspython3
-psycopg2
+psycopg2>=2.6.1
 PyYAML
 requests
 six

tests/test_ha.py

@@ -1,3 +1,4 @@
+import etcd
 import unittest
 
 from mock import Mock, patch
@@ -98,6 +99,7 @@ class TestHa(unittest.TestCase):
         self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
         self.e.client.read = etcd_read
         self.e.client.write = etcd_write
+        self.e.client.delete = Mock(side_effect=etcd.EtcdException())
         self.ha = Ha(MockPatroni(self.p, self.e))
         self.ha._async_executor.run_async = run_async
         self.ha.old_cluster = self.e.get_cluster()

tests/test_postgresql.py

@@ -118,12 +118,12 @@ Backup start location:                0/0
 Backup end location:                  0/0
 End-of-backup record required:        no
 wal_level setting:                    hot_standby
-wal_log_hints setting:                on
-max_connections setting:              100
-max_worker_processes setting:         8
-max_prepared_xacts setting:           0
-max_locks_per_xact setting:           64
-track_commit_timestamp setting:       off
+Current wal_log_hints setting:                on
+Current max_connections setting:              100
+Current max_worker_processes setting:         8
+Current max_prepared_xacts setting:           0
+Current max_locks_per_xact setting:           64
+Current track_commit_timestamp setting:       off
 Maximum data alignment:               8
 Database block size:                  8192
 Blocks per segment of large relation: 131072
@@ -232,8 +232,8 @@ class TestPostgresql(unittest.TestCase):
         self.p.follow_the_leader(self.leader)
         self.p.require_rewind()
         with mock.patch('os.path.islink', MagicMock(return_value=True)):
-            with mock.patch('os.unlink', MagicMock(return_value=True)):
-                with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
+            with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
+                with mock.patch('os.unlink', MagicMock(return_value=True)):
                     self.p.follow_the_leader(self.leader, recovery=True)
         self.p.require_rewind()
         with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):