Switch to boto3 (#2275)

Close https://github.com/zalando/patroni/issues/2237

patroni/scripts/aws.py

@@ -3,10 +3,12 @@
 import json
 import logging
 import sys
-import boto.ec2
+import boto3
 
-from patroni.utils import Retry, RetryFailedError
-from patroni.request import get as requests_get
+from ..utils import Retry, RetryFailedError
+from ..request import get as requests_get
+
+from botocore.exceptions import ClientError
 
 logger = logging.getLogger(__name__)
 
@@ -16,7 +18,7 @@ class AWSConnection(object):
     def __init__(self, cluster_name):
         self.available = False
         self.cluster_name = cluster_name if cluster_name is not None else 'unknown'
-        self._retry = Retry(deadline=300, max_delay=30, max_tries=-1, retry_exceptions=(boto.exception.StandardError,))
+        self._retry = Retry(deadline=300, max_delay=30, max_tries=-1, retry_exceptions=(ClientError,))
         try:
             # get the instance id
             r = requests_get('http://169.254.169.254/latest/dynamic/instance-identity/document', timeout=2.1)
@@ -42,20 +44,22 @@ class AWSConnection(object):
 
     def _tag_ebs(self, conn, role):
         """ set tags, carrying the cluster name, instance role and instance id for the EBS storage """
-        tags = {'Name': 'spilo_' + self.cluster_name, 'Role': role, 'Instance': self.instance_id}
-        volumes = conn.get_all_volumes(filters={'attachment.instance-id': self.instance_id})
-        conn.create_tags([v.id for v in volumes], tags)
+        tags = [{'Key': 'Name', 'Value': 'spilo_' + self.cluster_name},
+                {'Key': 'Role', 'Value': role},
+                {'Key': 'Instance', 'Value': self.instance_id}]
+        volumes = conn.volumes.filter(Filters=[{'Name': 'attachment.instance-id', 'Values': [self.instance_id]}])
+        conn.create_tags(Resources=[v.id for v in volumes], Tags=tags)
 
     def _tag_ec2(self, conn, role):
         """ tag the current EC2 instance with a cluster role """
-        tags = {'Role': role}
-        conn.create_tags([self.instance_id], tags)
+        tags = [{'Key': 'Role', 'Value': role}]
+        conn.create_tags(Resources=[self.instance_id], Tags=tags)
 
     def on_role_change(self, new_role):
         if not self.available:
             return False
         try:
-            conn = self.retry(boto.ec2.connect_to_region, self.region)
+            conn = boto3.resource('ec2', region_name=self.region)
             self.retry(self._tag_ec2, conn, new_role)
             self.retry(self._tag_ebs, conn, new_role)
         except RetryFailedError:

requirements.txt

@@ -1,6 +1,6 @@
 urllib3>=1.19.1,!=1.21
 ipaddress; python_version=="2.7"
-boto
+boto3
 PyYAML
 six >= 1.7
 kazoo>=1.3.1

setup.py

@@ -23,7 +23,7 @@ AUTHOR_EMAIL = '[email protected], [email protected], alexk
 KEYWORDS = 'etcd governor patroni postgresql postgres ha haproxy confd' +\
     ' zookeeper exhibitor consul streaming replication kubernetes k8s'
 
-EXTRAS_REQUIRE = {'aws': ['boto'], 'etcd': ['python-etcd'], 'etcd3': ['python-etcd'],
+EXTRAS_REQUIRE = {'aws': ['boto3'], 'etcd': ['python-etcd'], 'etcd3': ['python-etcd'],
                   'consul': ['python-consul'], 'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'],
                   'kubernetes': [], 'raft': ['pysyncobj', 'cryptography']}
 COVERAGE_XML = True

tests/test_aws.py

@@ -1,4 +1,4 @@
-import boto.ec2
+import botocore
 import sys
 import unittest
 import urllib3
@@ -8,21 +8,27 @@ from collections import namedtuple
 from patroni.scripts.aws import AWSConnection, main as _main
 
 
-class MockEc2Connection(object):
+class MockVolumes(object):
 
     @staticmethod
-    def get_all_volumes(*args, **kwargs):
+    def filter(*args, **kwargs):
         oid = namedtuple('Volume', 'id')
         return [oid(id='a'), oid(id='b')]
 
+
+class MockEc2Connection(object):
+
+    volumes = MockVolumes()
+
     @staticmethod
-    def create_tags(objects, *args, **kwargs):
-        if len(objects) == 0:
-            raise boto.exception.BotoServerError(503, 'Service Unavailable', 'Request limit exceeded')
+    def create_tags(Resources, **kwargs):
+        if len(Resources) == 0:
+            raise botocore.exceptions.ClientError({'Error': {'Code': 503, 'Message': 'Request limit exceeded'}},
+                                                  'create_tags')
         return True
 
 
-@patch('boto.ec2.connect_to_region', Mock(return_value=MockEc2Connection()))
+@patch('boto3.resource', Mock(return_value=MockEc2Connection()))
 class TestAWSConnection(unittest.TestCase):
 
     @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(
@@ -32,7 +38,7 @@ class TestAWSConnection(unittest.TestCase):
 
     def test_on_role_change(self):
         self.assertTrue(self.conn.on_role_change('master'))
-        with patch.object(MockEc2Connection, 'get_all_volumes', Mock(return_value=[])):
+        with patch.object(MockVolumes, 'filter', Mock(return_value=[])):
             self.conn._retry.max_tries = 1
             self.assertFalse(self.conn.on_role_change('master'))
 
@@ -46,8 +52,7 @@ class TestAWSConnection(unittest.TestCase):
         conn = AWSConnection('test')
         self.assertFalse(conn.aws_available())
 
-    @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(
-        status=200, body=b'{"instanceId": "012345", "region": "eu-west-1"}')))
+    @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(status=503, body=b'Error')))
     @patch('sys.exit', Mock())
     def test_main(self):
         self.assertIsNone(_main())