From aea0589404ca04f2419e50bafbb61b550b31f0c7 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 14 Apr 2022 10:47:16 +0200 Subject: [PATCH] Switch to boto3 (#2275) Close https://github.com/zalando/patroni/issues/2237 --- patroni/scripts/aws.py | 24 ++++++++++++++---------- requirements.txt | 2 +- setup.py | 2 +- tests/test_aws.py | 25 +++++++++++++++---------- 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/patroni/scripts/aws.py b/patroni/scripts/aws.py index ee7e0412..31e7e398 100755 --- a/patroni/scripts/aws.py +++ b/patroni/scripts/aws.py @@ -3,10 +3,12 @@ import json import logging import sys -import boto.ec2 +import boto3 -from patroni.utils import Retry, RetryFailedError -from patroni.request import get as requests_get +from ..utils import Retry, RetryFailedError +from ..request import get as requests_get + +from botocore.exceptions import ClientError logger = logging.getLogger(__name__) @@ -16,7 +18,7 @@ class AWSConnection(object): def __init__(self, cluster_name): self.available = False self.cluster_name = cluster_name if cluster_name is not None else 'unknown' - self._retry = Retry(deadline=300, max_delay=30, max_tries=-1, retry_exceptions=(boto.exception.StandardError,)) + self._retry = Retry(deadline=300, max_delay=30, max_tries=-1, retry_exceptions=(ClientError,)) try: # get the instance id r = requests_get('http://169.254.169.254/latest/dynamic/instance-identity/document', timeout=2.1) @@ -42,20 +44,22 @@ class AWSConnection(object): def _tag_ebs(self, conn, role): """ set tags, carrying the cluster name, instance role and instance id for the EBS storage """ - tags = {'Name': 'spilo_' + self.cluster_name, 'Role': role, 'Instance': self.instance_id} - volumes = conn.get_all_volumes(filters={'attachment.instance-id': self.instance_id}) - conn.create_tags([v.id for v in volumes], tags) + tags = [{'Key': 'Name', 'Value': 'spilo_' + self.cluster_name}, + {'Key': 'Role', 'Value': role}, + {'Key': 'Instance', 'Value': self.instance_id}] + volumes = conn.volumes.filter(Filters=[{'Name': 'attachment.instance-id', 'Values': [self.instance_id]}]) + conn.create_tags(Resources=[v.id for v in volumes], Tags=tags) def _tag_ec2(self, conn, role): """ tag the current EC2 instance with a cluster role """ - tags = {'Role': role} - conn.create_tags([self.instance_id], tags) + tags = [{'Key': 'Role', 'Value': role}] + conn.create_tags(Resources=[self.instance_id], Tags=tags) def on_role_change(self, new_role): if not self.available: return False try: - conn = self.retry(boto.ec2.connect_to_region, self.region) + conn = boto3.resource('ec2', region_name=self.region) self.retry(self._tag_ec2, conn, new_role) self.retry(self._tag_ebs, conn, new_role) except RetryFailedError: diff --git a/requirements.txt b/requirements.txt index c7438d3d..060d0dcd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ urllib3>=1.19.1,!=1.21 ipaddress; python_version=="2.7" -boto +boto3 PyYAML six >= 1.7 kazoo>=1.3.1 diff --git a/setup.py b/setup.py index 0e8c660e..786c1a26 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ AUTHOR_EMAIL = 'alexander.kukushkin@zalando.de, dmitrii.dolgov@zalando.de, alexk KEYWORDS = 'etcd governor patroni postgresql postgres ha haproxy confd' +\ ' zookeeper exhibitor consul streaming replication kubernetes k8s' -EXTRAS_REQUIRE = {'aws': ['boto'], 'etcd': ['python-etcd'], 'etcd3': ['python-etcd'], +EXTRAS_REQUIRE = {'aws': ['boto3'], 'etcd': ['python-etcd'], 'etcd3': ['python-etcd'], 'consul': ['python-consul'], 'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'], 'kubernetes': [], 'raft': ['pysyncobj', 'cryptography']} COVERAGE_XML = True diff --git a/tests/test_aws.py b/tests/test_aws.py index 63b6fa9b..4f9762a9 100644 --- a/tests/test_aws.py +++ b/tests/test_aws.py @@ -1,4 +1,4 @@ -import boto.ec2 +import botocore import sys import unittest import urllib3 @@ -8,21 +8,27 @@ from collections import namedtuple from patroni.scripts.aws import AWSConnection, main as _main -class MockEc2Connection(object): +class MockVolumes(object): @staticmethod - def get_all_volumes(*args, **kwargs): + def filter(*args, **kwargs): oid = namedtuple('Volume', 'id') return [oid(id='a'), oid(id='b')] + +class MockEc2Connection(object): + + volumes = MockVolumes() + @staticmethod - def create_tags(objects, *args, **kwargs): - if len(objects) == 0: - raise boto.exception.BotoServerError(503, 'Service Unavailable', 'Request limit exceeded') + def create_tags(Resources, **kwargs): + if len(Resources) == 0: + raise botocore.exceptions.ClientError({'Error': {'Code': 503, 'Message': 'Request limit exceeded'}}, + 'create_tags') return True -@patch('boto.ec2.connect_to_region', Mock(return_value=MockEc2Connection())) +@patch('boto3.resource', Mock(return_value=MockEc2Connection())) class TestAWSConnection(unittest.TestCase): @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse( @@ -32,7 +38,7 @@ class TestAWSConnection(unittest.TestCase): def test_on_role_change(self): self.assertTrue(self.conn.on_role_change('master')) - with patch.object(MockEc2Connection, 'get_all_volumes', Mock(return_value=[])): + with patch.object(MockVolumes, 'filter', Mock(return_value=[])): self.conn._retry.max_tries = 1 self.assertFalse(self.conn.on_role_change('master')) @@ -46,8 +52,7 @@ class TestAWSConnection(unittest.TestCase): conn = AWSConnection('test') self.assertFalse(conn.aws_available()) - @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse( - status=200, body=b'{"instanceId": "012345", "region": "eu-west-1"}'))) + @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(status=503, body=b'Error'))) @patch('sys.exit', Mock()) def test_main(self): self.assertIsNone(_main())