From c1bfb0e6d6efc5b41169f9b93df51e34283d4558 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 13 Mar 2023 17:00:04 +0100 Subject: [PATCH] Remove python 2.7 support (#2571) - get rid from 2.7 specific modules: `six`, `ipaddress` - use Python3 unpacking operator - use `shutil.which()` instead of `find_executable()` --- features/environment.py | 6 ++---- patroni/api.py | 24 +++++++++--------------- patroni/config.py | 5 ++--- patroni/ctl.py | 21 ++++++++++----------- patroni/daemon.py | 4 +--- patroni/dcs/__init__.py | 8 +++----- patroni/dcs/consul.py | 4 ++-- patroni/dcs/etcd.py | 16 +++++++--------- patroni/dcs/etcd3.py | 15 --------------- patroni/dcs/exhibitor.py | 4 +--- patroni/dcs/kubernetes.py | 12 +++++------- patroni/dcs/zookeeper.py | 5 +---- patroni/ha.py | 3 +-- patroni/log.py | 2 +- patroni/postgresql/__init__.py | 9 +++------ patroni/postgresql/bootstrap.py | 6 ++---- patroni/postgresql/citus.py | 2 +- patroni/postgresql/config.py | 6 ++---- patroni/postgresql/rewind.py | 5 ++--- patroni/postgresql/validator.py | 4 +--- patroni/request.py | 5 ++--- patroni/utils.py | 18 ------------------ patroni/validator.py | 31 +++++++++++++++---------------- patroni/watchdog/base.py | 4 +--- requirements.txt | 2 -- tests/test_api.py | 16 +++++++++------- tests/test_config.py | 5 ++--- tests/test_ctl.py | 8 ++++---- tests/test_ha.py | 11 +++++------ tests/test_kubernetes.py | 11 +++++------ tests/test_log.py | 2 +- tests/test_patroni.py | 10 +++++----- tests/test_postgresql.py | 11 +++++------ tests/test_postmaster.py | 5 ++--- tests/test_rewind.py | 3 +-- tests/test_utils.py | 11 +---------- tests/test_validator.py | 24 +++++++++++++++--------- tests/test_wale_restore.py | 5 ++--- tests/test_zookeeper.py | 15 +++++++-------- 39 files changed, 138 insertions(+), 220 deletions(-) diff --git a/features/environment.py b/features/environment.py index 76eb8316..2e03b310 100644 --- a/features/environment.py +++ b/features/environment.py @@ -7,7 +7,6 @@ import psutil import re import shutil import signal -import six import subprocess import sys import tempfile @@ -17,12 +16,11 @@ import yaml import patroni.psycopg as psycopg +from http.server import BaseHTTPRequestHandler, HTTPServer from patroni.request import PatroniRequest -from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer -@six.add_metaclass(abc.ABCMeta) -class AbstractController(object): +class AbstractController(abc.ABC): def __init__(self, context, name, work_directory, output_dir): self._context = context diff --git a/patroni/api.py b/patroni/api.py index 2be0777e..7a049bae 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -7,15 +7,14 @@ import traceback import dateutil.parser import datetime import os -import six import socket import sys -from ipaddress import ip_address, ip_network as _ip_network -from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer -from six.moves.socketserver import ThreadingMixIn -from six.moves.urllib_parse import urlparse, parse_qs +from http.server import BaseHTTPRequestHandler, HTTPServer +from ipaddress import ip_address, ip_network +from socketserver import ThreadingMixIn from threading import Thread +from urllib.parse import urlparse, parse_qs from . import psycopg from .exceptions import PostgresConnectionException, PostgresException @@ -26,10 +25,6 @@ from .utils import deep_compare, enable_keepalive, parse_bool, patch_config, Ret logger = logging.getLogger(__name__) -def ip_network(value): - return _ip_network(value.decode('utf-8') if six.PY2 else value, False) - - class RestApiHandler(BaseHTTPRequestHandler): def _write_status_code_only(self, status_code): @@ -172,7 +167,7 @@ class RestApiHandler(BaseHTTPRequestHandler): if instance_tag_value is None: status_code = 503 break - if not isinstance(instance_tag_value, six.string_types): + if not isinstance(instance_tag_value, str): instance_tag_value = str(instance_tag_value).lower() if instance_tag_value != qs_value: status_code = 503 @@ -768,7 +763,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread): def __resolve_ips(host, port): try: for _, _, _, _, sa in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP): - yield ip_network(sa[0]) + yield ip_network(sa[0], False) except Exception as e: logger.error('Failed to resolve %s: %r', host, e) @@ -789,8 +784,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread): def check_access(self, rh): if self.__allowlist or self.__allowlist_include_members: - incoming_ip = rh.client_address[0] - incoming_ip = ip_address(incoming_ip.decode('utf-8') if six.PY2 else incoming_ip) + incoming_ip = ip_address(rh.client_address[0]) if not any(incoming_ip in net for net in self.__allowlist + tuple(self.__members_ips())): return rh._write_response(403, 'Access is denied') @@ -915,7 +909,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread): for v in value: if '/' in v: # netmask try: - yield ip_network(v) + yield ip_network(v, False) except Exception as e: logger.error('Invalid value "%s" in the allowlist: %r', v, e) else: # ip or hostname, try to resolve it @@ -935,7 +929,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread): self.http_extra_headers = config.get('http_extra_headers') or {} self.http_extra_headers.update((config.get('https_extra_headers') or {}) if ssl_options.get('certfile') else {}) - if isinstance(config.get('verify_client'), six.string_types): + if isinstance(config.get('verify_client'), str): ssl_options['verify_client'] = config['verify_client'].lower() if self.__listen != config['listen'] or self.__ssl_options != ssl_options or self._received_new_cert: diff --git a/patroni/config.py b/patroni/config.py index d69ff55a..02379c20 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -2,7 +2,6 @@ import json import logging import os import shutil -import six import tempfile import yaml @@ -408,8 +407,8 @@ class Config(object): config = self._safe_copy_dynamic_configuration(dynamic_configuration) for name, value in local_configuration.items(): if name == 'citus': # remove invalid citus configuration - if isinstance(value, dict) and isinstance(value.get('group'), six.integer_types)\ - and isinstance(value.get('database'), six.string_types): + if isinstance(value, dict) and isinstance(value.get('group'), int)\ + and isinstance(value.get('database'), str): config[name] = value elif name == 'postgresql': for name, value in (value or {}).items(): diff --git a/patroni/ctl.py b/patroni/ctl.py index 047fc4bb..ab035f95 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -4,17 +4,17 @@ Patroni Control import click import codecs +import copy import datetime import dateutil.parser import dateutil.tz -import copy import difflib import io import json import logging import os import random -import six +import shutil import subprocess import sys import tempfile @@ -25,7 +25,7 @@ from click import ClickException from collections import defaultdict from contextlib import contextmanager from prettytable import ALL, FRAME, PrettyTable -from six.moves.urllib_parse import urlparse +from urllib.parse import urlparse try: from ydiff import markup_to_pager, PatchStream @@ -35,7 +35,7 @@ except ImportError: # pragma: no cover from .dcs import get_dcs as _get_dcs from .exceptions import PatroniException from .postgresql.misc import postgres_version_to_int -from .utils import cluster_as_json, find_executable, patch_config, polling_loop, is_standby_cluster +from .utils import cluster_as_json, patch_config, polling_loop, is_standby_cluster from .request import PatroniRequest from .version import __version__ @@ -203,7 +203,7 @@ def print_output(columns, rows, alignment=None, fmt='pretty', header=None, delim for r in ([columns] if columns else []) + rows: click.echo(delimiter.join(map(str, r))) else: - hrules = ALL if any(any(isinstance(c, six.string_types) and '\n' in c for c in r) for r in rows) else FRAME + hrules = ALL if any(any(isinstance(c, str) and '\n' in c for c in r) for r in rows) else FRAME table = PatronictlPrettyTable(header, columns, hrules=hrules) table.align = 'l' for k, v in (alignment or {}).items(): @@ -875,7 +875,7 @@ def output_members(obj, cluster, name, extended=False, fmt='pretty', group=None) member.update(cluster=name, member=member['name'], group=g, host=member.get('host', ''), tl=member.get('timeline', ''), role=member['role'].replace('_', ' ').title(), - lag_in_mb=round(lag/1024/1024) if isinstance(lag, six.integer_types) else lag, + lag_in_mb=round(lag/1024/1024) if isinstance(lag, int) else lag, pending_restart='*' if member.get('pending_restart') else '') if append_port and member['host'] and member.get('port'): @@ -1084,8 +1084,7 @@ def show_diff(before_editing, after_editing): if sys.stdout.isatty(): buf = io.StringIO() for line in unified_diff: - # Force cast to unicode as difflib on Python 2.7 returns a mix of unicode and str. - buf.write(six.text_type(line)) + buf.write(str(line)) buf.seek(0) class opts: @@ -1093,10 +1092,10 @@ def show_diff(before_editing, after_editing): width = 80 tab_width = 8 wrap = True - if find_executable('less'): + if shutil.which('less'): pager = None else: - pager = 'more.com' if sys.platform == 'win32' else 'more' + pager = os.path.basename(shutil.which('more') or 'more') pager_options = None markup_to_pager(PatchStream(buf), opts) @@ -1184,7 +1183,7 @@ def invoke_editor(before_editing, cluster_name): editor_cmd = os.environ.get('EDITOR') if not editor_cmd: for editor in ('editor', 'vi'): - editor_cmd = find_executable(editor) + editor_cmd = shutil.which(editor) if editor_cmd: logging.debug('Setting fallback editor_cmd=%s', editor) break diff --git a/patroni/daemon.py b/patroni/daemon.py index 3f4f93ac..6bac504b 100644 --- a/patroni/daemon.py +++ b/patroni/daemon.py @@ -3,14 +3,12 @@ from __future__ import print_function import abc import os import signal -import six import sys from threading import Lock -@six.add_metaclass(abc.ABCMeta) -class AbstractPatroniDaemon(object): +class AbstractPatroniDaemon(abc.ABC): def __init__(self, config): from patroni.log import PatroniLogger diff --git a/patroni/dcs/__init__.py b/patroni/dcs/__init__.py index ca5e05c2..09d4be66 100644 --- a/patroni/dcs/__init__.py +++ b/patroni/dcs/__init__.py @@ -7,15 +7,14 @@ import logging import os import pkgutil import re -import six import sys import time from collections import defaultdict, namedtuple from copy import deepcopy from random import randint -from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl from threading import Event, Lock +from urllib.parse import urlparse, urlunparse, parse_qsl from ..exceptions import PatroniFatalException from ..utils import deep_compare, parse_bool, uri @@ -654,8 +653,7 @@ def catch_return_false_exception(func): return wrapper -@six.add_metaclass(abc.ABCMeta) -class AbstractDCS(object): +class AbstractDCS(abc.ABC): _INITIALIZE = 'initialize' _CONFIG = 'config' @@ -676,7 +674,7 @@ class AbstractDCS(object): """ self._name = config['name'] self._base_path = re.sub('/+', '/', '/'.join(['', config.get('namespace', 'service'), config['scope']])) - self._citus_group = str(config['group']) if isinstance(config.get('group'), six.integer_types) else None + self._citus_group = str(config['group']) if isinstance(config.get('group'), int) else None self._set_loop_wait(config.get('loop_wait', 10)) self._ctl = bool(config.get('patronictl', False)) diff --git a/patroni/dcs/consul.py b/patroni/dcs/consul.py index 0f657467..5404c961 100644 --- a/patroni/dcs/consul.py +++ b/patroni/dcs/consul.py @@ -10,9 +10,9 @@ import urllib3 from collections import defaultdict, namedtuple from consul import ConsulException, NotFound, base +from http.client import HTTPException from urllib3.exceptions import HTTPError -from six.moves.urllib.parse import urlencode, urlparse, quote -from six.moves.http_client import HTTPException +from urllib.parse import urlencode, urlparse, quote from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\ TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re diff --git a/patroni/dcs/etcd.py b/patroni/dcs/etcd.py index 604c73ba..d0541c38 100644 --- a/patroni/dcs/etcd.py +++ b/patroni/dcs/etcd.py @@ -6,7 +6,6 @@ import logging import os import urllib3.util.connection import random -import six import socket import time @@ -14,12 +13,12 @@ from collections import defaultdict from copy import deepcopy from dns.exception import DNSException from dns import resolver +from http.client import HTTPException +from queue import Queue +from threading import Thread +from urllib.parse import urlparse from urllib3 import Timeout from urllib3.exceptions import HTTPError, ReadTimeoutError, ProtocolError -from six.moves.queue import Queue -from six.moves.http_client import HTTPException -from six.moves.urllib_parse import urlparse -from threading import Thread from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\ TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re @@ -86,8 +85,7 @@ class DnsCachingResolver(Thread): return [] -@six.add_metaclass(abc.ABCMeta) -class AbstractEtcdClientWithFailover(etcd.Client): +class AbstractEtcdClientWithFailover(abc.ABC, etcd.Client): def __init__(self, config, dns_resolver, cache_ttl=300): self._dns_resolver = dns_resolver @@ -496,12 +494,12 @@ class AbstractEtcd(AbstractDCS): default_port = config.pop('port', 2379) protocol = config.get('protocol', 'http') - if isinstance(hosts, six.string_types): + if isinstance(hosts, str): hosts = hosts.split(',') config['hosts'] = [] for value in hosts: - if isinstance(value, six.string_types): + if isinstance(value, str): config['hosts'].append(uri(protocol, split_host_port(value.strip(), default_port))) elif 'host' in config: host, port = split_host_port(config['host'], 2379) diff --git a/patroni/dcs/etcd3.py b/patroni/dcs/etcd3.py index bf9ea156..b89684dd 100644 --- a/patroni/dcs/etcd3.py +++ b/patroni/dcs/etcd3.py @@ -4,7 +4,6 @@ import etcd import json import logging import os -import six import socket import sys import time @@ -177,20 +176,6 @@ class Etcd3Client(AbstractEtcdClientWithFailover): self.version_prefix = '/v3beta' super(Etcd3Client, self).__init__(config, dns_resolver, cache_ttl) - if six.PY2: # pragma: no cover - # Old grpc-gateway sometimes sends double 'transfer-encoding: chunked' headers, - # what breaks the old (python2.7) httplib.HTTPConnection (it closes the socket). - def dedup_addheader(httpm, key, value): - prev = httpm.dict.get(key) - if prev is None: - httpm.dict[key] = value - elif key != 'transfer-encoding' or prev != value: - combined = ", ".join((prev, value)) - httpm.dict[key] = combined - - import httplib - httplib.HTTPMessage.addheader = dedup_addheader - try: self.authenticate() except AuthFailed as e: diff --git a/patroni/dcs/exhibitor.py b/patroni/dcs/exhibitor.py index c4e0f429..cca14a58 100644 --- a/patroni/dcs/exhibitor.py +++ b/patroni/dcs/exhibitor.py @@ -64,9 +64,7 @@ class Exhibitor(ZooKeeper): def __init__(self, config): interval = config.get('poll_interval', 300) self._ensemble_provider = ExhibitorEnsembleProvider(config['hosts'], config['port'], poll_interval=interval) - config = config.copy() - config['hosts'] = self._ensemble_provider.zookeeper_hosts - super(Exhibitor, self).__init__(config) + super(Exhibitor, self).__init__({**config, 'hosts': self._ensemble_provider.zookeeper_hosts}) def _load_cluster(self, path, loader): if self._ensemble_provider.poll(): diff --git a/patroni/dcs/kubernetes.py b/patroni/dcs/kubernetes.py index 506066b8..dbda2371 100644 --- a/patroni/dcs/kubernetes.py +++ b/patroni/dcs/kubernetes.py @@ -7,7 +7,6 @@ import logging import os import random import socket -import six import tempfile import time import urllib3 @@ -15,11 +14,10 @@ import yaml from collections import defaultdict from copy import deepcopy -from urllib3 import Timeout -from urllib3.exceptions import HTTPError -from six.moves.http_client import HTTPException +from http.client import HTTPException from threading import Condition, Lock, Thread from typing import Any, Dict, List, Optional +from urllib3.exceptions import HTTPError from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\ TimelineHistory, CITUS_COORDINATOR_GROUP_ID, citus_group_re @@ -177,7 +175,7 @@ class K8sObject(object): if isinstance(value, dict): # we know that `annotations` and `labels` are dicts and therefore don't want to convert them into K8sObject return value if parent in {'annotations', 'labels'} and \ - all(isinstance(v, six.string_types) for v in value.values()) else cls(value) + all(isinstance(v, str) for v in value.values()) else cls(value) elif isinstance(value, list): return [cls._wrap(None, v) for v in value] else: @@ -377,7 +375,7 @@ class K8sClient(object): api_servers = len(api_servers_cache) if timeout: - if isinstance(timeout, six.integer_types + (float,)): + if isinstance(timeout, (int, float)): timeout = urllib3.Timeout(total=timeout) elif isinstance(timeout, tuple) and len(timeout) == 2: timeout = urllib3.Timeout(connect=timeout[0], read=timeout[1]) @@ -576,7 +574,7 @@ class ObjectCache(Thread): raise def _watch(self, resource_version): - return self._func(_request_timeout=(self._retry.deadline, Timeout.DEFAULT_TIMEOUT), + return self._func(_request_timeout=(self._retry.deadline, urllib3.Timeout.DEFAULT_TIMEOUT), _preload_content=False, watch=True, resource_version=resource_version) def set(self, name, value): diff --git a/patroni/dcs/zookeeper.py b/patroni/dcs/zookeeper.py index 476b1c51..ac192bae 100644 --- a/patroni/dcs/zookeeper.py +++ b/patroni/dcs/zookeeper.py @@ -1,7 +1,6 @@ import json import logging import select -import six import time from kazoo.client import KazooClient, KazooState, KazooRetry @@ -63,10 +62,8 @@ class PatroniSequentialThreadingHandler(SequentialThreadingHandler): try: return super(PatroniSequentialThreadingHandler, self).select(*args, **kwargs) - except IOError as e: - raise (select.error(e.errno, e.strerror) if six.PY2 else e) except (TypeError, ValueError) as e: - raise (e if six.PY2 and isinstance(e, TypeError) else select.error(9, str(e))) + raise select.error(9, str(e)) class PatroniKazooClient(KazooClient): diff --git a/patroni/ha.py b/patroni/ha.py index 06294c3c..6eda0a3e 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -2,7 +2,6 @@ import datetime import functools import json import logging -import six import sys import time import uuid @@ -880,7 +879,7 @@ class Ha(object): not_allowed_reason = st.failover_limitation() if not_allowed_reason: logger.info('Member %s is %s', st.member.name, not_allowed_reason) - elif not isinstance(st.wal_position, six.integer_types): + elif not isinstance(st.wal_position, int): logger.info('Member %s does not report wal_position', st.member.name) elif cluster_lsn and st.wal_position < cluster_lsn or\ not cluster_lsn and self.is_lagging(st.wal_position): diff --git a/patroni/log.py b/patroni/log.py index 1231f1dd..4ba766d2 100644 --- a/patroni/log.py +++ b/patroni/log.py @@ -5,7 +5,7 @@ import sys from copy import deepcopy from logging.handlers import RotatingFileHandler from patroni.utils import deep_compare -from six.moves.queue import Queue, Full +from queue import Queue, Full from threading import Lock, Thread _LOGGER = logging.getLogger(__name__) diff --git a/patroni/postgresql/__init__.py b/patroni/postgresql/__init__.py index c8c7ef99..c58b3290 100644 --- a/patroni/postgresql/__init__.py +++ b/patroni/postgresql/__init__.py @@ -3,7 +3,6 @@ import os import re import shlex import shutil -import six import subprocess import time @@ -473,7 +472,7 @@ class Postgresql(object): return prev except Exception as e: logger.error('Exception when parsing WAL pg_%sdump output: %r', self.wal_name, e) - if isinstance(checkpoint_lsn, six.integer_types): + if isinstance(checkpoint_lsn, int): return checkpoint_lsn def is_running(self): @@ -866,8 +865,7 @@ class Postgresql(object): # Don't try to call pg_controldata during backup restore if self._version_file_exists() and self.state != 'creating replica': try: - env = os.environ.copy() - env.update(LANG='C', LC_ALL='C') + env = {**os.environ, 'LANG': 'C', 'LC_ALL': 'C'} data = subprocess.check_output([self.pgcommand('pg_controldata'), self._data_dir], env=env) if data: data = filter(lambda e: ':' in e, data.decode('utf-8').splitlines()) @@ -879,8 +877,7 @@ class Postgresql(object): def waldump(self, timeline, lsn, limit): cmd = self.pgcommand('pg_{0}dump'.format(self.wal_name)) - env = os.environ.copy() - env.update(LANG='C', LC_ALL='C', PGDATA=self._data_dir) + env = {**os.environ, 'LANG': 'C', 'LC_ALL': 'C', 'PGDATA': self._data_dir} try: waldump = subprocess.Popen([cmd, '-t', str(timeline), '-s', str(lsn), '-n', str(limit)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) diff --git a/patroni/postgresql/bootstrap.py b/patroni/postgresql/bootstrap.py index 654f346f..42c24c9c 100644 --- a/patroni/postgresql/bootstrap.py +++ b/patroni/postgresql/bootstrap.py @@ -4,8 +4,6 @@ import shlex import tempfile import time -from six import string_types - from ..dcs import RemoteMember from ..psycopg import quote_ident, quote_literal from ..utils import deep_compare @@ -43,11 +41,11 @@ class Bootstrap(object): user_options.append('--{0}={1}'.format(k, v)) elif isinstance(options, list): for opt in options: - if isinstance(opt, string_types) and option_is_allowed(opt): + if isinstance(opt, str) and option_is_allowed(opt): user_options.append('--{0}'.format(opt)) elif isinstance(opt, dict): keys = list(opt.keys()) - if len(keys) != 1 or not isinstance(opt[keys[0]], string_types) or not option_is_allowed(keys[0]): + if len(keys) != 1 or not isinstance(opt[keys[0]], str) or not option_is_allowed(keys[0]): error_handler('Error when parsing {0} key-value option {1}: only one key-value is allowed' ' and value should be a string'.format(tool, opt[keys[0]])) user_options.append('--{0}={1}'.format(keys[0], opt[keys[0]])) diff --git a/patroni/postgresql/citus.py b/patroni/postgresql/citus.py index 69fa96a7..2c7b1964 100644 --- a/patroni/postgresql/citus.py +++ b/patroni/postgresql/citus.py @@ -2,8 +2,8 @@ import logging import re import time -from six.moves.urllib_parse import urlparse from threading import Condition, Event, Thread +from urllib.parse import urlparse from .connection import Connection from ..dcs import CITUS_COORDINATOR_GROUP_ID diff --git a/patroni/postgresql/config.py b/patroni/postgresql/config.py index c995f177..97ee4dd5 100644 --- a/patroni/postgresql/config.py +++ b/patroni/postgresql/config.py @@ -6,7 +6,7 @@ import socket import stat import time -from six.moves.urllib_parse import urlparse, parse_qsl, unquote +from urllib.parse import urlparse, parse_qsl, unquote from .validator import CaseInsensitiveDict, recovery_parameters,\ transform_postgresql_parameter_value, transform_recovery_parameter_value @@ -768,9 +768,7 @@ class ConfigHandler(object): os.chmod(self._pgpass, stat.S_IWRITE | stat.S_IREAD) f.write(line) - env = os.environ.copy() - env['PGPASSFILE'] = self._pgpass - return env + return {**os.environ, 'PGPASSFILE': self._pgpass} def write_recovery_conf(self, recovery_params): self._recovery_params = recovery_params diff --git a/patroni/postgresql/rewind.py b/patroni/postgresql/rewind.py index 06c11a62..3dc5fcd1 100644 --- a/patroni/postgresql/rewind.py +++ b/patroni/postgresql/rewind.py @@ -3,7 +3,6 @@ import os import re import shlex import shutil -import six import subprocess from threading import Lock, Thread @@ -152,7 +151,7 @@ class Rewind(object): else: # otherwise analyze pg_controldata output in_recovery, timeline, lsn = self._get_local_timeline_lsn_from_controldata() - log_lsn = format_lsn(lsn) if isinstance(lsn, six.integer_types) else lsn + log_lsn = format_lsn(lsn) if isinstance(lsn, int) else lsn logger.info('Local timeline=%s lsn=%s', timeline, log_lsn) return in_recovery, timeline, lsn @@ -215,7 +214,7 @@ class Rewind(object): elif primary_timeline > 1: cur.execute('TIMELINE_HISTORY {0}'.format(primary_timeline)) history = cur.fetchone()[1] - if not isinstance(history, six.string_types): + if not isinstance(history, str): history = bytes(history).decode('utf-8') logger.debug('primary: history=%s', history) except Exception: diff --git a/patroni/postgresql/validator.py b/patroni/postgresql/validator.py index 26d121ab..61eb1bde 100644 --- a/patroni/postgresql/validator.py +++ b/patroni/postgresql/validator.py @@ -1,6 +1,5 @@ import abc import logging -import six from collections import namedtuple from urllib3.response import HTTPHeaderDict @@ -34,8 +33,7 @@ class Bool(namedtuple('Bool', 'version_from,version_till')): logger.warning('Removing bool parameter=%s from the config due to the invalid value=%s', name, value) -@six.add_metaclass(abc.ABCMeta) -class Number(namedtuple('Number', 'version_from,version_till,min_val,max_val,unit')): +class Number(abc.ABC, namedtuple('Number', 'version_from,version_till,min_val,max_val,unit')): @staticmethod @abc.abstractmethod diff --git a/patroni/request.py b/patroni/request.py index fce02e01..a027ec0f 100644 --- a/patroni/request.py +++ b/patroni/request.py @@ -1,8 +1,7 @@ import json import urllib3 -import six -from six.moves.urllib_parse import urlparse, urlunparse +from urllib.parse import urlparse, urlunparse from .utils import USER_AGENT @@ -51,7 +50,7 @@ class PatroniRequest(object): self._apply_pool_param('ca_certs', cacert) def request(self, method, url, body=None, **kwargs): - if body is not None and not isinstance(body, six.string_types): + if body is not None and not isinstance(body, str): body = json.dumps(body) return self._pool.request(method.upper(), url, body=body, **kwargs) diff --git a/patroni/utils.py b/patroni/utils.py index 1c9ed554..086617d5 100644 --- a/patroni/utils.py +++ b/patroni/utils.py @@ -514,21 +514,3 @@ def enable_keepalive(sock, timeout, idle, cnt=3): for opt in keepalive_socket_options(timeout, idle, cnt): sock.setsockopt(*opt) - - -def find_executable(executable, path=None): - _, ext = os.path.splitext(executable) - - if (sys.platform == 'win32') and (ext == ''): - executable = executable + '.exe' # Set default WIN extension - - if os.path.isfile(executable): - return executable - - if path is None: - path = os.environ.get('PATH', os.defpath) - - for p in path.split(os.pathsep): - f = os.path.join(p, executable) - if os.path.isfile(f): - return f diff --git a/patroni/validator.py b/patroni/validator.py index 5eb7b7f7..bb0641c6 100644 --- a/patroni/validator.py +++ b/patroni/validator.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 import os -import socket import re +import shutil +import socket import subprocess -from six import string_types - -from .utils import find_executable, split_host_port, data_directory_is_empty +from .utils import split_host_port, data_directory_is_empty from .dcs import dcs_modules from .exceptions import ConfigParseError @@ -173,7 +172,7 @@ class Directory(object): yield Result(False, "'{}' does not contain '{}'".format(name, path)) if self.contains_executable: for program in self.contains_executable: - if not find_executable(program, name): + if not shutil.which(program, path=name): yield Result(False, "'{}' does not contain '{}'".format(name, program)) @@ -190,12 +189,12 @@ class Schema(object): def validate(self, data): self.data = data - if isinstance(self.validator, string_types): - yield Result(isinstance(self.data, string_types), "is not a string", level=1, data=self.data) + if isinstance(self.validator, str): + yield Result(isinstance(self.data, str), "is not a string", level=1, data=self.data) elif issubclass(type(self.validator), type): validator = self.validator if self.validator == str: - validator = string_types + validator = str yield Result(isinstance(self.data, validator), "is not {}".format(_get_type_name(self.validator)), level=1, data=self.data) elif callable(self.validator): @@ -290,8 +289,8 @@ class Schema(object): def _get_type_name(python_type): - return {str: 'a string', int: 'and integer', float: 'a number', bool: 'a boolean', - list: 'an array', dict: 'a dictionary', string_types: "a string"}.get( + return {str: 'a string', int: 'and integer', float: 'a number', + bool: 'a boolean', list: 'an array', dict: 'a dictionary'}.get( python_type, getattr(python_type, __name__, "unknown type")) @@ -302,11 +301,11 @@ def assert_(condition, message="Wrong value"): userattributes = {"username": "", Optional("password"): ""} available_dcs = [m.split(".")[-1] for m in dcs_modules()] validate_host_port_list.expected_type = list -comma_separated_host_port.expected_type = string_types -validate_connect_address.expected_type = string_types -validate_host_port_listen.expected_type = string_types -validate_host_port_listen_multiple_hosts.expected_type = string_types -validate_data_dir.expected_type = string_types +comma_separated_host_port.expected_type = str +validate_connect_address.expected_type = str +validate_host_port_listen.expected_type = str +validate_host_port_listen_multiple_hosts.expected_type = str +validate_data_dir.expected_type = str validate_etcd = { Or("host", "hosts", "srv", "srv_suffix", "url", "proxy"): Case({ "host": validate_host_port, @@ -385,7 +384,7 @@ schema = Schema({ Optional("bin_dir"): Directory(contains_executable=["pg_ctl", "initdb", "pg_controldata", "pg_basebackup", "postgres", "pg_isready"]), Optional("parameters"): { - Optional("unix_socket_directories"): lambda s: assert_(all([isinstance(s, string_types), len(s)])) + Optional("unix_socket_directories"): lambda s: assert_(all([isinstance(s, str), len(s)])) }, Optional("pg_hba"): [str], Optional("pg_ident"): [str], diff --git a/patroni/watchdog/base.py b/patroni/watchdog/base.py index 51c14604..6ca57fc3 100644 --- a/patroni/watchdog/base.py +++ b/patroni/watchdog/base.py @@ -1,7 +1,6 @@ import abc import logging import platform -import six import sys from threading import RLock @@ -235,8 +234,7 @@ class Watchdog(object): return self.config.timing_slack >= 0 and self.impl.is_healthy -@six.add_metaclass(abc.ABCMeta) -class WatchdogBase(object): +class WatchdogBase(abc.ABC): """A watchdog object when opened requires periodic calls to keepalive. When keepalive is not called within a timeout the system will be terminated.""" is_null = False diff --git a/requirements.txt b/requirements.txt index 060d0dcd..aea3284f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,6 @@ urllib3>=1.19.1,!=1.21 -ipaddress; python_version=="2.7" boto3 PyYAML -six >= 1.7 kazoo>=1.3.1 python-etcd>=0.4.3,<0.5 python-consul>=0.7.1 diff --git a/tests/test_api.py b/tests/test_api.py index 6cce5baf..ab2e8f04 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -5,14 +5,16 @@ import socket import patroni.psycopg as psycopg +from http.server import HTTPServer +from io import BytesIO as IO from mock import Mock, PropertyMock, patch +from socketserver import ThreadingMixIn + from patroni.api import RestApiHandler, RestApiServer from patroni.dcs import ClusterConfig, Member from patroni.ha import _MemberStatus from patroni.utils import tzutc -from six import BytesIO as IO -from six.moves import BaseHTTPServer -from six.moves.socketserver import ThreadingMixIn + from . import psycopg_connect, MockCursor from .test_ha import get_cluster_initialized_without_leader @@ -175,7 +177,7 @@ class MockRestApiServer(RestApiServer): @patch('ssl.SSLContext.load_cert_chain', Mock()) @patch('ssl.SSLContext.wrap_socket', Mock(return_value=0)) -@patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock()) +@patch.object(HTTPServer, '__init__', Mock()) class TestRestApiHandler(unittest.TestCase): _authorization = '\nAuthorization: Basic dGVzdDp0ZXN0' @@ -587,14 +589,14 @@ class TestRestApiServer(unittest.TestCase): @patch('ssl.SSLContext.load_cert_chain', Mock()) @patch('ssl.SSLContext.set_ciphers', Mock()) @patch('ssl.SSLContext.wrap_socket', Mock(return_value=0)) - @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock()) + @patch.object(HTTPServer, '__init__', Mock()) def setUp(self): self.srv = MockRestApiServer(Mock(), '', {'listen': '*:8008', 'certfile': 'a', 'verify_client': 'required', 'ciphers': '!SSLv1:!SSLv2:!SSLv3:!TLSv1:!TLSv1.1', 'allowlist': ['127.0.0.1', '::1/128', '::1/zxc'], 'allowlist_include_members': True}) - @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock()) + @patch.object(HTTPServer, '__init__', Mock()) def test_reload_config(self): bad_config = {'listen': 'foo'} self.assertRaises(ValueError, MockRestApiServer, None, '', bad_config) @@ -622,7 +624,7 @@ class TestRestApiServer(unittest.TestCase): except Exception: self.assertIsNone(MockRestApiServer.handle_error(None, ('127.0.0.1', 55555))) - @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock(side_effect=socket.error)) + @patch.object(HTTPServer, '__init__', Mock(side_effect=socket.error)) def test_socket_error(self): self.assertRaises(socket.error, MockRestApiServer, Mock(), '', {'listen': '*:8008'}) diff --git a/tests/test_config.py b/tests/test_config.py index 172e8562..4a042945 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,14 +5,13 @@ import io from mock import MagicMock, Mock, patch from patroni.config import Config, ConfigParseError -from six.moves import builtins class TestConfig(unittest.TestCase): @patch('os.path.isfile', Mock(return_value=True)) @patch('json.load', Mock(side_effect=Exception)) - @patch.object(builtins, 'open', MagicMock()) + @patch('builtins.open', MagicMock()) def setUp(self): sys.argv = ['patroni.py'] os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}' @@ -137,7 +136,7 @@ class TestConfig(unittest.TestCase): new-attr: True ''') - with patch.object(builtins, 'open', MagicMock(side_effect=open_mock)): + with patch('builtins.open', MagicMock(side_effect=open_mock)): config = Config('postgres0') self.assertEqual(config._local_configuration, {'test': False, 'test2': {'child-1': 'somestring', 'child-2': 10}, diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 55964c9f..b8294956 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -559,8 +559,8 @@ class TestCtl(unittest.TestCase): @patch('sys.stdout.isatty', return_value=False) @patch('patroni.ctl.markup_to_pager') - @patch('patroni.ctl.find_executable', return_value=None) - def test_show_diff(self, mock_find_executable, mock_markup_to_pager, mock_isatty): + @patch('shutil.which', return_value=None) + def test_show_diff(self, mock_which, mock_markup_to_pager, mock_isatty): show_diff("foo:\n bar: 1\n", "foo:\n bar: 2\n") mock_markup_to_pager.assert_not_called() @@ -571,7 +571,7 @@ class TestCtl(unittest.TestCase): show_diff("foo:\n bar: 1\n", "foo:\n bar: 2\n") # Test that unicode handling doesn't fail with an exception - mock_find_executable.return_value = '/usr/bin/less' + mock_which.return_value = '/usr/bin/less' show_diff(b"foo:\n bar: \xc3\xb6\xc3\xb6\n".decode('utf-8'), b"foo:\n bar: \xc3\xbc\xc3\xbc\n".decode('utf-8')) @@ -579,7 +579,7 @@ class TestCtl(unittest.TestCase): def test_invoke_editor(self, mock_subprocess_call): os.environ.pop('EDITOR', None) for e in ('', '/bin/vi'): - with patch('patroni.ctl.find_executable', Mock(return_value=e)): + with patch('shutil.which', Mock(return_value=e)): self.assertRaises(PatroniCtlException, invoke_editor, 'foo: bar\n', 'test') @patch('patroni.ctl.get_dcs') diff --git a/tests/test_ha.py b/tests/test_ha.py index 10ae87af..78872623 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -18,7 +18,6 @@ from patroni.postgresql.rewind import Rewind from patroni.postgresql.slots import SlotsHandler from patroni.utils import tzutc from patroni.watchdog import Watchdog -from six.moves import builtins from . import PostgresInit, MockPostmaster, psycopg_connect, requests_get from .test_etcd import socket_getaddrinfo, etcd_read, etcd_write @@ -942,7 +941,7 @@ class TestHa(PostgresInit): self.assertEqual(self.ha.run_cycle(), 'PAUSE: waiting to become primary after promote...') @patch('patroni.postgresql.mtime', Mock(return_value=1588316884)) - @patch.object(builtins, 'open', mock_open(read_data='1\t0/40159C0\tno recovery target specified\n')) + @patch('builtins.open', mock_open(read_data='1\t0/40159C0\tno recovery target specified\n')) def test_process_healthy_standby_cluster_as_standby_leader(self): self.p.is_leader = false self.p.name = 'leader' @@ -1274,7 +1273,7 @@ class TestHa(PostgresInit): self.assertEqual(self.ha.get_effective_tags(), {'foo': 'bar'}) @patch('patroni.postgresql.mtime', Mock(return_value=1588316884)) - @patch.object(builtins, 'open', Mock(side_effect=Exception)) + @patch('builtins.open', Mock(side_effect=Exception)) def test_restore_cluster_config(self): self.ha.cluster.config.data.clear() self.ha.has_lock = true @@ -1327,8 +1326,8 @@ class TestHa(PostgresInit): "data directory is not accessible: [Errno 5] Input/output error: '{}'".format(self.p.data_dir)) @patch('patroni.postgresql.mtime', Mock(return_value=1588316884)) - @patch.object(builtins, 'open', mock_open(read_data=('1\t0/40159C0\tno recovery target specified\n\n' - '2\t1/40159C0\tno recovery target specified\n'))) + @patch('builtins.open', mock_open(read_data=('1\t0/40159C0\tno recovery target specified\n\n' + '2\t1/40159C0\tno recovery target specified\n'))) def test_update_cluster_history(self): self.ha.has_lock = true self.ha.cluster.is_unlocked = false @@ -1392,7 +1391,7 @@ class TestHa(PostgresInit): @patch('os.close', Mock()) @patch('os.rename', Mock()) @patch('patroni.postgresql.Postgresql.is_starting', Mock(return_value=False)) - @patch.object(builtins, 'open', mock_open()) + @patch('builtins.open', mock_open()) @patch.object(ConfigHandler, 'check_recovery_conf', Mock(return_value=(False, False))) @patch.object(Postgresql, 'major_version', PropertyMock(return_value=130000)) @patch.object(SlotsHandler, 'sync_replication_slots', Mock(return_value=['ls'])) diff --git a/tests/test_kubernetes.py b/tests/test_kubernetes.py index eacdcea1..f92f1f40 100644 --- a/tests/test_kubernetes.py +++ b/tests/test_kubernetes.py @@ -10,7 +10,6 @@ from mock import Mock, PropertyMock, mock_open, patch from patroni.dcs.kubernetes import Cluster, k8s_client, k8s_config, K8sConfig, K8sConnectionFailed,\ K8sException, K8sObject, Kubernetes, KubernetesError, KubernetesRetriableException,\ Retry, RetryFailedError, SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME -from six.moves import builtins from threading import Thread from . import MockResponse, SleepException @@ -85,7 +84,7 @@ class TestK8sConfig(unittest.TestCase): with patch('os.environ', {SERVICE_HOST_ENV_NAME: 'a', SERVICE_PORT_ENV_NAME: '1'}),\ patch('os.path.isfile', Mock(side_effect=[False, True, True, False, True, True, True, True])),\ - patch.object(builtins, 'open', Mock(side_effect=[ + patch('builtins.open', Mock(side_effect=[ mock_open()(), mock_open(read_data='a')(), mock_open(read_data='a')(), mock_open()(), mock_open(read_data='a')(), mock_open(read_data='a')()])): for _ in range(0, 4): @@ -97,7 +96,7 @@ class TestK8sConfig(unittest.TestCase): def test_refresh_token(self): with patch('os.environ', {SERVICE_HOST_ENV_NAME: 'a', SERVICE_PORT_ENV_NAME: '1'}),\ patch('os.path.isfile', Mock(side_effect=[True, True, False, True, True, True])),\ - patch.object(builtins, 'open', Mock(side_effect=[ + patch('builtins.open', Mock(side_effect=[ mock_open(read_data='cert')(), mock_open(read_data='a')(), mock_open()(), mock_open(read_data='b')(), mock_open(read_data='c')()])): k8s_config.load_incluster_config(token_refresh_interval=datetime.timedelta(milliseconds=100)) @@ -122,20 +121,20 @@ class TestK8sConfig(unittest.TestCase): "clusters": [{"name": "local", "cluster": {"server": "https://a:1/", "certificate-authority": "a"}}], "users": [{"name": "local", "user": {"username": "a", "password": "b", "client-certificate": "c"}}] } - with patch.object(builtins, 'open', mock_open(read_data=json.dumps(config))): + with patch('builtins.open', mock_open(read_data=json.dumps(config))): k8s_config.load_kube_config() self.assertEqual(k8s_config.server, 'https://a:1') self.assertEqual(k8s_config.pool_config, {'ca_certs': 'a', 'cert_file': 'c', 'cert_reqs': 'CERT_REQUIRED', 'maxsize': 10, 'num_pools': 10}) config["users"][0]["user"]["token"] = "token" - with patch.object(builtins, 'open', mock_open(read_data=json.dumps(config))): + with patch('builtins.open', mock_open(read_data=json.dumps(config))): k8s_config.load_kube_config() self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer token') config["users"][0]["user"]["client-key-data"] = base64.b64encode(b'foobar').decode('utf-8') config["clusters"][0]["cluster"]["certificate-authority-data"] = base64.b64encode(b'foobar').decode('utf-8') - with patch.object(builtins, 'open', mock_open(read_data=json.dumps(config))),\ + with patch('builtins.open', mock_open(read_data=json.dumps(config))),\ patch('os.write', Mock()), patch('os.close', Mock()),\ patch('os.remove') as mock_remove,\ patch('atexit.register') as mock_atexit,\ diff --git a/tests/test_log.py b/tests/test_log.py index e5482c67..1a383908 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -7,7 +7,7 @@ import yaml from mock import Mock, patch from patroni.config import Config from patroni.log import PatroniLogger -from six.moves.queue import Queue, Full +from queue import Queue, Full _LOG = logging.getLogger(__name__) diff --git a/tests/test_patroni.py b/tests/test_patroni.py index 2b2094a0..a3706e78 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -6,6 +6,7 @@ import time import unittest import patroni.config as config +from http.server import HTTPServer from mock import Mock, PropertyMock, patch from patroni.api import RestApiServer from patroni.async_executor import AsyncExecutor @@ -15,7 +16,6 @@ from patroni.postgresql import Postgresql from patroni.postgresql.config import ConfigHandler from patroni import check_psycopg from patroni.__main__ import Patroni, main as _main, patroni_main -from six.moves import BaseHTTPServer, builtins from threading import Thread from . import psycopg_connect, SleepException @@ -44,7 +44,7 @@ class MockFrozenImporter(object): @patch.object(ConfigHandler, 'write_recovery_conf', Mock()) @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster())) @patch.object(Postgresql, 'call_nowait', Mock()) -@patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock()) +@patch.object(HTTPServer, '__init__', Mock()) @patch.object(AsyncExecutor, 'run', Mock()) @patch.object(etcd.Client, 'write', etcd_write) @patch.object(etcd.Client, 'read', etcd_read) @@ -63,7 +63,7 @@ class TestPatroni(unittest.TestCase): @patch('pkgutil.iter_importers', Mock(return_value=[MockFrozenImporter()])) @patch('sys.frozen', Mock(return_value=True), create=True) - @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock()) + @patch.object(HTTPServer, '__init__', Mock()) @patch.object(etcd.Client, 'read', etcd_read) @patch.object(Thread, 'start', Mock()) @patch.object(AbstractEtcdClientWithFailover, 'machines', PropertyMock(return_value=['http://remotehost:2379'])) @@ -196,7 +196,7 @@ class TestPatroni(unittest.TestCase): self.p.shutdown() def test_check_psycopg(self): - with patch.object(builtins, '__import__', Mock(side_effect=ImportError)): + with patch('builtins.__import__', Mock(side_effect=ImportError)): self.assertRaises(SystemExit, check_psycopg) - with patch.object(builtins, '__import__', mock_import): + with patch('builtins.__import__', mock_import): self.assertRaises(SystemExit, check_psycopg) diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index 29dcbe93..64c534ca 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -17,7 +17,6 @@ from patroni.postgresql.bootstrap import Bootstrap from patroni.postgresql.callback_executor import CallbackAction from patroni.postgresql.postmaster import PostmasterProcess from patroni.utils import RetryFailedError -from six.moves import builtins from threading import Thread, current_thread from . import BaseTestPostgresql, MockCursor, MockPostmaster, psycopg_connect @@ -231,7 +230,7 @@ class TestPostgresql(BaseTestPostgresql): self.assertEqual(self.p.state, 'restart failed (restarting)') @patch('os.chmod', Mock()) - @patch.object(builtins, 'open', MagicMock()) + @patch('builtins.open', MagicMock()) def test_write_pgpass(self): self.p.config.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo'}) self.p.config.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'}) @@ -312,11 +311,11 @@ class TestPostgresql(BaseTestPostgresql): mock_read_auto = mock_open(read_data=read_data) mock_read_auto.return_value.__iter__ = lambda o: iter(o.readline, '') - with patch.object(builtins, 'open', Mock(side_effect=[mock_open()(), mock_read_auto(), IOError])),\ + with patch('builtins.open', Mock(side_effect=[mock_open()(), mock_read_auto(), IOError])),\ patch('os.chmod', Mock()): self.p.config.write_postgresql_conf() - with patch.object(builtins, 'open', Mock(side_effect=[mock_open()(), IOError])), patch('os.chmod', Mock()): + with patch('builtins.open', Mock(side_effect=[mock_open()(), IOError])), patch('os.chmod', Mock()): self.p.config.write_postgresql_conf() self.p.config.write_recovery_conf({'foo': 'bar'}) self.p.config.write_postgresql_conf() @@ -552,9 +551,9 @@ class TestPostgresql(BaseTestPostgresql): @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True)) def test_get_major_version(self): - with patch.object(builtins, 'open', mock_open(read_data='9.4')): + with patch('builtins.open', mock_open(read_data='9.4')): self.assertEqual(self.p.get_major_version(), 90400) - with patch.object(builtins, 'open', Mock(side_effect=Exception)): + with patch('builtins.open', Mock(side_effect=Exception)): self.assertEqual(self.p.get_major_version(), 0) def test_postmaster_start_time(self): diff --git a/tests/test_postmaster.py b/tests/test_postmaster.py index 61a45049..69e790d2 100644 --- a/tests/test_postmaster.py +++ b/tests/test_postmaster.py @@ -4,7 +4,6 @@ import unittest from mock import Mock, patch, mock_open from patroni.postgresql.postmaster import PostmasterProcess -from six.moves import builtins class MockProcess(object): @@ -169,7 +168,7 @@ class TestPostmasterProcess(unittest.TestCase): @patch('psutil.Process.__init__', Mock(side_effect=psutil.NoSuchProcess(123))) def test_read_postmaster_pidfile(self): - with patch.object(builtins, 'open', Mock(side_effect=IOError)): + with patch('builtins.open', Mock(side_effect=IOError)): self.assertIsNone(PostmasterProcess.from_pidfile('')) - with patch.object(builtins, 'open', mock_open(read_data='123\n')): + with patch('builtins.open', mock_open(read_data='123\n')): self.assertIsNone(PostmasterProcess.from_pidfile('')) diff --git a/tests/test_rewind.py b/tests/test_rewind.py index 63010bb2..e6a79026 100644 --- a/tests/test_rewind.py +++ b/tests/test_rewind.py @@ -3,7 +3,6 @@ from mock import Mock, PropertyMock, patch, mock_open from patroni.postgresql import Postgresql from patroni.postgresql.cancellable import CancellableSubprocess from patroni.postgresql.rewind import Rewind -from six.moves import builtins from . import BaseTestPostgresql, MockCursor, psycopg_connect @@ -193,7 +192,7 @@ class TestRewind(BaseTestPostgresql): m = mock_open(read_data='/usr/lib/postgres/9.6/bin/postgres "-D" "data/postgresql0" \ "--listen_addresses=127.0.0.1" "--port=5432" "--hot_standby=on" "--wal_level=hot_standby" \ "--wal_log_hints=on" "--max_wal_senders=5" "--max_replication_slots=5"\n') - with patch.object(builtins, 'open', m): + with patch('builtins.open', m): data = self.r.read_postmaster_opts() self.assertEqual(data['wal_level'], 'hot_standby') self.assertEqual(int(data['max_replication_slots']), 5) diff --git a/tests/test_utils.py b/tests/test_utils.py index 80bad3e5..0c6b21e4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,7 +2,7 @@ import unittest from mock import Mock, patch from patroni.exceptions import PatroniException -from patroni.utils import Retry, RetryFailedError, enable_keepalive, find_executable, polling_loop, validate_directory +from patroni.utils import Retry, RetryFailedError, enable_keepalive, polling_loop, validate_directory class TestUtils(unittest.TestCase): @@ -41,15 +41,6 @@ class TestUtils(unittest.TestCase): with patch('sys.platform', platform): self.assertIsNone(enable_keepalive(Mock(), 10, 5)) - @patch('sys.platform', 'win32') - def test_find_executable(self): - with patch('os.path.isfile', Mock(return_value=True)): - self.assertEqual(find_executable('vim'), 'vim.exe') - with patch('os.path.isfile', Mock(return_value=False)): - self.assertIsNone(find_executable('vim')) - with patch('os.path.isfile', Mock(side_effect=[False, True])): - self.assertEqual(find_executable('vim', '/'), '/vim.exe') - @patch('time.sleep', Mock()) class TestRetrySleeper(unittest.TestCase): diff --git a/tests/test_validator.py b/tests/test_validator.py index 9f4d261c..1558a17c 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -4,10 +4,10 @@ import socket import tempfile import unittest +from io import StringIO from mock import Mock, patch, mock_open from patroni.dcs import dcs_modules from patroni.validator import schema -from six import StringIO available_dcs = [m.split(".")[-1] for m in dcs_modules()] config = { @@ -94,14 +94,18 @@ config = { directories = [] files = [] +binaries = [] def isfile_side_effect(arg): - if arg.endswith('.exe'): - arg = arg[:-4] return arg in files +def which_side_effect(arg, path=None): + binary = arg if path is None else os.path.join(path, arg) + return arg if binary in binaries else None + + def isdir_side_effect(arg): return arg in directories @@ -134,6 +138,7 @@ def parse_output(output): @patch('os.path.exists', Mock(side_effect=exists_side_effect)) @patch('os.path.isdir', Mock(side_effect=isdir_side_effect)) @patch('os.path.isfile', Mock(side_effect=isfile_side_effect)) +@patch('shutil.which', Mock(side_effect=which_side_effect)) @patch('sys.stderr', new_callable=StringIO) @patch('sys.stdout', new_callable=StringIO) class TestValidator(unittest.TestCase): @@ -141,6 +146,7 @@ class TestValidator(unittest.TestCase): def setUp(self): del files[:] del directories[:] + del binaries[:] def test_empty_config(self, mock_out, mock_err): errors = schema({}) @@ -191,12 +197,12 @@ class TestValidator(unittest.TestCase): directories.append(os.path.join(config["postgresql"]["data_dir"], "pg_wal")) files.append(os.path.join(config["postgresql"]["data_dir"], "global", "pg_control")) files.append(os.path.join(config["postgresql"]["data_dir"], "PG_VERSION")) - files.append(os.path.join(config["postgresql"]["bin_dir"], "pg_ctl")) - files.append(os.path.join(config["postgresql"]["bin_dir"], "initdb")) - files.append(os.path.join(config["postgresql"]["bin_dir"], "pg_controldata")) - files.append(os.path.join(config["postgresql"]["bin_dir"], "pg_basebackup")) - files.append(os.path.join(config["postgresql"]["bin_dir"], "postgres")) - files.append(os.path.join(config["postgresql"]["bin_dir"], "pg_isready")) + binaries.append(os.path.join(config["postgresql"]["bin_dir"], "pg_ctl")) + binaries.append(os.path.join(config["postgresql"]["bin_dir"], "initdb")) + binaries.append(os.path.join(config["postgresql"]["bin_dir"], "pg_controldata")) + binaries.append(os.path.join(config["postgresql"]["bin_dir"], "pg_basebackup")) + binaries.append(os.path.join(config["postgresql"]["bin_dir"], "postgres")) + binaries.append(os.path.join(config["postgresql"]["bin_dir"], "pg_isready")) with patch('patroni.validator.open', mock_open(read_data='12')): errors = schema(config) output = "\n".join(errors) diff --git a/tests/test_wale_restore.py b/tests/test_wale_restore.py index 2aff0e09..568073c9 100644 --- a/tests/test_wale_restore.py +++ b/tests/test_wale_restore.py @@ -6,7 +6,6 @@ import patroni.psycopg as psycopg from mock import Mock, PropertyMock, patch, mock_open from patroni.scripts import wale_restore from patroni.scripts.wale_restore import WALERestore, main as _main, get_major_version -from six.moves import builtins from threading import current_thread from . import MockConnect, psycopg_connect @@ -128,9 +127,9 @@ class TestWALERestore(unittest.TestCase): @patch('os.path.isfile', Mock(return_value=True)) def test_get_major_version(self): - with patch.object(builtins, 'open', mock_open(read_data='9.4')): + with patch('builtins.open', mock_open(read_data='9.4')): self.assertEqual(get_major_version("data"), 9.4) - with patch.object(builtins, 'open', side_effect=OSError): + with patch('builtins.open', side_effect=OSError): self.assertEqual(get_major_version("data"), 0.0) @patch('os.path.islink', Mock(return_value=True)) diff --git a/tests/test_zookeeper.py b/tests/test_zookeeper.py index d98fe77f..0fa0817b 100644 --- a/tests/test_zookeeper.py +++ b/tests/test_zookeeper.py @@ -1,5 +1,4 @@ import select -import six import unittest from kazoo.client import KazooClient, KazooState @@ -30,7 +29,7 @@ class MockKazooClient(Mock): return func(*args, **kwargs) def get(self, path, watch=None): - if not isinstance(path, six.string_types): + if not isinstance(path, str): raise TypeError("Invalid type for 'path' (string expected)") if path == '/broken/status': return (b'{', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0)) @@ -57,7 +56,7 @@ class MockKazooClient(Mock): @staticmethod def get_children(path, watch=None, include_data=False): - if not isinstance(path, six.string_types): + if not isinstance(path, str): raise TypeError("Invalid type for 'path' (string expected)") if path.startswith('/no_node'): raise NoNodeError @@ -66,9 +65,9 @@ class MockKazooClient(Mock): return ['foo', 'bar', 'buzz'] def create(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False): - if not isinstance(path, six.string_types): + if not isinstance(path, str): raise TypeError("Invalid type for 'path' (string expected)") - if not isinstance(value, (six.binary_type,)): + if not isinstance(value, bytes): raise TypeError("Invalid type for 'value' (must be a byte string)") if b'Exception' in value: raise Exception @@ -82,9 +81,9 @@ class MockKazooClient(Mock): @staticmethod def set(path, value, version=-1): - if not isinstance(path, six.string_types): + if not isinstance(path, str): raise TypeError("Invalid type for 'path' (string expected)") - if not isinstance(value, (six.binary_type,)): + if not isinstance(value, bytes): raise TypeError("Invalid type for 'value' (must be a byte string)") if path == '/service/bla/optime/leader': raise Exception @@ -101,7 +100,7 @@ class MockKazooClient(Mock): return self.set(path, value, version) or Mock() def delete(self, path, version=-1, recursive=False): - if not isinstance(path, six.string_types): + if not isinstance(path, str): raise TypeError("Invalid type for 'path' (string expected)") self.exists = False if path == '/service/test/leader':