From 93eb4edbe6b82e36cad06879d8b308b651fcd3fd Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 13 Aug 2024 17:53:59 +0200 Subject: [PATCH] Reformat imports with isort (#3123) Besides that: 1. Introduce `setup.py isort` for quick check 2. Introduce GH actions to check imports --- .github/workflows/tests.yaml | 17 ++++++ features/archive-restore.py | 2 +- features/callback2.py | 1 + features/environment.py | 9 ++- features/steps/basic_replication.py | 6 +- features/steps/citus.py | 7 ++- features/steps/patroni_api.py | 6 +- features/steps/slots.py | 1 + features/steps/watchdog.py | 3 +- patroni/__main__.py | 4 +- patroni/api.py | 16 +++--- patroni/config.py | 9 +-- patroni/config_generator.py | 11 ++-- patroni/ctl.py | 21 ++++--- patroni/dcs/__init__.py | 6 +- patroni/dcs/consul.py | 16 ++++-- patroni/dcs/etcd.py | 22 +++++--- patroni/dcs/etcd3.py | 15 +++-- patroni/dcs/exhibitor.py | 4 +- patroni/dcs/kubernetes.py | 17 +++--- patroni/dcs/raft.py | 10 ++-- patroni/dcs/zookeeper.py | 10 ++-- patroni/dynamic_loader.py | 4 +- patroni/global_config.py | 2 +- patroni/ha.py | 6 +- patroni/log.py | 9 ++- patroni/postgresql/__init__.py | 19 ++++--- patroni/postgresql/bootstrap.py | 2 +- patroni/postgresql/cancellable.py | 8 ++- patroni/postgresql/config.py | 12 ++-- patroni/postgresql/connection.py | 3 +- patroni/postgresql/mpp/__init__.py | 4 +- patroni/postgresql/mpp/citus.py | 6 +- patroni/postgresql/postmaster.py | 7 ++- patroni/postgresql/rewind.py | 8 +-- patroni/postgresql/slots.py | 8 ++- patroni/postgresql/sync.py | 1 + patroni/postgresql/validator.py | 7 ++- patroni/psycopg.py | 7 ++- patroni/raft_controller.py | 2 +- patroni/request.py | 4 +- patroni/scripts/aws.py | 5 +- patroni/scripts/barman/cli.py | 5 +- patroni/scripts/barman/config_switch.py | 6 +- patroni/scripts/barman/recover.py | 6 +- patroni/scripts/barman/utils.py | 5 +- patroni/tags.py | 2 +- patroni/utils.py | 8 +-- patroni/validator.py | 4 +- patroni/watchdog/__init__.py | 3 +- patroni/watchdog/base.py | 2 +- setup.py | 74 ++++++++++++++++++------- tests/__init__.py | 3 +- tests/test_api.py | 5 +- tests/test_async_executor.py | 3 +- tests/test_aws.py | 8 ++- tests/test_barman.py | 9 +-- tests/test_bootstrap.py | 5 +- tests/test_callback_executor.py | 4 +- tests/test_cancellable.py | 4 +- tests/test_citus.py | 2 +- tests/test_config.py | 2 +- tests/test_config_generator.py | 12 ++-- tests/test_consul.py | 12 ++-- tests/test_ctl.py | 27 +++++---- tests/test_etcd.py | 15 +++-- tests/test_etcd3.py | 16 +++--- tests/test_exhibitor.py | 8 ++- tests/test_file_perm.py | 3 +- tests/test_ha.py | 16 +++--- tests/test_kubernetes.py | 13 +++-- tests/test_log.py | 6 +- tests/test_mpp.py | 1 + tests/test_patroni.py | 13 +++-- tests/test_postgresql.py | 25 ++++----- tests/test_postmaster.py | 6 +- tests/test_quorum.py | 2 +- tests/test_raft.py | 12 ++-- tests/test_raft_controller.py | 4 +- tests/test_rewind.py | 2 +- tests/test_slots.py | 5 +- tests/test_sync.py | 2 +- tests/test_utils.py | 3 +- tests/test_validator.py | 5 +- tests/test_wale_restore.py | 6 +- tests/test_watchdog.py | 8 ++- tests/test_zookeeper.py | 6 +- tox.ini | 9 +++ 88 files changed, 447 insertions(+), 287 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d6e18ba2..dacfb0fb 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -211,3 +211,20 @@ jobs: - name: Generate documentation run: tox -m docs + + isort: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: 3.12 + cache: pip + + - name: isort + uses: isort/isort-action@master + with: + requirementsFiles: "requirements.txt requirements.dev.txt requirements.docs.txt" + sort-paths: "patroni tests features setup.py" diff --git a/features/archive-restore.py b/features/archive-restore.py index 91c216db..8b02154c 100644 --- a/features/archive-restore.py +++ b/features/archive-restore.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import os import argparse +import os import shutil if __name__ == "__main__": diff --git a/features/callback2.py b/features/callback2.py index 6dad933a..1afcc14c 100755 --- a/features/callback2.py +++ b/features/callback2.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import sys + with open("data/{0}/{0}_cb.log".format(sys.argv[1]), "a+") as log: log.write(" ".join(sys.argv[-3:]) + "\n") diff --git a/features/environment.py b/features/environment.py index 9f9d5fa7..fb4220d0 100644 --- a/features/environment.py +++ b/features/environment.py @@ -1,9 +1,8 @@ import abc import datetime import glob -import os import json -import psutil +import os import re import shutil import signal @@ -13,11 +12,14 @@ import sys import tempfile import threading import time + +from http.server import BaseHTTPRequestHandler, HTTPServer + +import psutil import yaml import patroni.psycopg as psycopg -from http.server import BaseHTTPRequestHandler, HTTPServer from patroni.request import PatroniRequest @@ -498,6 +500,7 @@ class AbstractEtcdController(AbstractDcsController): def _is_running(self): from patroni.dcs.etcd import DnsCachingResolver + # if etcd is running, but we didn't start it try: self._client = self._client_cls({'host': 'localhost', 'port': 2379, 'retry_timeout': 30, diff --git a/features/steps/basic_replication.py b/features/steps/basic_replication.py index 6cb0ac42..3ec61ca6 100644 --- a/features/steps/basic_replication.py +++ b/features/steps/basic_replication.py @@ -1,8 +1,10 @@ import json -import patroni.psycopg as pg + +from time import sleep, time from behave import step, then -from time import sleep, time + +import patroni.psycopg as pg @step('I start {name:w}') diff --git a/features/steps/citus.py b/features/steps/citus.py index 7277cccc..274a8b51 100644 --- a/features/steps/citus.py +++ b/features/steps/citus.py @@ -1,11 +1,12 @@ import json import time -from behave import step, then -from dateutil import tz from datetime import datetime from functools import partial -from threading import Thread, Event +from threading import Event, Thread + +from behave import step, then +from dateutil import tz tzutc = tz.tzutc() diff --git a/features/steps/patroni_api.py b/features/steps/patroni_api.py index 74a7c0da..ded5ce2a 100644 --- a/features/steps/patroni_api.py +++ b/features/steps/patroni_api.py @@ -1,14 +1,16 @@ import json -import parse import shlex import subprocess import sys import time + +from datetime import datetime, timedelta + +import parse import yaml from behave import register_type, step, then from dateutil import tz -from datetime import datetime, timedelta tzutc = tz.tzutc() diff --git a/features/steps/slots.py b/features/steps/slots.py index 182aa87c..84d7e350 100644 --- a/features/steps/slots.py +++ b/features/steps/slots.py @@ -2,6 +2,7 @@ import json import time from behave import step, then + import patroni.psycopg as pg diff --git a/features/steps/watchdog.py b/features/steps/watchdog.py index 05eba73c..ebec646e 100644 --- a/features/steps/watchdog.py +++ b/features/steps/watchdog.py @@ -1,6 +1,7 @@ -from behave import step, then import time +from behave import step, then + def polling_loop(timeout, interval=1): """Returns an iterator that returns values until timeout has passed. Timeout is measured from start of iteration.""" diff --git a/patroni/__main__.py b/patroni/__main__.py index c4c9a497..04d53c9e 100644 --- a/patroni/__main__.py +++ b/patroni/__main__.py @@ -13,7 +13,7 @@ from argparse import Namespace from typing import Any, Dict, List, Optional, TYPE_CHECKING from patroni import MIN_PSYCOPG2, MIN_PSYCOPG3, parse_version -from patroni.daemon import AbstractPatroniDaemon, abstract_main, get_base_arg_parser +from patroni.daemon import abstract_main, AbstractPatroniDaemon, get_base_arg_parser from patroni.tags import Tags if TYPE_CHECKING: # pragma: no cover @@ -268,8 +268,8 @@ def process_arguments() -> Namespace: generate_config(args.configfile, False, args.dsn) sys.exit(0) elif args.validate_config: - from patroni.validator import schema from patroni.config import Config, ConfigParseError + from patroni.validator import schema try: Config(args.configfile, validator=schema) diff --git a/patroni/api.py b/patroni/api.py index 6f3c1714..d7425326 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -7,32 +7,32 @@ utilises the API to perform these functions. """ import base64 +import datetime import hmac import json import logging -import time -import traceback -import dateutil.parser -import datetime import os import socket import sys +import time +import traceback from http.server import BaseHTTPRequestHandler, HTTPServer from ipaddress import ip_address, ip_network, IPv4Network, IPv6Network from socketserver import ThreadingMixIn from threading import Thread -from urllib.parse import urlparse, parse_qs - from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union +from urllib.parse import parse_qs, urlparse + +import dateutil.parser from . import global_config, psycopg from .__main__ import Patroni from .dcs import Cluster from .exceptions import PostgresConnectionException, PostgresException from .postgresql.misc import postgres_version_to_int -from .utils import deep_compare, enable_keepalive, parse_bool, patch_config, Retry, \ - RetryFailedError, parse_int, split_host_port, tzutc, uri, cluster_as_json +from .utils import cluster_as_json, deep_compare, enable_keepalive, parse_bool, \ + parse_int, patch_config, Retry, RetryFailedError, split_host_port, tzutc, uri logger = logging.getLogger(__name__) diff --git a/patroni/config.py b/patroni/config.py index eeac3f1f..88f3a1a6 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -1,15 +1,16 @@ """Facilities related to Patroni configuration.""" -import re import json import logging import os +import re import shutil import tempfile -import yaml from collections import defaultdict from copy import deepcopy -from typing import Any, Callable, Collection, Dict, List, Optional, Union, TYPE_CHECKING +from typing import Any, Callable, Collection, Dict, List, Optional, TYPE_CHECKING, Union + +import yaml from . import PATRONI_ENV_PREFIX from .collections import CaseInsensitiveDict, EMPTY_DICT @@ -17,8 +18,8 @@ from .dcs import ClusterConfig from .exceptions import ConfigParseError from .file_perm import pg_perm from .postgresql.config import ConfigHandler -from .validator import IntValidator from .utils import deep_compare, parse_bool, parse_int, patch_config +from .validator import IntValidator logger = logging.getLogger(__name__) diff --git a/patroni/config_generator.py b/patroni/config_generator.py index 8a1d6078..4977962d 100644 --- a/patroni/config_generator.py +++ b/patroni/config_generator.py @@ -2,14 +2,16 @@ import abc import logging import os -import psutil import socket import sys + +from contextlib import contextmanager +from getpass import getpass, getuser +from typing import Any, Dict, Iterator, List, Optional, TextIO, Tuple, TYPE_CHECKING, Union + +import psutil import yaml -from getpass import getuser, getpass -from contextlib import contextmanager -from typing import Any, Dict, Iterator, List, Optional, TextIO, Tuple, TYPE_CHECKING, Union if TYPE_CHECKING: # pragma: no cover from psycopg import Cursor from psycopg2 import cursor @@ -23,7 +25,6 @@ from .postgresql.config import ConfigHandler, parse_dsn from .postgresql.misc import postgres_major_version_to_int from .utils import get_major_version, parse_bool, patch_config, read_stripped - # Mapping between the libpq connection parameters and the environment variables. # This dict should be kept in sync with `patroni.utils._AUTH_ALLOWED_PARAMETERS` # (we use "username" in the Patroni config for some reason, other parameter names are the same). diff --git a/patroni/ctl.py b/patroni/ctl.py index 4e05ab8d..dd813a77 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -12,12 +12,9 @@ If it is also missing in the configuration file we assume that this is just a normal Patroni cluster (not Citus). """ -import click import codecs import copy import datetime -import dateutil.parser -import dateutil.tz import difflib import io import json @@ -28,15 +25,21 @@ import shutil import subprocess import sys import tempfile -import urllib3 import time -import yaml from collections import defaultdict from contextlib import contextmanager -from prettytable import ALL, FRAME, PrettyTable +from typing import Any, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union from urllib.parse import urlparse -from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING + +import click +import dateutil.parser +import dateutil.tz +import urllib3 +import yaml + +from prettytable import ALL, FRAME, PrettyTable + if TYPE_CHECKING: # pragma: no cover from psycopg import Cursor from psycopg2 import cursor @@ -52,12 +55,12 @@ except ImportError: # pragma: no cover from . import global_config from .config import Config -from .dcs import get_dcs as _get_dcs, AbstractDCS, Cluster, Member +from .dcs import AbstractDCS, Cluster, get_dcs as _get_dcs, Member from .exceptions import PatroniException from .postgresql.misc import postgres_version_to_int from .postgresql.mpp import get_mpp -from .utils import cluster_as_json, patch_config, polling_loop from .request import PatroniRequest +from .utils import cluster_as_json, patch_config, polling_loop from .version import __version__ CONFIG_DIR_PATH = click.get_app_dir('patroni') diff --git a/patroni/dcs/__init__.py b/patroni/dcs/__init__.py index 3c50803e..c41b7904 100644 --- a/patroni/dcs/__init__.py +++ b/patroni/dcs/__init__.py @@ -5,22 +5,22 @@ import json import logging import re import time + from collections import defaultdict from copy import deepcopy from random import randint from threading import Event, Lock from typing import Any, Callable, Collection, Dict, Iterator, List, \ NamedTuple, Optional, Set, Tuple, Type, TYPE_CHECKING, Union -from urllib.parse import urlparse, urlunparse, parse_qsl +from urllib.parse import parse_qsl, urlparse, urlunparse import dateutil.parser from .. import global_config from ..dynamic_loader import iter_classes, iter_modules from ..exceptions import PatroniFatalException -from ..utils import deep_compare, uri from ..tags import Tags -from ..utils import parse_int +from ..utils import deep_compare, parse_int, uri if TYPE_CHECKING: # pragma: no cover from ..config import Config diff --git a/patroni/dcs/consul.py b/patroni/dcs/consul.py index deedfdb1..6a8829db 100644 --- a/patroni/dcs/consul.py +++ b/patroni/dcs/consul.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + import json import logging import os @@ -6,20 +7,23 @@ import re import socket import ssl import time -import urllib3 from collections import defaultdict -from consul import ConsulException, NotFound, base from http.client import HTTPException +from typing import Any, Callable, Dict, List, Mapping, NamedTuple, Optional, Tuple, TYPE_CHECKING, Union +from urllib.parse import quote, urlencode, urlparse + +import urllib3 + +from consul import base, ConsulException, NotFound from urllib3.exceptions import HTTPError -from urllib.parse import urlencode, urlparse, quote -from typing import Any, Callable, Dict, List, Mapping, NamedTuple, Optional, Union, Tuple, TYPE_CHECKING -from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, \ - TimelineHistory, ReturnFalseException, catch_return_false_exception from ..exceptions import DCSError from ..postgresql.mpp import AbstractMPP from ..utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port, uri, USER_AGENT +from . import AbstractDCS, catch_return_false_exception, Cluster, ClusterConfig, \ + Failover, Leader, Member, ReturnFalseException, Status, SyncState, TimelineHistory + if TYPE_CHECKING: # pragma: no cover from ..config import Config diff --git a/patroni/dcs/etcd.py b/patroni/dcs/etcd.py index 0b6c6dd3..5b9365a8 100644 --- a/patroni/dcs/etcd.py +++ b/patroni/dcs/etcd.py @@ -1,32 +1,36 @@ from __future__ import absolute_import + import abc -import etcd import json import logging import os -import urllib3.util.connection import random import socket import time from collections import defaultdict from copy import deepcopy -from dns.exception import DNSException -from dns import resolver from http.client import HTTPException from queue import Queue from threading import Thread -from typing import Any, Callable, Collection, Dict, List, Optional, Union, Tuple, Type, TYPE_CHECKING +from typing import Any, Callable, Collection, Dict, List, Optional, Tuple, Type, TYPE_CHECKING, Union from urllib.parse import urlparse -from urllib3 import Timeout -from urllib3.exceptions import HTTPError, ReadTimeoutError, ProtocolError -from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, \ - TimelineHistory, ReturnFalseException, catch_return_false_exception +import etcd +import urllib3.util.connection + +from dns import resolver +from dns.exception import DNSException +from urllib3 import Timeout +from urllib3.exceptions import HTTPError, ProtocolError, ReadTimeoutError + from ..exceptions import DCSError from ..postgresql.mpp import AbstractMPP from ..request import get as requests_get from ..utils import Retry, RetryFailedError, split_host_port, uri, USER_AGENT +from . import AbstractDCS, catch_return_false_exception, Cluster, ClusterConfig, \ + Failover, Leader, Member, ReturnFalseException, Status, SyncState, TimelineHistory + if TYPE_CHECKING: # pragma: no cover from ..config import Config diff --git a/patroni/dcs/etcd3.py b/patroni/dcs/etcd3.py index 4a840c7f..1539b9ea 100644 --- a/patroni/dcs/etcd3.py +++ b/patroni/dcs/etcd3.py @@ -1,26 +1,29 @@ from __future__ import absolute_import + import base64 -import etcd import json import logging import os import socket import sys import time -import urllib3 from collections import defaultdict from enum import IntEnum -from urllib3.exceptions import ReadTimeoutError, ProtocolError from threading import Condition, Lock, Thread from typing import Any, Callable, Collection, Dict, Iterator, List, Optional, Tuple, Type, TYPE_CHECKING, Union -from . import ClusterConfig, Cluster, Failover, Leader, Member, Status, SyncState, \ - TimelineHistory, catch_return_false_exception -from .etcd import AbstractEtcdClientWithFailover, AbstractEtcd, catch_etcd_errors, DnsCachingResolver, Retry +import etcd +import urllib3 + +from urllib3.exceptions import ProtocolError, ReadTimeoutError + from ..exceptions import DCSError, PatroniException from ..postgresql.mpp import AbstractMPP from ..utils import deep_compare, enable_keepalive, iter_response_objects, RetryFailedError, USER_AGENT +from . import catch_return_false_exception, Cluster, ClusterConfig, \ + Failover, Leader, Member, Status, SyncState, TimelineHistory +from .etcd import AbstractEtcd, AbstractEtcdClientWithFailover, catch_etcd_errors, DnsCachingResolver, Retry logger = logging.getLogger(__name__) diff --git a/patroni/dcs/exhibitor.py b/patroni/dcs/exhibitor.py index 03d23575..b5b45319 100644 --- a/patroni/dcs/exhibitor.py +++ b/patroni/dcs/exhibitor.py @@ -5,11 +5,11 @@ import time from typing import Any, Callable, Dict, List, Union -from . import Cluster -from .zookeeper import ZooKeeper from ..postgresql.mpp import AbstractMPP from ..request import get as requests_get from ..utils import uri +from . import Cluster +from .zookeeper import ZooKeeper logger = logging.getLogger(__name__) diff --git a/patroni/dcs/kubernetes.py b/patroni/dcs/kubernetes.py index bf4e7fe1..d41d9006 100644 --- a/patroni/dcs/kubernetes.py +++ b/patroni/dcs/kubernetes.py @@ -9,22 +9,25 @@ import random import socket import tempfile import time -import urllib3 -import yaml from collections import defaultdict from copy import deepcopy from http.client import HTTPException -from urllib3.exceptions import HTTPError from threading import Condition, Lock, Thread -from typing import Any, Callable, Collection, Dict, List, Optional, Tuple, Type, Union, TYPE_CHECKING +from typing import Any, Callable, Collection, Dict, List, Optional, Tuple, Type, TYPE_CHECKING, Union + +import urllib3 +import yaml + +from urllib3.exceptions import HTTPError -from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, TimelineHistory from ..collections import EMPTY_DICT from ..exceptions import DCSError from ..postgresql.mpp import AbstractMPP -from ..utils import deep_compare, iter_response_objects, keepalive_socket_options, \ - Retry, RetryFailedError, tzutc, uri, USER_AGENT +from ..utils import deep_compare, iter_response_objects, \ + keepalive_socket_options, Retry, RetryFailedError, tzutc, uri, USER_AGENT +from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, TimelineHistory + if TYPE_CHECKING: # pragma: no cover from ..config import Config diff --git a/patroni/dcs/raft.py b/patroni/dcs/raft.py index 9965611e..9d0ab4a0 100644 --- a/patroni/dcs/raft.py +++ b/patroni/dcs/raft.py @@ -5,17 +5,19 @@ import threading import time from collections import defaultdict -from pysyncobj import SyncObj, SyncObjConf, replicated, FAIL_REASON +from typing import Any, Callable, Collection, Dict, List, Optional, Set, TYPE_CHECKING, Union + +from pysyncobj import FAIL_REASON, replicated, SyncObj, SyncObjConf from pysyncobj.dns_resolver import globalDnsResolver from pysyncobj.node import TCPNode -from pysyncobj.transport import TCPTransport, CONNECTION_STATE +from pysyncobj.transport import CONNECTION_STATE, TCPTransport from pysyncobj.utility import TcpUtility -from typing import Any, Callable, Collection, Dict, List, Optional, Set, Union, TYPE_CHECKING -from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, Status, SyncState, TimelineHistory from ..exceptions import DCSError from ..postgresql.mpp import AbstractMPP from ..utils import validate_directory +from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, TimelineHistory + if TYPE_CHECKING: # pragma: no cover from ..config import Config diff --git a/patroni/dcs/zookeeper.py b/patroni/dcs/zookeeper.py index 00c8e4bd..b6686982 100644 --- a/patroni/dcs/zookeeper.py +++ b/patroni/dcs/zookeeper.py @@ -4,18 +4,20 @@ import select import socket import time -from kazoo.client import KazooClient, KazooState, KazooRetry -from kazoo.exceptions import ConnectionClosedError, NoNodeError, NodeExistsError, SessionExpiredError +from typing import Any, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING, Union + +from kazoo.client import KazooClient, KazooRetry, KazooState +from kazoo.exceptions import ConnectionClosedError, NodeExistsError, NoNodeError, SessionExpiredError from kazoo.handlers.threading import AsyncResult, SequentialThreadingHandler from kazoo.protocol.states import KeeperState, WatchedEvent, ZnodeStat from kazoo.retry import RetryFailedError from kazoo.security import ACL, make_acl -from typing import Any, Callable, Dict, List, Optional, Union, Tuple, TYPE_CHECKING -from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, Status, SyncState, TimelineHistory from ..exceptions import DCSError from ..postgresql.mpp import AbstractMPP from ..utils import deep_compare +from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, TimelineHistory + if TYPE_CHECKING: # pragma: no cover from ..config import Config diff --git a/patroni/dynamic_loader.py b/patroni/dynamic_loader.py index 6c207349..48881fe7 100644 --- a/patroni/dynamic_loader.py +++ b/patroni/dynamic_loader.py @@ -5,9 +5,9 @@ import logging import os import pkgutil import sys -from types import ModuleType -from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, TYPE_CHECKING, Type, TypeVar, Union +from types import ModuleType +from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, Type, TYPE_CHECKING, TypeVar, Union if TYPE_CHECKING: # pragma: no cover from .config import Config diff --git a/patroni/global_config.py b/patroni/global_config.py index 924961fd..403b29af 100644 --- a/patroni/global_config.py +++ b/patroni/global_config.py @@ -8,7 +8,7 @@ import sys import types from copy import deepcopy -from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING +from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union from .collections import EMPTY_DICT from .utils import parse_bool, parse_int diff --git a/patroni/ha.py b/patroni/ha.py index 385d26da..72d52b32 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -8,14 +8,14 @@ import uuid from multiprocessing.pool import ThreadPool from threading import RLock -from typing import Any, Callable, Collection, Dict, List, NamedTuple, Optional, Union, Tuple, TYPE_CHECKING +from typing import Any, Callable, Collection, Dict, List, NamedTuple, Optional, Tuple, TYPE_CHECKING, Union from . import global_config, psycopg from .__main__ import Patroni from .async_executor import AsyncExecutor, CriticalTask from .collections import CaseInsensitiveSet -from .dcs import AbstractDCS, Cluster, Leader, Member, RemoteMember, Status, SyncState, slot_name_from_member_name -from .exceptions import DCSError, PostgresConnectionException, PatroniFatalException +from .dcs import AbstractDCS, Cluster, Leader, Member, RemoteMember, slot_name_from_member_name, Status, SyncState +from .exceptions import DCSError, PatroniFatalException, PostgresConnectionException from .postgresql.callback_executor import CallbackAction from .postgresql.misc import postgres_version_to_int from .postgresql.postmaster import PostmasterProcess diff --git a/patroni/log.py b/patroni/log.py index 2a896958..618ebf49 100644 --- a/patroni/log.py +++ b/patroni/log.py @@ -8,12 +8,11 @@ import os import sys from copy import deepcopy -from logging.handlers import RotatingFileHandler -from queue import Queue, Full -from threading import Lock, Thread - from io import TextIOWrapper -from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING +from logging.handlers import RotatingFileHandler +from queue import Full, Queue +from threading import Lock, Thread +from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union from .file_perm import pg_perm from .utils import deep_compare, parse_int diff --git a/patroni/postgresql/__init__.py b/patroni/postgresql/__init__.py index c5404ac6..2876951c 100644 --- a/patroni/postgresql/__init__.py +++ b/patroni/postgresql/__init__.py @@ -9,11 +9,19 @@ import time from contextlib import contextmanager from copy import deepcopy from datetime import datetime +from threading import current_thread, Lock +from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union + from dateutil import tz from psutil import TimeoutExpired -from threading import current_thread, Lock -from typing import Any, Callable, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING +from .. import global_config, psycopg +from ..async_executor import CriticalTask +from ..collections import CaseInsensitiveDict, CaseInsensitiveSet, EMPTY_DICT +from ..dcs import Cluster, Leader, Member +from ..exceptions import PostgresConnectionException +from ..tags import Tags +from ..utils import data_directory_is_empty, parse_int, polling_loop, Retry, RetryFailedError from .bootstrap import Bootstrap from .callback_executor import CallbackAction, CallbackExecutor from .cancellable import CancellableSubprocess @@ -24,13 +32,6 @@ from .mpp import AbstractMPP from .postmaster import PostmasterProcess from .slots import SlotsHandler from .sync import SyncHandler -from .. import global_config, psycopg -from ..async_executor import CriticalTask -from ..collections import CaseInsensitiveSet, CaseInsensitiveDict, EMPTY_DICT -from ..dcs import Cluster, Leader, Member -from ..exceptions import PostgresConnectionException -from ..utils import Retry, RetryFailedError, polling_loop, data_directory_is_empty, parse_int -from ..tags import Tags if TYPE_CHECKING: # pragma: no cover from psycopg import Connection as Connection3, Cursor diff --git a/patroni/postgresql/bootstrap.py b/patroni/postgresql/bootstrap.py index 2be038f9..e355dec2 100644 --- a/patroni/postgresql/bootstrap.py +++ b/patroni/postgresql/bootstrap.py @@ -4,7 +4,7 @@ import shlex import tempfile import time -from typing import Any, Callable, Dict, List, Optional, Union, Tuple, TYPE_CHECKING +from typing import Any, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING, Union from ..async_executor import CriticalTask from ..collections import EMPTY_DICT diff --git a/patroni/postgresql/cancellable.py b/patroni/postgresql/cancellable.py index e44729b0..5e75fd20 100644 --- a/patroni/postgresql/cancellable.py +++ b/patroni/postgresql/cancellable.py @@ -1,11 +1,13 @@ import logging -import psutil import subprocess +from threading import Lock +from typing import Any, Dict, List, Optional, Union + +import psutil + from patroni.exceptions import PostgresException from patroni.utils import polling_loop -from threading import Lock -from typing import Any, Dict, List, Optional, Union logger = logging.getLogger(__name__) diff --git a/patroni/postgresql/config.py b/patroni/postgresql/config.py index f7678045..f857bbdf 100644 --- a/patroni/postgresql/config.py +++ b/patroni/postgresql/config.py @@ -7,19 +7,19 @@ import stat import time from contextlib import contextmanager -from urllib.parse import urlparse, parse_qsl, unquote from types import TracebackType -from typing import Any, Callable, Collection, Dict, Iterator, List, Optional, Union, Tuple, Type, TYPE_CHECKING +from typing import Any, Callable, Collection, Dict, Iterator, List, Optional, Tuple, Type, TYPE_CHECKING, Union +from urllib.parse import parse_qsl, unquote, urlparse -from .validator import recovery_parameters, transform_postgresql_parameter_value, transform_recovery_parameter_value from .. import global_config from ..collections import CaseInsensitiveDict, CaseInsensitiveSet, EMPTY_DICT from ..dcs import Leader, Member, RemoteMember, slot_name_from_member_name from ..exceptions import PatroniFatalException, PostgresConnectionException from ..file_perm import pg_perm -from ..utils import (compare_values, maybe_convert_from_base_unit, parse_bool, parse_int, - split_host_port, uri, validate_directory, is_subpath) -from ..validator import IntValidator, EnumValidator +from ..utils import compare_values, is_subpath, maybe_convert_from_base_unit, \ + parse_bool, parse_int, split_host_port, uri, validate_directory +from ..validator import EnumValidator, IntValidator +from .validator import recovery_parameters, transform_postgresql_parameter_value, transform_recovery_parameter_value if TYPE_CHECKING: # pragma: no cover from . import Postgresql diff --git a/patroni/postgresql/connection.py b/patroni/postgresql/connection.py index 040dcf78..b48d1da1 100644 --- a/patroni/postgresql/connection.py +++ b/patroni/postgresql/connection.py @@ -2,7 +2,8 @@ import logging from contextlib import contextmanager from threading import Lock -from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING +from typing import Any, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union + if TYPE_CHECKING: # pragma: no cover from psycopg import Connection, Cursor from psycopg2 import connection, cursor diff --git a/patroni/postgresql/mpp/__init__.py b/patroni/postgresql/mpp/__init__.py index 5120db35..b29d9f01 100644 --- a/patroni/postgresql/mpp/__init__.py +++ b/patroni/postgresql/mpp/__init__.py @@ -5,15 +5,15 @@ supported MPP cluster. However, we may consider adapting other databases such as """ import abc -from typing import Any, Dict, Iterator, Optional, Union, Tuple, Type, TYPE_CHECKING +from typing import Any, Dict, Iterator, Optional, Tuple, Type, TYPE_CHECKING, Union from ...dcs import Cluster from ...dynamic_loader import iter_classes from ...exceptions import PatroniException if TYPE_CHECKING: # pragma: no cover - from .. import Postgresql from ...config import Config + from .. import Postgresql class AbstractMPP(abc.ABC): diff --git a/patroni/postgresql/mpp/citus.py b/patroni/postgresql/mpp/citus.py index c7286dc3..d4b26efe 100644 --- a/patroni/postgresql/mpp/citus.py +++ b/patroni/postgresql/mpp/citus.py @@ -3,13 +3,13 @@ import re import time from threading import Condition, Event, Thread +from typing import Any, Collection, Dict, Iterator, List, Optional, Set, Tuple, TYPE_CHECKING, Union from urllib.parse import urlparse -from typing import Any, Collection, Dict, Iterator, List, Optional, Union, Set, Tuple, TYPE_CHECKING -from . import AbstractMPP, AbstractMPPHandler from ...dcs import Cluster -from ...psycopg import connect, quote_ident, ProgrammingError +from ...psycopg import connect, ProgrammingError, quote_ident from ...utils import parse_int +from . import AbstractMPP, AbstractMPPHandler if TYPE_CHECKING: # pragma: no cover from .. import Postgresql diff --git a/patroni/postgresql/postmaster.py b/patroni/postgresql/postmaster.py index 359067c2..ea218778 100644 --- a/patroni/postgresql/postmaster.py +++ b/patroni/postgresql/postmaster.py @@ -1,16 +1,17 @@ import logging import multiprocessing import os -import psutil import re import signal import subprocess import sys from multiprocessing.connection import Connection -from typing import Dict, Optional, List +from typing import Dict, List, Optional -from patroni import PATRONI_ENV_PREFIX, KUBERNETES_ENV_PREFIX +import psutil + +from patroni import KUBERNETES_ENV_PREFIX, PATRONI_ENV_PREFIX # avoid spawning the resource tracker process if sys.version_info >= (3, 8): # pragma: no cover diff --git a/patroni/postgresql/rewind.py b/patroni/postgresql/rewind.py index 91cddd59..762e65e2 100644 --- a/patroni/postgresql/rewind.py +++ b/patroni/postgresql/rewind.py @@ -7,14 +7,14 @@ import subprocess from enum import IntEnum from threading import Lock, Thread -from typing import Any, Callable, Dict, List, Optional, Union, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple, Union -from . import Postgresql -from .connection import get_connection_cursor -from .misc import format_lsn, fsync_dir, parse_history, parse_lsn from ..async_executor import CriticalTask from ..collections import EMPTY_DICT from ..dcs import Leader, RemoteMember +from . import Postgresql +from .connection import get_connection_cursor +from .misc import format_lsn, fsync_dir, parse_history, parse_lsn logger = logging.getLogger(__name__) diff --git a/patroni/postgresql/slots.py b/patroni/postgresql/slots.py index e8ae8a18..9e5216fb 100644 --- a/patroni/postgresql/slots.py +++ b/patroni/postgresql/slots.py @@ -6,22 +6,24 @@ Provides classes for the creation, monitoring, management and synchronisation of import logging import os import shutil + from collections import defaultdict from contextlib import contextmanager from threading import Condition, Thread -from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING, Collection +from typing import Any, Collection, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union -from .connection import get_connection_cursor -from .misc import format_lsn, fsync_dir from .. import global_config from ..dcs import Cluster, Leader from ..file_perm import pg_perm from ..psycopg import OperationalError from ..tags import Tags +from .connection import get_connection_cursor +from .misc import format_lsn, fsync_dir if TYPE_CHECKING: # pragma: no cover from psycopg import Cursor from psycopg2 import cursor + from . import Postgresql logger = logging.getLogger(__name__) diff --git a/patroni/postgresql/sync.py b/patroni/postgresql/sync.py index 6901fa11..b4c6bf05 100644 --- a/patroni/postgresql/sync.py +++ b/patroni/postgresql/sync.py @@ -9,6 +9,7 @@ from .. import global_config from ..collections import CaseInsensitiveDict, CaseInsensitiveSet from ..dcs import Cluster from ..psycopg import quote_ident as _quote_ident + if TYPE_CHECKING: # pragma: no cover from . import Postgresql diff --git a/patroni/postgresql/validator.py b/patroni/postgresql/validator.py index f118c973..66c5ada3 100644 --- a/patroni/postgresql/validator.py +++ b/patroni/postgresql/validator.py @@ -1,14 +1,15 @@ import abc -from copy import deepcopy import logging -import yaml +from copy import deepcopy from typing import Any, Dict, Iterator, List, MutableMapping, Optional, Tuple, Type, Union -from .available_parameters import get_validator_files, PathLikeObj +import yaml + from ..collections import CaseInsensitiveDict, CaseInsensitiveSet from ..exceptions import PatroniException from ..utils import parse_bool, parse_int, parse_real +from .available_parameters import get_validator_files, PathLikeObj logger = logging.getLogger(__name__) diff --git a/patroni/psycopg.py b/patroni/psycopg.py index 5d7c6a37..a91519f3 100644 --- a/patroni/psycopg.py +++ b/patroni/psycopg.py @@ -5,6 +5,7 @@ This module is able to handle both :mod:`pyscopg2` and :mod:`psycopg`, and it ex ``2.5.4``. """ from typing import Any, Optional, TYPE_CHECKING, Union + if TYPE_CHECKING: # pragma: no cover from psycopg import Connection from psycopg2 import connection, cursor @@ -14,10 +15,11 @@ __all__ = ['connect', 'quote_ident', 'quote_literal', 'DatabaseError', 'Error', _legacy = False try: from psycopg2 import __version__ + from . import MIN_PSYCOPG2, parse_version if parse_version(__version__) < MIN_PSYCOPG2: raise ImportError - from psycopg2 import connect as _connect, Error, DatabaseError, OperationalError, ProgrammingError + from psycopg2 import connect as _connect, DatabaseError, Error, OperationalError, ProgrammingError from psycopg2.extensions import adapt try: @@ -44,8 +46,9 @@ try: except ImportError: import types + from psycopg import DatabaseError, Error, OperationalError, ProgrammingError, sql + # isort: off from psycopg import connect as __connect # pyright: ignore [reportUnknownVariableType] - from psycopg import sql, Error, DatabaseError, OperationalError, ProgrammingError def __get_parameter_status(self: 'Connection[Any]', param_name: str) -> Optional[str]: """Helper function to be injected into :class:`Connection` object. diff --git a/patroni/raft_controller.py b/patroni/raft_controller.py index a9d7424c..10845d24 100644 --- a/patroni/raft_controller.py +++ b/patroni/raft_controller.py @@ -1,7 +1,7 @@ import logging from .config import Config -from .daemon import AbstractPatroniDaemon, abstract_main, get_base_arg_parser +from .daemon import abstract_main, AbstractPatroniDaemon, get_base_arg_parser from .dcs.raft import KVStoreTTL logger = logging.getLogger(__name__) diff --git a/patroni/request.py b/patroni/request.py index 16659c96..2bba134b 100644 --- a/patroni/request.py +++ b/patroni/request.py @@ -1,11 +1,11 @@ """Facilities for handling communication with Patroni's REST API.""" import json -import urllib3 from typing import Any, Dict, Optional, Union - from urllib.parse import urlparse, urlunparse +import urllib3 + from .config import Config from .dcs import Member from .utils import USER_AGENT diff --git a/patroni/scripts/aws.py b/patroni/scripts/aws.py index 9fcc8160..d987f785 100755 --- a/patroni/scripts/aws.py +++ b/patroni/scripts/aws.py @@ -3,13 +3,14 @@ import json import logging import sys + +from typing import Any, Optional + import boto3 from botocore.exceptions import ClientError from botocore.utils import IMDSFetcher -from typing import Any, Optional - from ..utils import Retry, RetryFailedError logger = logging.getLogger(__name__) diff --git a/patroni/scripts/barman/cli.py b/patroni/scripts/barman/cli.py index fcec14e6..148947a6 100644 --- a/patroni/scripts/barman/cli.py +++ b/patroni/scripts/barman/cli.py @@ -9,11 +9,12 @@ builds the CLI that makes an interface with the actual commands. See :class:ExitCode` for possible exit codes of this main script. """ -from argparse import ArgumentParser -from enum import IntEnum import logging import sys +from argparse import ArgumentParser +from enum import IntEnum + from .config_switch import run_barman_config_switch from .recover import run_barman_recover from .utils import ApiNotOk, PgBackupApi, set_up_logging diff --git a/patroni/scripts/barman/config_switch.py b/patroni/scripts/barman/config_switch.py index f65dc3f1..c4540108 100644 --- a/patroni/scripts/barman/config_switch.py +++ b/patroni/scripts/barman/config_switch.py @@ -14,15 +14,15 @@ the same host as Barman. Refer to :class:`ExitCode` for possible exit codes of this sub-command. """ -from argparse import Namespace -from enum import IntEnum import logging import time + +from argparse import Namespace +from enum import IntEnum from typing import Optional, TYPE_CHECKING from .utils import OperationStatus, RetriesExceeded - if TYPE_CHECKING: # pragma: no cover from .utils import PgBackupApi diff --git a/patroni/scripts/barman/recover.py b/patroni/scripts/barman/recover.py index 5a07a7a5..a4d12b8a 100644 --- a/patroni/scripts/barman/recover.py +++ b/patroni/scripts/barman/recover.py @@ -14,15 +14,15 @@ have ``pg-backup-api`` configured and running in the same host as Barman. Refer to :class:`ExitCode` for possible exit codes of this sub-command. """ -from argparse import Namespace -from enum import IntEnum import logging import time + +from argparse import Namespace +from enum import IntEnum from typing import TYPE_CHECKING from .utils import OperationStatus, RetriesExceeded - if TYPE_CHECKING: # pragma: no cover from .utils import PgBackupApi diff --git a/patroni/scripts/barman/utils.py b/patroni/scripts/barman/utils.py index 96b68bbf..6145d9fa 100644 --- a/patroni/scripts/barman/utils.py +++ b/patroni/scripts/barman/utils.py @@ -2,11 +2,12 @@ """Utilitary stuff to be used by Barman related scripts.""" -from enum import IntEnum import json import logging -from typing import Any, Callable, Dict, Optional, Tuple, Type, Union import time + +from enum import IntEnum +from typing import Any, Callable, Dict, Optional, Tuple, Type, Union from urllib.parse import urljoin from urllib3 import PoolManager diff --git a/patroni/tags.py b/patroni/tags.py index e1e68f26..844a1761 100644 --- a/patroni/tags.py +++ b/patroni/tags.py @@ -3,7 +3,7 @@ import abc from typing import Any, Dict, Optional -from patroni.utils import parse_int, parse_bool +from patroni.utils import parse_bool, parse_int class Tags(abc.ABC): diff --git a/patroni/utils.py b/patroni/utils.py index 53ae1080..021fa416 100644 --- a/patroni/utils.py +++ b/patroni/utils.py @@ -21,13 +21,13 @@ import subprocess import sys import tempfile import time -from shlex import split - -from typing import Any, Callable, Dict, Iterator, List, Optional, Union, Tuple, Type, TYPE_CHECKING from collections import OrderedDict -from dateutil import tz from json import JSONDecoder +from shlex import split +from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Type, TYPE_CHECKING, Union + +from dateutil import tz from urllib3.response import HTTPResponse from .exceptions import PatroniException diff --git a/patroni/validator.py b/patroni/validator.py index d1473f11..c1438d38 100644 --- a/patroni/validator.py +++ b/patroni/validator.py @@ -9,13 +9,13 @@ import os import shutil import socket -from typing import Any, Dict, Union, Iterator, List, Optional as OptionalType, Tuple, TYPE_CHECKING +from typing import Any, Dict, Iterator, List, Optional as OptionalType, Tuple, TYPE_CHECKING, Union from .collections import CaseInsensitiveSet, EMPTY_DICT from .dcs import dcs_modules from .exceptions import ConfigParseError -from .utils import parse_int, split_host_port, data_directory_is_empty, get_major_version from .log import type_logformat +from .utils import data_directory_is_empty, get_major_version, parse_int, split_host_port def validate_log_field(field: Union[str, Dict[str, Any], Any]) -> bool: diff --git a/patroni/watchdog/__init__.py b/patroni/watchdog/__init__.py index 4acc855e..77a53263 100644 --- a/patroni/watchdog/__init__.py +++ b/patroni/watchdog/__init__.py @@ -1,2 +1,3 @@ -from patroni.watchdog.base import WatchdogError, Watchdog +from patroni.watchdog.base import Watchdog, WatchdogError + __all__ = ['WatchdogError', 'Watchdog'] diff --git a/patroni/watchdog/base.py b/patroni/watchdog/base.py index 7d1cdca6..ad80dfe5 100644 --- a/patroni/watchdog/base.py +++ b/patroni/watchdog/base.py @@ -2,8 +2,8 @@ import abc import logging import platform import sys -from threading import RLock +from threading import RLock from typing import Any, Callable, Dict, Optional, Union from ..config import Config diff --git a/setup.py b/setup.py index b8b8a062..0d69a840 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ Setup file for patroni """ +import glob import inspect import logging import os @@ -47,6 +48,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', ] @@ -68,33 +70,67 @@ class _Command(Command): pass -class Flake8(_Command): +class _Lint(_Command): - def package_files(self): - seen_package_directories = () - directories = self.distribution.package_dir or {} - empty_directory_exists = "" in directories - packages = self.distribution.packages or [] - for package in packages: - if package in directories: - package_directory = directories[package] - elif empty_directory_exists: - package_directory = os.path.join(directories[""], package) + def package_modules(self): + package_dirs = self.distribution.package_dir or {} + for package in self.distribution.packages or []: + if package in package_dirs: + yield package_dirs[package] + elif '' in package_dirs: + yield os.path.join(package_dirs[''], package) else: - package_directory = package + yield package - if not package_directory.startswith(seen_package_directories): - seen_package_directories += (package_directory + ".",) - yield package_directory + def package_directories(self): + for module in self.package_modules(): + yield module.replace('.', os.path.sep) - def targets(self): - return [package for package in self.package_files()] + ['tests', 'features', 'setup.py'] + def aux_directories(self): + for dir_name in ('tests', 'features'): + for root, dirs, files in os.walk(dir_name): + for name in dirs: + yield os.path.join(root, name) + + def dirs_to_check(self): + yield from self.package_directories() + yield from self.aux_directories() + + def files_to_check(self): + for path in self.dirs_to_check(): + for python_file in glob.iglob(os.path.join(path, '*.py')): + yield python_file + + for filename in self.distribution.py_modules or []: + yield f'{filename}.py' + + yield 'setup.py' + + +class Flake8(_Lint): def run(self): from flake8.main.cli import main logging.getLogger().setLevel(logging.ERROR) - raise SystemExit(main(self.targets())) + raise SystemExit(main(list(self.files_to_check()))) + + +class ISort(_Lint): + + def run(self): + from isort import api + + wrong_sorted_files = False + for python_file in self.files_to_check(): + try: + if not api.check_file(python_file, settings_path=__location__, show_diff=True): + wrong_sorted_files = True + except OSError as error: + logging.warning('Unable to parse file %s due to %r', python_file, error) + wrong_sorted_files = True + if wrong_sorted_files: + sys.exit(1) class PyTest(_Command): @@ -177,7 +213,7 @@ def main(): ]}, install_requires=install_requires, extras_require=EXTRAS_REQUIRE, - cmdclass={'test': PyTest, 'flake8': Flake8}, + cmdclass={'test': PyTest, 'flake8': Flake8, 'isort': ISort}, entry_points={'console_scripts': CONSOLE_SCRIPTS}, ) diff --git a/tests/__init__.py b/tests/__init__.py index a9307e6f..66659dee 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,7 +2,8 @@ import datetime import os import shutil import unittest -from unittest.mock import Mock, PropertyMock, patch + +from unittest.mock import Mock, patch, PropertyMock import urllib3 diff --git a/tests/test_api.py b/tests/test_api.py index 4ae8821f..55e0016b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,12 +1,12 @@ import datetime import json -import unittest import socket +import unittest from http.server import HTTPServer from io import BytesIO as IO from socketserver import ThreadingMixIn -from unittest.mock import Mock, PropertyMock, patch +from unittest.mock import Mock, patch, PropertyMock from patroni import global_config from patroni.api import RestApiHandler, RestApiServer @@ -20,7 +20,6 @@ from patroni.utils import RetryFailedError, tzutc from . import MockConnect, psycopg_connect from .test_ha import get_cluster_initialized_without_leader - future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5) postmaster_start_time = datetime.datetime.now(tzutc) diff --git a/tests/test_async_executor.py b/tests/test_async_executor.py index 08a91917..853d685a 100644 --- a/tests/test_async_executor.py +++ b/tests/test_async_executor.py @@ -1,8 +1,9 @@ import unittest + +from threading import Thread from unittest.mock import Mock, patch from patroni.async_executor import AsyncExecutor, CriticalTask -from threading import Thread class TestAsyncExecutor(unittest.TestCase): diff --git a/tests/test_aws.py b/tests/test_aws.py index 69ca3285..f63f188c 100644 --- a/tests/test_aws.py +++ b/tests/test_aws.py @@ -1,10 +1,12 @@ -import botocore -import botocore.awsrequest import sys import unittest -from unittest.mock import Mock, PropertyMock, patch from collections import namedtuple +from unittest.mock import Mock, patch, PropertyMock + +import botocore +import botocore.awsrequest + from patroni.scripts.aws import AWSConnection, main as _main diff --git a/tests/test_barman.py b/tests/test_barman.py index eda9cdda..260f4c4d 100644 --- a/tests/test_barman.py +++ b/tests/test_barman.py @@ -1,16 +1,17 @@ import logging import unittest + from unittest import mock from unittest.mock import MagicMock, Mock, patch + from urllib3.exceptions import MaxRetryError from patroni.scripts.barman.cli import main -from patroni.scripts.barman.config_switch import (ExitCode as BarmanConfigSwitchExitCode, _should_skip_switch, - _switch_config, run_barman_config_switch) -from patroni.scripts.barman.recover import ExitCode as BarmanRecoverExitCode, _restore_backup, run_barman_recover +from patroni.scripts.barman.config_switch import _should_skip_switch, _switch_config, \ + ExitCode as BarmanConfigSwitchExitCode, run_barman_config_switch +from patroni.scripts.barman.recover import _restore_backup, ExitCode as BarmanRecoverExitCode, run_barman_recover from patroni.scripts.barman.utils import ApiNotOk, OperationStatus, PgBackupApi, RetriesExceeded, set_up_logging - API_URL = "http://localhost:7480" BARMAN_SERVER = "my_server" BARMAN_MODEL = "my_model" diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 37725ae5..6371f6b1 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -1,6 +1,7 @@ import os import sys -from unittest.mock import Mock, PropertyMock, patch + +from unittest.mock import Mock, patch, PropertyMock from patroni.async_executor import CriticalTask from patroni.collections import CaseInsensitiveDict @@ -9,7 +10,7 @@ from patroni.postgresql.bootstrap import Bootstrap from patroni.postgresql.cancellable import CancellableSubprocess from patroni.postgresql.config import ConfigHandler, get_param_diff -from . import psycopg_connect, BaseTestPostgresql, mock_available_gucs +from . import BaseTestPostgresql, mock_available_gucs, psycopg_connect @patch('subprocess.call', Mock(return_value=0)) diff --git a/tests/test_callback_executor.py b/tests/test_callback_executor.py index 6ec53a3d..e76c9657 100644 --- a/tests/test_callback_executor.py +++ b/tests/test_callback_executor.py @@ -1,7 +1,9 @@ -import psutil import unittest + from unittest.mock import Mock, patch +import psutil + from patroni.postgresql.callback_executor import CallbackExecutor diff --git a/tests/test_cancellable.py b/tests/test_cancellable.py index 23510cf9..41568ba7 100644 --- a/tests/test_cancellable.py +++ b/tests/test_cancellable.py @@ -1,7 +1,9 @@ -import psutil import unittest + from unittest.mock import Mock, patch +import psutil + from patroni.exceptions import PostgresException from patroni.postgresql.cancellable import CancellableSubprocess diff --git a/tests/test_citus.py b/tests/test_citus.py index ce4860ac..63bbf330 100644 --- a/tests/test_citus.py +++ b/tests/test_citus.py @@ -2,8 +2,8 @@ import time import unittest from copy import deepcopy -from unittest.mock import Mock, patch, PropertyMock from typing import List +from unittest.mock import Mock, patch, PropertyMock from patroni.postgresql.mpp.citus import CitusHandler, PgDistGroup, PgDistNode from patroni.psycopg import ProgrammingError diff --git a/tests/test_config.py b/tests/test_config.py index c45bf420..ef4ef1ae 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,7 +1,7 @@ +import io import os import sys import unittest -import io from copy import deepcopy from unittest.mock import MagicMock, Mock, patch diff --git a/tests/test_config_generator.py b/tests/test_config_generator.py index d40cee1f..ffba14ee 100644 --- a/tests/test_config_generator.py +++ b/tests/test_config_generator.py @@ -1,19 +1,19 @@ import os -import psutil import unittest -import yaml -from . import MockConnect, MockCursor from copy import deepcopy -from unittest.mock import MagicMock, Mock, PropertyMock, mock_open as _mock_open, patch +from unittest.mock import MagicMock, Mock, mock_open as _mock_open, patch, PropertyMock + +import psutil +import yaml from patroni.__main__ import main as _main from patroni.config import Config from patroni.config_generator import AbstractConfigGenerator, get_address, NO_VALUE_MSG from patroni.log import PatroniLogger -from patroni.utils import patch_config, parse_bool +from patroni.utils import parse_bool, patch_config -from . import psycopg_connect +from . import MockConnect, MockCursor, psycopg_connect HOSTNAME = 'test_hostname' IP = '1.9.8.4' diff --git a/tests/test_consul.py b/tests/test_consul.py index 49ec3118..d7288c52 100644 --- a/tests/test_consul.py +++ b/tests/test_consul.py @@ -1,12 +1,16 @@ -import consul import unittest -from unittest.mock import Mock, PropertyMock, patch + +from unittest.mock import Mock, patch, PropertyMock + +import consul from consul import ConsulException, NotFound + from patroni.dcs import get_dcs -from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulInternalError, \ - ConsulError, ConsulClient, HTTPClient, InvalidSessionTTL, InvalidSession, RetryFailedError +from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulClient, ConsulError, \ + ConsulInternalError, HTTPClient, InvalidSession, InvalidSessionTTL, RetryFailedError from patroni.postgresql.mpp import get_mpp + from . import SleepException diff --git a/tests/test_ctl.py b/tests/test_ctl.py index d5d83740..f126e939 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -1,28 +1,31 @@ -import click -import etcd import os import unittest -from click.testing import CliRunner from datetime import datetime, timedelta +from unittest import mock +from unittest.mock import Mock, patch, PropertyMock + +import click +import etcd + +from click.testing import CliRunner +from prettytable import ALL, PrettyTable +from urllib3 import PoolManager + from patroni import global_config -from patroni.ctl import ctl, load_config, output_members, get_dcs, parse_dcs, \ - get_all_members, get_any_member, get_cursor, query_member, PatroniCtlException, apply_config_changes, \ - format_config_for_editing, show_diff, invoke_editor, format_pg_version, CONFIG_FILE_PATH, PatronictlPrettyTable +from patroni.ctl import apply_config_changes, CONFIG_FILE_PATH, ctl, format_config_for_editing, \ + format_pg_version, get_all_members, get_any_member, get_cursor, get_dcs, invoke_editor, load_config, \ + output_members, parse_dcs, PatroniCtlException, PatronictlPrettyTable, query_member, show_diff from patroni.dcs import Cluster, Failover from patroni.postgresql.config import get_param_diff from patroni.postgresql.mpp import get_mpp from patroni.psycopg import OperationalError from patroni.utils import tzutc -from prettytable import PrettyTable, ALL -from unittest import mock -from unittest.mock import patch, Mock, PropertyMock -from urllib3 import PoolManager from . import MockConnect, MockCursor, MockResponse, psycopg_connect from .test_etcd import etcd_read, socket_getaddrinfo -from .test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \ - get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader, get_cluster, Member +from .test_ha import get_cluster, get_cluster_initialized_with_leader, get_cluster_initialized_with_only_leader, \ + get_cluster_initialized_without_leader, get_cluster_not_initialized_without_leader, Member def get_default_config(*args): diff --git a/tests/test_etcd.py b/tests/test_etcd.py index 1d40829d..1fe48a8e 100644 --- a/tests/test_etcd.py +++ b/tests/test_etcd.py @@ -1,18 +1,21 @@ -import etcd -import urllib3.util.connection import socket import unittest +from unittest.mock import Mock, patch, PropertyMock + +import etcd +import urllib3.util.connection + from dns.exception import DNSException +from urllib3.exceptions import ReadTimeoutError + from patroni.dcs import get_dcs -from patroni.dcs.etcd import AbstractDCS, EtcdClient, Cluster, Etcd, EtcdError, DnsCachingResolver +from patroni.dcs.etcd import AbstractDCS, Cluster, DnsCachingResolver, Etcd, EtcdClient, EtcdError from patroni.exceptions import DCSError from patroni.postgresql.mpp import get_mpp from patroni.utils import Retry -from unittest.mock import Mock, PropertyMock, patch -from urllib3.exceptions import ReadTimeoutError -from . import SleepException, MockResponse, requests_get +from . import MockResponse, requests_get, SleepException def etcd_watch(self, key, index=None, timeout=None, recursive=None): diff --git a/tests/test_etcd3.py b/tests/test_etcd3.py index 2ea3699e..54941130 100644 --- a/tests/test_etcd3.py +++ b/tests/test_etcd3.py @@ -1,18 +1,20 @@ -import etcd import json import unittest + +from threading import Thread +from unittest.mock import Mock, patch, PropertyMock + +import etcd import urllib3 -from unittest.mock import Mock, PropertyMock, patch from patroni.dcs import get_dcs from patroni.dcs.etcd import DnsCachingResolver -from patroni.dcs.etcd3 import PatroniEtcd3Client, Cluster, Etcd3, Etcd3Client, \ - Etcd3Error, Etcd3ClientError, RetryFailedError, InvalidAuthToken, Unavailable, \ - Unknown, UnsupportedEtcdVersion, UserEmpty, AuthFailed, AuthOldRevision, base64_encode +from patroni.dcs.etcd3 import AuthFailed, AuthOldRevision, base64_encode, Cluster, Etcd3, \ + Etcd3Client, Etcd3ClientError, Etcd3Error, InvalidAuthToken, PatroniEtcd3Client, \ + RetryFailedError, Unavailable, Unknown, UnsupportedEtcdVersion, UserEmpty from patroni.postgresql.mpp import get_mpp -from threading import Thread -from . import SleepException, MockResponse +from . import MockResponse, SleepException def mock_urlopen(self, method, url, **kwargs): diff --git a/tests/test_exhibitor.py b/tests/test_exhibitor.py index 0620292c..fae1586d 100644 --- a/tests/test_exhibitor.py +++ b/tests/test_exhibitor.py @@ -1,12 +1,14 @@ import unittest -import urllib3 + from unittest.mock import Mock, patch +import urllib3 + from patroni.dcs import get_dcs -from patroni.dcs.exhibitor import ExhibitorEnsembleProvider, Exhibitor +from patroni.dcs.exhibitor import Exhibitor, ExhibitorEnsembleProvider from patroni.dcs.zookeeper import ZooKeeperError -from . import SleepException, requests_get +from . import requests_get, SleepException from .test_zookeeper import MockKazooClient diff --git a/tests/test_file_perm.py b/tests/test_file_perm.py index 4a845ebe..afd1dfa4 100644 --- a/tests/test_file_perm.py +++ b/tests/test_file_perm.py @@ -1,5 +1,6 @@ -import unittest import stat +import unittest + from unittest.mock import Mock, patch from patroni.file_perm import pg_perm diff --git a/tests/test_ha.py b/tests/test_ha.py index 286c99ed..d6f04358 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -1,16 +1,18 @@ import datetime -import etcd import os import sys -from unittest.mock import Mock, MagicMock, PropertyMock, patch, mock_open + +from unittest.mock import MagicMock, Mock, mock_open, patch, PropertyMock + +import etcd from patroni import global_config from patroni.collections import CaseInsensitiveSet from patroni.config import Config -from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, Status, SyncState, TimelineHistory +from patroni.dcs import Cluster, ClusterConfig, Failover, get_dcs, Leader, Member, Status, SyncState, TimelineHistory from patroni.dcs.etcd import AbstractEtcdClientWithFailover -from patroni.exceptions import DCSError, PostgresConnectionException, PatroniFatalException -from patroni.ha import Ha, _MemberStatus +from patroni.exceptions import DCSError, PatroniFatalException, PostgresConnectionException +from patroni.ha import _MemberStatus, Ha from patroni.postgresql import Postgresql from patroni.postgresql.bootstrap import Bootstrap from patroni.postgresql.cancellable import CancellableSubprocess @@ -22,8 +24,8 @@ from patroni.postgresql.sync import _SyncState from patroni.utils import tzutc from patroni.watchdog import Watchdog -from . import PostgresInit, MockPostmaster, psycopg_connect, requests_get -from .test_etcd import socket_getaddrinfo, etcd_read, etcd_write +from . import MockPostmaster, PostgresInit, psycopg_connect, requests_get +from .test_etcd import etcd_read, etcd_write, socket_getaddrinfo SYSID = '12345678901' diff --git a/tests/test_kubernetes.py b/tests/test_kubernetes.py index 389c6472..432158de 100644 --- a/tests/test_kubernetes.py +++ b/tests/test_kubernetes.py @@ -4,16 +4,19 @@ import json import socket import time import unittest -import urllib3 + +from threading import Thread from unittest import mock -from unittest.mock import Mock, PropertyMock, mock_open, patch +from unittest.mock import Mock, mock_open, patch, PropertyMock + +import urllib3 from patroni.dcs import get_dcs from patroni.dcs.kubernetes import Cluster, k8s_client, k8s_config, K8sConfig, K8sConnectionFailed, \ - K8sException, K8sObject, Kubernetes, KubernetesError, KubernetesRetriableException, \ - Retry, RetryFailedError, SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME + K8sException, K8sObject, Kubernetes, KubernetesError, KubernetesRetriableException, Retry, \ + RetryFailedError, SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME from patroni.postgresql.mpp import get_mpp -from threading import Thread + from . import MockResponse, SleepException diff --git a/tests/test_log.py b/tests/test_log.py index 165fd522..d0ab7ca8 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -2,11 +2,13 @@ import logging import os import sys import unittest -import yaml + from io import StringIO -from queue import Queue, Full +from queue import Full, Queue from unittest.mock import Mock, patch +import yaml + from patroni.config import Config from patroni.log import PatroniLogger diff --git a/tests/test_mpp.py b/tests/test_mpp.py index 9eb87633..12e48778 100644 --- a/tests/test_mpp.py +++ b/tests/test_mpp.py @@ -1,4 +1,5 @@ from typing import Any + from patroni.exceptions import PatroniException from patroni.postgresql.mpp import AbstractMPP, get_mpp, Null diff --git a/tests/test_patroni.py b/tests/test_patroni.py index fb5c0fc7..e65b1870 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -1,13 +1,18 @@ -import etcd import logging import os import signal import time import unittest -from unittest.mock import Mock, PropertyMock, patch + +from http.server import HTTPServer +from threading import Thread +from unittest.mock import Mock, patch, PropertyMock + +import etcd import patroni.config as config -from http.server import HTTPServer + +from patroni.__main__ import check_psycopg, main as _main, Patroni from patroni.api import RestApiServer from patroni.async_executor import AsyncExecutor from patroni.dcs import Cluster, Member @@ -15,8 +20,6 @@ from patroni.dcs.etcd import AbstractEtcdClientWithFailover from patroni.exceptions import DCSError from patroni.postgresql import Postgresql from patroni.postgresql.config import ConfigHandler -from patroni.__main__ import check_psycopg, Patroni, main as _main -from threading import Thread from . import psycopg_connect, SleepException from .test_etcd import etcd_read, etcd_write diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index ff76ade1..c249bdbc 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -1,13 +1,15 @@ import datetime import os -import psutil import re import subprocess import time from copy import deepcopy from pathlib import Path -from unittest.mock import Mock, MagicMock, PropertyMock, patch, mock_open +from threading import current_thread, Thread +from unittest.mock import MagicMock, Mock, mock_open, patch, PropertyMock + +import psutil import patroni.psycopg as psycopg @@ -15,22 +17,19 @@ from patroni import global_config from patroni.async_executor import CriticalTask from patroni.collections import CaseInsensitiveDict, CaseInsensitiveSet from patroni.dcs import RemoteMember -from patroni.exceptions import PostgresConnectionException, PatroniException -from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE +from patroni.exceptions import PatroniException, PostgresConnectionException +from patroni.postgresql import Postgresql, STATE_NO_RESPONSE, STATE_REJECT from patroni.postgresql.bootstrap import Bootstrap from patroni.postgresql.callback_executor import CallbackAction -from patroni.postgresql.config import get_param_diff, _false_validator +from patroni.postgresql.config import _false_validator, get_param_diff from patroni.postgresql.postmaster import PostmasterProcess -from patroni.postgresql.validator import (ValidatorFactoryNoType, ValidatorFactoryInvalidType, - ValidatorFactoryInvalidSpec, ValidatorFactory, InvalidGucValidatorsFile, - _get_postgres_guc_validators, _read_postgres_gucs_validators_file, - _load_postgres_gucs_validators, Bool, Integer, Real, Enum, EnumBool, String) +from patroni.postgresql.validator import _get_postgres_guc_validators, _load_postgres_gucs_validators, \ + _read_postgres_gucs_validators_file, Bool, Enum, EnumBool, Integer, InvalidGucValidatorsFile, Real, String, \ + ValidatorFactory, ValidatorFactoryInvalidSpec, ValidatorFactoryInvalidType, ValidatorFactoryNoType from patroni.utils import RetryFailedError -from threading import Thread, current_thread - -from . import (BaseTestPostgresql, MockCursor, MockPostmaster, psycopg_connect, mock_available_gucs, - GET_PG_SETTINGS_RESULT) +from . import BaseTestPostgresql, GET_PG_SETTINGS_RESULT, \ + mock_available_gucs, MockCursor, MockPostmaster, psycopg_connect mtime_ret = {} diff --git a/tests/test_postmaster.py b/tests/test_postmaster.py index 0924e6f7..1075902d 100644 --- a/tests/test_postmaster.py +++ b/tests/test_postmaster.py @@ -1,7 +1,9 @@ import multiprocessing -import psutil import unittest -from unittest.mock import Mock, patch, mock_open + +from unittest.mock import Mock, mock_open, patch + +import psutil from patroni.postgresql.postmaster import PostmasterProcess diff --git a/tests/test_quorum.py b/tests/test_quorum.py index 8dddb4f1..2ae380fe 100644 --- a/tests/test_quorum.py +++ b/tests/test_quorum.py @@ -2,7 +2,7 @@ import unittest from typing import List, Set, Tuple -from patroni.quorum import QuorumStateResolver, QuorumError +from patroni.quorum import QuorumError, QuorumStateResolver class QuorumTest(unittest.TestCase): diff --git a/tests/test_raft.py b/tests/test_raft.py index 53bbe72c..12101900 100644 --- a/tests/test_raft.py +++ b/tests/test_raft.py @@ -1,14 +1,16 @@ import os -import unittest import tempfile import time -from unittest.mock import Mock, PropertyMock, patch +import unittest + +from unittest.mock import Mock, patch, PropertyMock + +from pysyncobj import FAIL_REASON, SyncObjConf from patroni.dcs import get_dcs -from patroni.dcs.raft import Cluster, DynMemberSyncObj, KVStoreTTL, \ - Raft, RaftError, SyncObjUtility, TCPTransport, _TCPTransport +from patroni.dcs.raft import _TCPTransport, Cluster, DynMemberSyncObj, \ + KVStoreTTL, Raft, RaftError, SyncObjUtility, TCPTransport from patroni.postgresql.mpp import get_mpp -from pysyncobj import SyncObjConf, FAIL_REASON def remove_files(prefix): diff --git a/tests/test_raft_controller.py b/tests/test_raft_controller.py index fdd09f71..7f7b0bc6 100644 --- a/tests/test_raft_controller.py +++ b/tests/test_raft_controller.py @@ -1,11 +1,13 @@ import logging import os import unittest + from unittest.mock import Mock, patch from pysyncobj import SyncObj + from patroni.config import Config -from patroni.raft_controller import RaftController, main as _main +from patroni.raft_controller import main as _main, RaftController from . import SleepException from .test_raft import remove_files diff --git a/tests/test_rewind.py b/tests/test_rewind.py index a51a8bec..60702528 100644 --- a/tests/test_rewind.py +++ b/tests/test_rewind.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock, PropertyMock, patch, mock_open +from unittest.mock import Mock, mock_open, patch, PropertyMock from patroni.postgresql import Postgresql from patroni.postgresql.cancellable import CancellableSubprocess diff --git a/tests/test_slots.py b/tests/test_slots.py index d7ea61bf..3739dc5c 100644 --- a/tests/test_slots.py +++ b/tests/test_slots.py @@ -1,8 +1,9 @@ import os import unittest + from threading import Thread from unittest import mock -from unittest.mock import Mock, PropertyMock, patch +from unittest.mock import Mock, patch, PropertyMock from patroni import global_config, psycopg from patroni.dcs import Cluster, ClusterConfig, Member, Status, SyncState @@ -11,7 +12,7 @@ from patroni.postgresql.misc import fsync_dir from patroni.postgresql.slots import SlotsAdvanceThread, SlotsHandler from patroni.tags import Tags -from . import BaseTestPostgresql, psycopg_connect, MockCursor +from . import BaseTestPostgresql, MockCursor, psycopg_connect class TestTags(Tags): diff --git a/tests/test_sync.py b/tests/test_sync.py index 70913b12..3dfa99de 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -7,7 +7,7 @@ from patroni.collections import CaseInsensitiveSet from patroni.dcs import Cluster, ClusterConfig, Status, SyncState from patroni.postgresql import Postgresql -from . import BaseTestPostgresql, psycopg_connect, mock_available_gucs +from . import BaseTestPostgresql, mock_available_gucs, psycopg_connect @patch('subprocess.call', Mock(return_value=0)) diff --git a/tests/test_utils.py b/tests/test_utils.py index de20927a..dc7d3cce 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,8 +1,9 @@ import unittest + from unittest.mock import Mock, patch from patroni.exceptions import PatroniException -from patroni.utils import Retry, RetryFailedError, enable_keepalive, polling_loop, validate_directory, unquote +from patroni.utils import enable_keepalive, polling_loop, Retry, RetryFailedError, unquote, validate_directory class TestUtils(unittest.TestCase): diff --git a/tests/test_validator.py b/tests/test_validator.py index 098c7b07..217a4b5c 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -5,9 +5,10 @@ import tempfile import unittest from io import StringIO -from unittest.mock import Mock, patch, mock_open +from unittest.mock import Mock, mock_open, patch + from patroni.dcs import dcs_modules -from patroni.validator import schema, Directory, Schema +from patroni.validator import Directory, schema, Schema available_dcs = [m.split(".")[-1] for m in dcs_modules()] config = { diff --git a/tests/test_wale_restore.py b/tests/test_wale_restore.py index a3a4274c..427de53f 100644 --- a/tests/test_wale_restore.py +++ b/tests/test_wale_restore.py @@ -1,11 +1,13 @@ import subprocess import unittest + from threading import current_thread -from unittest.mock import Mock, PropertyMock, patch, mock_open +from unittest.mock import Mock, mock_open, patch, PropertyMock import patroni.psycopg as psycopg + from patroni.scripts import wale_restore -from patroni.scripts.wale_restore import WALERestore, main as _main, get_major_version +from patroni.scripts.wale_restore import get_major_version, main as _main, WALERestore from . import MockConnect, psycopg_connect diff --git a/tests/test_watchdog.py b/tests/test_watchdog.py index eb70778b..62dc6ccb 100644 --- a/tests/test_watchdog.py +++ b/tests/test_watchdog.py @@ -1,9 +1,11 @@ import ctypes -import patroni.watchdog.linux as linuxwd +import os import sys import unittest -import os -from unittest.mock import patch, Mock, PropertyMock + +from unittest.mock import Mock, patch, PropertyMock + +import patroni.watchdog.linux as linuxwd from patroni.watchdog import Watchdog, WatchdogError from patroni.watchdog.base import NullWatchdog diff --git a/tests/test_zookeeper.py b/tests/test_zookeeper.py index 92796624..7a4c3538 100644 --- a/tests/test_zookeeper.py +++ b/tests/test_zookeeper.py @@ -1,12 +1,14 @@ import select import unittest -from unittest.mock import Mock, PropertyMock, patch + +from unittest.mock import Mock, patch, PropertyMock from kazoo.client import KazooClient -from kazoo.exceptions import NoNodeError, NodeExistsError +from kazoo.exceptions import NodeExistsError, NoNodeError from kazoo.handlers.threading import SequentialThreadingHandler from kazoo.protocol.states import KeeperState, WatchedEvent, ZnodeStat from kazoo.retry import RetryFailedError + from patroni.dcs import get_dcs from patroni.dcs.zookeeper import Cluster, PatroniKazooClient, \ PatroniSequentialThreadingHandler, ZooKeeper, ZooKeeperError diff --git a/tox.ini b/tox.ini index 339344a5..783bb2fb 100644 --- a/tox.ini +++ b/tox.ini @@ -241,3 +241,12 @@ change_dir = docs [flake8] max-line-length = 120 ignore = D401,W503 + +[isort] +line_length = 120 +multi_line_output = 2 +balanced_wrapping = true +combine_as_imports = true +force_alphabetical_sort_within_sections = true +lines_between_types = 1 +known_third_party = consul