From c7a925a238e22b339ba18563767882b1b43d43f2 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 6 Dec 2022 13:15:56 +0100 Subject: [PATCH] Switch from localkube to kind and/or k3d (#2465) The only advantage of localkube was being a low weight. Anything else started creating only problems: 1. It is not properly maintained for many years. 2. It effectively worked only on Linux, but stopped on modern version due to changes in iptables. Instead, we will use widely adoped tools like kind or k3s. The "kind-kind" is the default K8s context (see ~/.kube/config), but it could be overriden using `PATRONI_KUBERNETES_CONTEXT` environment variable. When executed from GH actions the context is set to k3d-k3s-default, because K3s is much faster to start. --- .github/workflows/install_deps.py | 54 +------------------------------ .github/workflows/run_tests.py | 2 ++ .github/workflows/tests.yaml | 8 ++++- features/environment.py | 4 +-- patroni/dcs/kubernetes.py | 41 ++++++++++++++++++++--- tests/test_kubernetes.py | 17 +++++++++- 6 files changed, 64 insertions(+), 62 deletions(-) diff --git a/.github/workflows/install_deps.py b/.github/workflows/install_deps.py index 5fe211db..18c47e4b 100644 --- a/.github/workflows/install_deps.py +++ b/.github/workflows/install_deps.py @@ -5,7 +5,6 @@ import subprocess import stat import sys import tarfile -import time import zipfile @@ -47,13 +46,7 @@ def install_packages(what): packages = packages.get(what, []) ver = versions.get(what) subprocess.call(['sudo', 'apt-get', 'update', '-y']) - subprocess.call(['sudo', 'apt-get', 'install', '-y', 'wget', 'ca-certificates', 'gnupg', 'expect-dev']) - subprocess.call(['sudo', 'sh', '-c', "wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc" - " | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg"]) - subprocess.call(['sudo', 'sed', '-i', 's/pgdg main.*$/pgdg main {0}/'.format(ver), - '/etc/apt/sources.list.d/pgdg.list']) - subprocess.call(['sudo', 'apt-get', 'update', '-y']) - return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver] + packages) + return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev'] + packages) def get_file(url, name): @@ -127,57 +120,12 @@ def install_postgres(): return 0 -def setup_kubernetes(): - get_file('https://storage.googleapis.com/minikube/k8sReleases/v1.7.0/localkube-linux-amd64', 'localkube') - chmod_755('localkube') - - devnull = open(os.devnull, 'w') - subprocess.Popen(['sudo', 'nohup', './localkube', '--logtostderr=true', '--enable-dns=false'], - stdout=devnull, stderr=devnull) - for _ in range(0, 120): - if subprocess.call(['wget', '-qO', '-', 'http://127.0.0.1:8080/'], stdout=devnull, stderr=devnull) == 0: - break - time.sleep(1) - else: - print('localkube did not start') - return 1 - - subprocess.call('sudo chmod 644 /var/lib/localkube/certs/*', shell=True) - print('Set up .kube/config') - kube = os.path.join(os.path.expanduser('~'), '.kube') - os.makedirs(kube) - with open(os.path.join(kube, 'config'), 'w') as f: - f.write("""apiVersion: v1 -clusters: -- cluster: - certificate-authority: /var/lib/localkube/certs/ca.crt - server: https://127.0.0.1:8443 - name: local -contexts: -- context: - cluster: local - user: myself - name: local -current-context: local -kind: Config -preferences: {} -users: -- name: myself - user: - client-certificate: /var/lib/localkube/certs/apiserver.crt - client-key: /var/lib/localkube/certs/apiserver.key -""") - return 0 - - def main(): what = os.environ.get('DCS', sys.argv[1] if len(sys.argv) > 1 else 'all') if what != 'all': if sys.platform.startswith('linux'): r = install_packages(what) - if r == 0 and what == 'kubernetes': - r = setup_kubernetes() else: r = install_postgres() diff --git a/.github/workflows/run_tests.py b/.github/workflows/run_tests.py index 3ca52b81..7395c393 100644 --- a/.github/workflows/run_tests.py +++ b/.github/workflows/run_tests.py @@ -35,6 +35,8 @@ def main(): unbuffer = [] env['PATH'] = path + os.pathsep + env['PATH'] env['DCS'] = what + if what == 'kubernetes': + env['PATRONI_KUBERNETES_CONTEXT'] = 'k3d-k3s-default' ret = subprocess.call(unbuffer + [sys.executable, '-m', 'behave'], env=env) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1c261f87..0ff62f03 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -120,8 +120,14 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - uses: nolar/setup-k3d-k3s@v1 + if: matrix.dcs == 'kubernetes' - name: Add postgresql apt repo - run: sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + run: | + sudo apt-get update -y + sudo apt-get install -y wget ca-certificates gnupg + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + sudo sh -c 'wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg' if: matrix.os == 'ubuntu' - name: Install dependencies run: python .github/workflows/install_deps.py diff --git a/features/environment.py b/features/environment.py index 83bc83ea..09ea4d18 100644 --- a/features/environment.py +++ b/features/environment.py @@ -476,10 +476,10 @@ class KubernetesController(AbstractDcsController): self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items()) os.environ['PATRONI_KUBERNETES_LABELS'] = json.dumps(self._labels) os.environ['PATRONI_KUBERNETES_USE_ENDPOINTS'] = 'true' - os.environ['PATRONI_KUBERNETES_BYPASS_API_SERVICE'] = 'true' + os.environ.setdefault('PATRONI_KUBERNETES_BYPASS_API_SERVICE', 'true') from patroni.dcs.kubernetes import k8s_client, k8s_config - k8s_config.load_kube_config(context='local') + k8s_config.load_kube_config(context=os.environ.setdefault('PATRONI_KUBERNETES_CONTEXT', 'kind-kind')) self._client = k8s_client self._api = self._client.CoreV1Api() diff --git a/patroni/dcs/kubernetes.py b/patroni/dcs/kubernetes.py index 2b31e1bc..21625b87 100644 --- a/patroni/dcs/kubernetes.py +++ b/patroni/dcs/kubernetes.py @@ -1,3 +1,5 @@ +import atexit +import base64 import datetime import functools import json @@ -7,6 +9,7 @@ import random import socket import six import sys +import tempfile import time import urllib3 import yaml @@ -28,12 +31,34 @@ SERVICE_HOST_ENV_NAME = 'KUBERNETES_SERVICE_HOST' SERVICE_PORT_ENV_NAME = 'KUBERNETES_SERVICE_PORT' SERVICE_TOKEN_FILENAME = '/var/run/secrets/kubernetes.io/serviceaccount/token' SERVICE_CERT_FILENAME = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' +__temp_files = [] class KubernetesError(DCSError): pass +def _cleanup_temp_files(): + global __temp_files + for temp_file in __temp_files: + try: + os.remove(temp_file) + except OSError: + pass + __temp_files = [] + + +def _create_temp_file(content): + if len(__temp_files) == 0: + atexit.register(_cleanup_temp_files) + + fd, name = tempfile.mkstemp() + os.write(fd, content) + os.close(fd) + __temp_files.append(name) + return name + + # this function does the same mapping of snake_case => camelCase for > 97% of cases as autogenerated swagger code def to_camel_case(value): reserved = {'api', 'apiv3', 'cidr', 'cpu', 'csi', 'id', 'io', 'ip', 'ipc', 'pid', 'tls', 'uri', 'url', 'uuid'} @@ -93,6 +118,13 @@ class K8sConfig(object): if c['name'] == name: return c[section] + def _pool_config_from_file_or_data(self, config, file_key_name, pool_key_name): + data_key_name = file_key_name + '-data' + if data_key_name in config: + self.pool_config[pool_key_name] = _create_temp_file(base64.b64decode(config[data_key_name])) + elif file_key_name in config: + self.pool_config[pool_key_name] = config[file_key_name] + def load_kube_config(self, context=None): with open(os.path.expanduser(KUBE_CONFIG_DEFAULT_LOCATION)) as f: config = yaml.safe_load(f) @@ -103,10 +135,9 @@ class K8sConfig(object): self._server = cluster['server'].rstrip('/') if self._server.startswith('https'): - self.pool_config.update({v: user[k] for k, v in {'client-certificate': 'cert_file', - 'client-key': 'key_file'}.items() if k in user}) - if 'certificate-authority' in cluster: - self.pool_config['ca_certs'] = cluster['certificate-authority'] + self._pool_config_from_file_or_data(user, 'client-certificate', 'cert_file') + self._pool_config_from_file_or_data(user, 'client-key', 'key_file') + self._pool_config_from_file_or_data(cluster, 'certificate-authority', 'ca_certs') self.pool_config['cert_reqs'] = 'CERT_NONE' if cluster.get('insecure-skip-tls-verify') else 'CERT_REQUIRED' if user.get('token'): self._make_headers(token=user['token']) @@ -677,7 +708,7 @@ class Kubernetes(AbstractDCS): try: k8s_config.load_incluster_config(ca_certs=self._ca_certs) except k8s_config.ConfigException: - k8s_config.load_kube_config(context=config.get('context', 'local')) + k8s_config.load_kube_config(context=config.get('context', 'kind-kind')) self.__my_pod = None self.__ips = [] if config.get('patronictl') else [config.get('pod_ip')] diff --git a/tests/test_kubernetes.py b/tests/test_kubernetes.py index fe50f4b7..79bde32b 100644 --- a/tests/test_kubernetes.py +++ b/tests/test_kubernetes.py @@ -1,10 +1,11 @@ +import base64 import datetime import json import socket import time import unittest -from mock import Mock, PropertyMock, mock_open, patch +from mock import call, Mock, PropertyMock, mock_open, patch from patroni.dcs.kubernetes import k8s_client, k8s_config, K8sConfig, K8sConnectionFailed,\ K8sException, K8sObject, Kubernetes, KubernetesError, KubernetesRetriableException,\ Retry, RetryFailedError, SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME @@ -121,6 +122,20 @@ class TestK8sConfig(unittest.TestCase): k8s_config.load_kube_config() self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer token') + config["users"][0]["user"]["client-key-data"] = base64.b64encode(b'foobar').decode('utf-8') + config["clusters"][0]["cluster"]["certificate-authority-data"] = base64.b64encode(b'foobar').decode('utf-8') + with patch.object(builtins, 'open', mock_open(read_data=json.dumps(config))),\ + patch('os.write', Mock()), patch('os.close', Mock()),\ + patch('os.remove') as mock_remove,\ + patch('atexit.register') as mock_atexit,\ + patch('tempfile.mkstemp') as mock_mkstemp: + mock_mkstemp.side_effect = [(3, '1.tmp'), (4, '2.tmp')] + k8s_config.load_kube_config() + mock_atexit.assert_called_once() + mock_remove.side_effect = OSError + mock_atexit.call_args[0][0]() # call _cleanup_temp_files + mock_remove.assert_has_calls([call('1.tmp'), call('2.tmp')]) + @patch('urllib3.PoolManager.request') class TestApiClient(unittest.TestCase):