diff --git a/.github/workflows/install_deps.py b/.github/workflows/install_deps.py index 5fe211db..18c47e4b 100644 --- a/.github/workflows/install_deps.py +++ b/.github/workflows/install_deps.py @@ -5,7 +5,6 @@ import subprocess import stat import sys import tarfile -import time import zipfile @@ -47,13 +46,7 @@ def install_packages(what): packages = packages.get(what, []) ver = versions.get(what) subprocess.call(['sudo', 'apt-get', 'update', '-y']) - subprocess.call(['sudo', 'apt-get', 'install', '-y', 'wget', 'ca-certificates', 'gnupg', 'expect-dev']) - subprocess.call(['sudo', 'sh', '-c', "wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc" - " | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg"]) - subprocess.call(['sudo', 'sed', '-i', 's/pgdg main.*$/pgdg main {0}/'.format(ver), - '/etc/apt/sources.list.d/pgdg.list']) - subprocess.call(['sudo', 'apt-get', 'update', '-y']) - return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver] + packages) + return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev'] + packages) def get_file(url, name): @@ -127,57 +120,12 @@ def install_postgres(): return 0 -def setup_kubernetes(): - get_file('https://storage.googleapis.com/minikube/k8sReleases/v1.7.0/localkube-linux-amd64', 'localkube') - chmod_755('localkube') - - devnull = open(os.devnull, 'w') - subprocess.Popen(['sudo', 'nohup', './localkube', '--logtostderr=true', '--enable-dns=false'], - stdout=devnull, stderr=devnull) - for _ in range(0, 120): - if subprocess.call(['wget', '-qO', '-', 'http://127.0.0.1:8080/'], stdout=devnull, stderr=devnull) == 0: - break - time.sleep(1) - else: - print('localkube did not start') - return 1 - - subprocess.call('sudo chmod 644 /var/lib/localkube/certs/*', shell=True) - print('Set up .kube/config') - kube = os.path.join(os.path.expanduser('~'), '.kube') - os.makedirs(kube) - with open(os.path.join(kube, 'config'), 'w') as f: - f.write("""apiVersion: v1 -clusters: -- cluster: - certificate-authority: /var/lib/localkube/certs/ca.crt - server: https://127.0.0.1:8443 - name: local -contexts: -- context: - cluster: local - user: myself - name: local -current-context: local -kind: Config -preferences: {} -users: -- name: myself - user: - client-certificate: /var/lib/localkube/certs/apiserver.crt - client-key: /var/lib/localkube/certs/apiserver.key -""") - return 0 - - def main(): what = os.environ.get('DCS', sys.argv[1] if len(sys.argv) > 1 else 'all') if what != 'all': if sys.platform.startswith('linux'): r = install_packages(what) - if r == 0 and what == 'kubernetes': - r = setup_kubernetes() else: r = install_postgres() diff --git a/.github/workflows/run_tests.py b/.github/workflows/run_tests.py index 3ca52b81..7395c393 100644 --- a/.github/workflows/run_tests.py +++ b/.github/workflows/run_tests.py @@ -35,6 +35,8 @@ def main(): unbuffer = [] env['PATH'] = path + os.pathsep + env['PATH'] env['DCS'] = what + if what == 'kubernetes': + env['PATRONI_KUBERNETES_CONTEXT'] = 'k3d-k3s-default' ret = subprocess.call(unbuffer + [sys.executable, '-m', 'behave'], env=env) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1c261f87..0ff62f03 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -120,8 +120,14 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - uses: nolar/setup-k3d-k3s@v1 + if: matrix.dcs == 'kubernetes' - name: Add postgresql apt repo - run: sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + run: | + sudo apt-get update -y + sudo apt-get install -y wget ca-certificates gnupg + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + sudo sh -c 'wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg' if: matrix.os == 'ubuntu' - name: Install dependencies run: python .github/workflows/install_deps.py diff --git a/features/environment.py b/features/environment.py index 83bc83ea..09ea4d18 100644 --- a/features/environment.py +++ b/features/environment.py @@ -476,10 +476,10 @@ class KubernetesController(AbstractDcsController): self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items()) os.environ['PATRONI_KUBERNETES_LABELS'] = json.dumps(self._labels) os.environ['PATRONI_KUBERNETES_USE_ENDPOINTS'] = 'true' - os.environ['PATRONI_KUBERNETES_BYPASS_API_SERVICE'] = 'true' + os.environ.setdefault('PATRONI_KUBERNETES_BYPASS_API_SERVICE', 'true') from patroni.dcs.kubernetes import k8s_client, k8s_config - k8s_config.load_kube_config(context='local') + k8s_config.load_kube_config(context=os.environ.setdefault('PATRONI_KUBERNETES_CONTEXT', 'kind-kind')) self._client = k8s_client self._api = self._client.CoreV1Api() diff --git a/patroni/dcs/kubernetes.py b/patroni/dcs/kubernetes.py index 2b31e1bc..21625b87 100644 --- a/patroni/dcs/kubernetes.py +++ b/patroni/dcs/kubernetes.py @@ -1,3 +1,5 @@ +import atexit +import base64 import datetime import functools import json @@ -7,6 +9,7 @@ import random import socket import six import sys +import tempfile import time import urllib3 import yaml @@ -28,12 +31,34 @@ SERVICE_HOST_ENV_NAME = 'KUBERNETES_SERVICE_HOST' SERVICE_PORT_ENV_NAME = 'KUBERNETES_SERVICE_PORT' SERVICE_TOKEN_FILENAME = '/var/run/secrets/kubernetes.io/serviceaccount/token' SERVICE_CERT_FILENAME = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' +__temp_files = [] class KubernetesError(DCSError): pass +def _cleanup_temp_files(): + global __temp_files + for temp_file in __temp_files: + try: + os.remove(temp_file) + except OSError: + pass + __temp_files = [] + + +def _create_temp_file(content): + if len(__temp_files) == 0: + atexit.register(_cleanup_temp_files) + + fd, name = tempfile.mkstemp() + os.write(fd, content) + os.close(fd) + __temp_files.append(name) + return name + + # this function does the same mapping of snake_case => camelCase for > 97% of cases as autogenerated swagger code def to_camel_case(value): reserved = {'api', 'apiv3', 'cidr', 'cpu', 'csi', 'id', 'io', 'ip', 'ipc', 'pid', 'tls', 'uri', 'url', 'uuid'} @@ -93,6 +118,13 @@ class K8sConfig(object): if c['name'] == name: return c[section] + def _pool_config_from_file_or_data(self, config, file_key_name, pool_key_name): + data_key_name = file_key_name + '-data' + if data_key_name in config: + self.pool_config[pool_key_name] = _create_temp_file(base64.b64decode(config[data_key_name])) + elif file_key_name in config: + self.pool_config[pool_key_name] = config[file_key_name] + def load_kube_config(self, context=None): with open(os.path.expanduser(KUBE_CONFIG_DEFAULT_LOCATION)) as f: config = yaml.safe_load(f) @@ -103,10 +135,9 @@ class K8sConfig(object): self._server = cluster['server'].rstrip('/') if self._server.startswith('https'): - self.pool_config.update({v: user[k] for k, v in {'client-certificate': 'cert_file', - 'client-key': 'key_file'}.items() if k in user}) - if 'certificate-authority' in cluster: - self.pool_config['ca_certs'] = cluster['certificate-authority'] + self._pool_config_from_file_or_data(user, 'client-certificate', 'cert_file') + self._pool_config_from_file_or_data(user, 'client-key', 'key_file') + self._pool_config_from_file_or_data(cluster, 'certificate-authority', 'ca_certs') self.pool_config['cert_reqs'] = 'CERT_NONE' if cluster.get('insecure-skip-tls-verify') else 'CERT_REQUIRED' if user.get('token'): self._make_headers(token=user['token']) @@ -677,7 +708,7 @@ class Kubernetes(AbstractDCS): try: k8s_config.load_incluster_config(ca_certs=self._ca_certs) except k8s_config.ConfigException: - k8s_config.load_kube_config(context=config.get('context', 'local')) + k8s_config.load_kube_config(context=config.get('context', 'kind-kind')) self.__my_pod = None self.__ips = [] if config.get('patronictl') else [config.get('pod_ip')] diff --git a/tests/test_kubernetes.py b/tests/test_kubernetes.py index fe50f4b7..79bde32b 100644 --- a/tests/test_kubernetes.py +++ b/tests/test_kubernetes.py @@ -1,10 +1,11 @@ +import base64 import datetime import json import socket import time import unittest -from mock import Mock, PropertyMock, mock_open, patch +from mock import call, Mock, PropertyMock, mock_open, patch from patroni.dcs.kubernetes import k8s_client, k8s_config, K8sConfig, K8sConnectionFailed,\ K8sException, K8sObject, Kubernetes, KubernetesError, KubernetesRetriableException,\ Retry, RetryFailedError, SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME @@ -121,6 +122,20 @@ class TestK8sConfig(unittest.TestCase): k8s_config.load_kube_config() self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer token') + config["users"][0]["user"]["client-key-data"] = base64.b64encode(b'foobar').decode('utf-8') + config["clusters"][0]["cluster"]["certificate-authority-data"] = base64.b64encode(b'foobar').decode('utf-8') + with patch.object(builtins, 'open', mock_open(read_data=json.dumps(config))),\ + patch('os.write', Mock()), patch('os.close', Mock()),\ + patch('os.remove') as mock_remove,\ + patch('atexit.register') as mock_atexit,\ + patch('tempfile.mkstemp') as mock_mkstemp: + mock_mkstemp.side_effect = [(3, '1.tmp'), (4, '2.tmp')] + k8s_config.load_kube_config() + mock_atexit.assert_called_once() + mock_remove.side_effect = OSError + mock_atexit.call_args[0][0]() # call _cleanup_temp_files + mock_remove.assert_has_calls([call('1.tmp'), call('2.tmp')]) + @patch('urllib3.PoolManager.request') class TestApiClient(unittest.TestCase):