Switch from localkube to kind and/or k3d (#2465)

The only advantage of localkube was being a low weight. Anything else started creating only problems:
1. It is not properly maintained for many years.
2. It effectively worked only on Linux, but stopped on modern version due to changes in iptables.

Instead, we will use widely adoped tools like kind or k3s. The "kind-kind" is the default K8s context (see ~/.kube/config), but it could be overriden using `PATRONI_KUBERNETES_CONTEXT` environment variable. When executed from GH actions the context is set to k3d-k3s-default, because K3s is much faster to start.

.github/workflows/install_deps.py

@@ -5,7 +5,6 @@ import subprocess
 import stat
 import sys
 import tarfile
-import time
 import zipfile
 
 
@@ -47,13 +46,7 @@ def install_packages(what):
     packages = packages.get(what, [])
     ver = versions.get(what)
     subprocess.call(['sudo', 'apt-get', 'update', '-y'])
-    subprocess.call(['sudo', 'apt-get', 'install', '-y', 'wget', 'ca-certificates', 'gnupg', 'expect-dev'])
-    subprocess.call(['sudo', 'sh', '-c', "wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc"
-                     " | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg"])
-    subprocess.call(['sudo', 'sed', '-i', 's/pgdg main.*$/pgdg main {0}/'.format(ver),
-                     '/etc/apt/sources.list.d/pgdg.list'])
-    subprocess.call(['sudo', 'apt-get', 'update', '-y'])
-    return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver] + packages)
+    return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev'] + packages)
 
 
 def get_file(url, name):
@@ -127,57 +120,12 @@ def install_postgres():
     return 0
 
 
-def setup_kubernetes():
-    get_file('https://storage.googleapis.com/minikube/k8sReleases/v1.7.0/localkube-linux-amd64', 'localkube')
-    chmod_755('localkube')
-
-    devnull = open(os.devnull, 'w')
-    subprocess.Popen(['sudo', 'nohup', './localkube', '--logtostderr=true', '--enable-dns=false'],
-                     stdout=devnull, stderr=devnull)
-    for _ in range(0, 120):
-        if subprocess.call(['wget', '-qO', '-', 'http://127.0.0.1:8080/'], stdout=devnull, stderr=devnull) == 0:
-            break
-        time.sleep(1)
-    else:
-        print('localkube did not start')
-        return 1
-
-    subprocess.call('sudo chmod 644 /var/lib/localkube/certs/*', shell=True)
-    print('Set up .kube/config')
-    kube = os.path.join(os.path.expanduser('~'), '.kube')
-    os.makedirs(kube)
-    with open(os.path.join(kube, 'config'), 'w') as f:
-        f.write("""apiVersion: v1
-clusters:
-- cluster:
-    certificate-authority: /var/lib/localkube/certs/ca.crt
-    server: https://127.0.0.1:8443
-  name: local
-contexts:
-- context:
-    cluster: local
-    user: myself
-  name: local
-current-context: local
-kind: Config
-preferences: {}
-users:
-- name: myself
-  user:
-    client-certificate: /var/lib/localkube/certs/apiserver.crt
-    client-key: /var/lib/localkube/certs/apiserver.key
-""")
-    return 0
-
-
 def main():
     what = os.environ.get('DCS', sys.argv[1] if len(sys.argv) > 1 else 'all')
 
     if what != 'all':
         if sys.platform.startswith('linux'):
             r = install_packages(what)
-            if r == 0 and what == 'kubernetes':
-                r = setup_kubernetes()
         else:
             r = install_postgres()
 

.github/workflows/run_tests.py

@@ -35,6 +35,8 @@ def main():
         unbuffer = []
     env['PATH'] = path + os.pathsep + env['PATH']
     env['DCS'] = what
+    if what == 'kubernetes':
+        env['PATRONI_KUBERNETES_CONTEXT'] = 'k3d-k3s-default'
 
     ret = subprocess.call(unbuffer + [sys.executable, '-m', 'behave'], env=env)
 

.github/workflows/tests.yaml

@@ -120,8 +120,14 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
+    - uses: nolar/setup-k3d-k3s@v1
+      if: matrix.dcs == 'kubernetes'
     - name: Add postgresql apt repo
-      run: sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+      run: |
+        sudo apt-get update -y
+        sudo apt-get install -y wget ca-certificates gnupg
+        sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+        sudo sh -c 'wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg'
       if: matrix.os == 'ubuntu'
     - name: Install dependencies
       run: python .github/workflows/install_deps.py

features/environment.py

@@ -476,10 +476,10 @@ class KubernetesController(AbstractDcsController):
         self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
         os.environ['PATRONI_KUBERNETES_LABELS'] = json.dumps(self._labels)
         os.environ['PATRONI_KUBERNETES_USE_ENDPOINTS'] = 'true'
-        os.environ['PATRONI_KUBERNETES_BYPASS_API_SERVICE'] = 'true'
+        os.environ.setdefault('PATRONI_KUBERNETES_BYPASS_API_SERVICE', 'true')
 
         from patroni.dcs.kubernetes import k8s_client, k8s_config
-        k8s_config.load_kube_config(context='local')
+        k8s_config.load_kube_config(context=os.environ.setdefault('PATRONI_KUBERNETES_CONTEXT', 'kind-kind'))
         self._client = k8s_client
         self._api = self._client.CoreV1Api()
 

patroni/dcs/kubernetes.py

@@ -1,3 +1,5 @@
+import atexit
+import base64
 import datetime
 import functools
 import json
@@ -7,6 +9,7 @@ import random
 import socket
 import six
 import sys
+import tempfile
 import time
 import urllib3
 import yaml
@@ -28,12 +31,34 @@ SERVICE_HOST_ENV_NAME = 'KUBERNETES_SERVICE_HOST'
 SERVICE_PORT_ENV_NAME = 'KUBERNETES_SERVICE_PORT'
 SERVICE_TOKEN_FILENAME = '/var/run/secrets/kubernetes.io/serviceaccount/token'
 SERVICE_CERT_FILENAME = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'
+__temp_files = []
 
 
 class KubernetesError(DCSError):
     pass
 
 
+def _cleanup_temp_files():
+    global __temp_files
+    for temp_file in __temp_files:
+        try:
+            os.remove(temp_file)
+        except OSError:
+            pass
+    __temp_files = []
+
+
+def _create_temp_file(content):
+    if len(__temp_files) == 0:
+        atexit.register(_cleanup_temp_files)
+
+    fd, name = tempfile.mkstemp()
+    os.write(fd, content)
+    os.close(fd)
+    __temp_files.append(name)
+    return name
+
+
 # this function does the same mapping of snake_case => camelCase for > 97% of cases as autogenerated swagger code
 def to_camel_case(value):
     reserved = {'api', 'apiv3', 'cidr', 'cpu', 'csi', 'id', 'io', 'ip', 'ipc', 'pid', 'tls', 'uri', 'url', 'uuid'}
@@ -93,6 +118,13 @@ class K8sConfig(object):
             if c['name'] == name:
                 return c[section]
 
+    def _pool_config_from_file_or_data(self, config, file_key_name, pool_key_name):
+        data_key_name = file_key_name + '-data'
+        if data_key_name in config:
+            self.pool_config[pool_key_name] = _create_temp_file(base64.b64decode(config[data_key_name]))
+        elif file_key_name in config:
+            self.pool_config[pool_key_name] = config[file_key_name]
+
     def load_kube_config(self, context=None):
         with open(os.path.expanduser(KUBE_CONFIG_DEFAULT_LOCATION)) as f:
             config = yaml.safe_load(f)
@@ -103,10 +135,9 @@ class K8sConfig(object):
 
         self._server = cluster['server'].rstrip('/')
         if self._server.startswith('https'):
-            self.pool_config.update({v: user[k] for k, v in {'client-certificate': 'cert_file',
-                                                             'client-key': 'key_file'}.items() if k in user})
-            if 'certificate-authority' in cluster:
-                self.pool_config['ca_certs'] = cluster['certificate-authority']
+            self._pool_config_from_file_or_data(user, 'client-certificate', 'cert_file')
+            self._pool_config_from_file_or_data(user, 'client-key', 'key_file')
+            self._pool_config_from_file_or_data(cluster, 'certificate-authority', 'ca_certs')
             self.pool_config['cert_reqs'] = 'CERT_NONE' if cluster.get('insecure-skip-tls-verify') else 'CERT_REQUIRED'
         if user.get('token'):
             self._make_headers(token=user['token'])
@@ -677,7 +708,7 @@ class Kubernetes(AbstractDCS):
         try:
             k8s_config.load_incluster_config(ca_certs=self._ca_certs)
         except k8s_config.ConfigException:
-            k8s_config.load_kube_config(context=config.get('context', 'local'))
+            k8s_config.load_kube_config(context=config.get('context', 'kind-kind'))
 
         self.__my_pod = None
         self.__ips = [] if config.get('patronictl') else [config.get('pod_ip')]

tests/test_kubernetes.py

@@ -1,10 +1,11 @@
+import base64
 import datetime
 import json
 import socket
 import time
 import unittest
 
-from mock import Mock, PropertyMock, mock_open, patch
+from mock import call, Mock, PropertyMock, mock_open, patch
 from patroni.dcs.kubernetes import k8s_client, k8s_config, K8sConfig, K8sConnectionFailed,\
         K8sException, K8sObject, Kubernetes, KubernetesError, KubernetesRetriableException,\
         Retry, RetryFailedError, SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME
@@ -121,6 +122,20 @@ class TestK8sConfig(unittest.TestCase):
             k8s_config.load_kube_config()
             self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer token')
 
+        config["users"][0]["user"]["client-key-data"] = base64.b64encode(b'foobar').decode('utf-8')
+        config["clusters"][0]["cluster"]["certificate-authority-data"] = base64.b64encode(b'foobar').decode('utf-8')
+        with patch.object(builtins, 'open', mock_open(read_data=json.dumps(config))),\
+                patch('os.write', Mock()), patch('os.close', Mock()),\
+                patch('os.remove') as mock_remove,\
+                patch('atexit.register') as mock_atexit,\
+                patch('tempfile.mkstemp') as mock_mkstemp:
+            mock_mkstemp.side_effect = [(3, '1.tmp'), (4, '2.tmp')]
+            k8s_config.load_kube_config()
+            mock_atexit.assert_called_once()
+            mock_remove.side_effect = OSError
+            mock_atexit.call_args[0][0]()  # call _cleanup_temp_files
+            mock_remove.assert_has_calls([call('1.tmp'), call('2.tmp')])
+
 
 @patch('urllib3.PoolManager.request')
 class TestApiClient(unittest.TestCase):