gitops/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-08-25 14:53:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use LIST + WATCH when working with K8s API (#1276)

Browse Source 
There is an opinion that LIST requests with labelSelector to K8s API are expensive and Patroni was doing two such requests per HA loop (LIST pods and LIST endpoints/configmaps).
To efficiently detect object changes we will switch to the LIST+WATCH approach.
The initial LIST request populates the ObjectCache and events from the WATCH request update it.

In addition to that, the ObjectCache will be updated after performing the UPDATE operations on the K8s objects. To avoid race conditions, all operations on ObjectCache are performed after comparing the resource_version of the old and the new objects and rejected if the new resource_version value is smaller than the old one.

The disadvantage of such an approach is that it will require keeping three connections to the K8s API from each Patroni Pod (previously it was two).

Yesterday I deployed this feature branch on our biggest K8s cluster, with ~300 Patroni pods.
The CPU Utilization on K8s master nodes immediately dropped from ~20% to ~10% (two times), and the incoming traffic on master nodes dropped ~7-8 times!

Last, but not least, we get more or less the same impact on etcd cluster behind K8s master nodes, the CPU Utilization dropped nearly twice and outgoing traffic ~7-8 times.
This commit is contained in:
Alexander Kukushkin
2019-11-14 14:54:57 +01:00
committed by GitHub
parent c1adbafbc5
commit 66d77697ae
2 changed files with 162 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_kubernetes.py
+41 -33
View File
@@ -1,8 +1,9 @@
import json
import time
import unittest
from mock import Mock, patch
from patroni.dcs.kubernetes import Kubernetes, KubernetesError, k8s_client, k8s_watch, RetryFailedError
from patroni.dcs.kubernetes import Kubernetes, KubernetesError, k8s_client, RetryFailedError
from threading import Thread
from . import SleepException
@@ -18,17 +19,23 @@ def mock_list_namespaced_config_map(self, *args, **kwargs):
metadata.update({'name': 'test-sync', 'annotations': {'leader': 'p-0'}})
items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
metadata = k8s_client.V1ObjectMeta(resource_version='1')
return k8s_client.V1ConfigMapList(metadata=metadata, items=items)
return k8s_client.V1ConfigMapList(metadata=metadata, items=items, kind='ConfigMapList')
def mock_list_namespaced_pod(self, *args, **kwargs):
metadata = k8s_client.V1ObjectMeta(resource_version='1', name='p-0', annotations={'status': '{}'})
items = [k8s_client.V1Pod(metadata=metadata)]
return k8s_client.V1PodList(items=items)
return k8s_client.V1PodList(items=items, kind='PodList')
@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', Mock())
@patch.object(k8s_client.CoreV1Api, 'create_namespaced_config_map', Mock())
def mock_config_map(*args, **kwargs):
mock = Mock()
mock.metadata.resource_version = '2'
return mock
@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', mock_config_map)
@patch.object(k8s_client.CoreV1Api, 'create_namespaced_config_map', mock_config_map)
@patch('kubernetes.client.api_client.ThreadPool', Mock(), create=True)
@patch.object(Thread, 'start', Mock())
class TestKubernetes(unittest.TestCase):
@@ -40,15 +47,25 @@ class TestKubernetes(unittest.TestCase):
@patch.object(Thread, 'start', Mock())
def setUp(self):
self.k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10, 'labels': {'f': 'b'}})
self.assertRaises(AttributeError, self.k._pods._build_cache)
self.k._pods._is_ready = True
self.assertRaises(AttributeError, self.k._kinds._build_cache)
self.k._kinds._is_ready = True
self.k.get_cluster()
@patch('time.time', Mock(side_effect=[1, 10.9, 100]))
def test__wait_caches(self):
self.k._pods._is_ready = False
with self.k._condition:
self.assertRaises(RetryFailedError, self.k._wait_caches)
def test_get_cluster(self):
with patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map), \
patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod), \
patch('time.time', Mock(return_value=time.time() + 31)):
self.k.get_cluster()
with patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', Mock(side_effect=Exception)):
with patch.object(Kubernetes, '_wait_caches', Mock(side_effect=Exception)):
self.assertRaises(KubernetesError, self.k.get_cluster)
@patch('kubernetes.config.load_kube_config', Mock())
@@ -107,10 +124,6 @@ class TestKubernetes(unittest.TestCase):
'labels': {'f': 'b'}, 'use_endpoints': True, 'pod_ip': '10.0.0.0'})
self.assertFalse(k.delete_sync_state())
@patch.object(k8s_watch.Watch, 'stream', Mock(return_value=[{'raw_object': {'metadata': {}}}]))
def test_start_watch_stream(self):
self.assertIsNotNone(self.k.start_watch_stream())
def test_watch(self):
self.k.set_ttl(10)
self.k.watch(None, 0)
@@ -120,6 +133,7 @@ class TestKubernetes(unittest.TestCase):
self.k.set_history_value('{}')
@patch('kubernetes.config.load_kube_config', Mock())
@patch('patroni.dcs.kubernetes.ObjectCache', Mock())
@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_pod', Mock(return_value=True))
@patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints', Mock())
@patch.object(k8s_client.CoreV1Api, 'create_namespaced_service',
@@ -132,9 +146,7 @@ class TestKubernetes(unittest.TestCase):
k.touch_member({'state': 'running', 'role': 'replica'})
@patch('time.sleep', Mock(side_effect=SleepException))
@patch.object(Kubernetes, 'start_watch_stream')
class TestKubernetesWatcher(unittest.TestCase):
class TestCacheBuilder(unittest.TestCase):
@patch('kubernetes.config.load_kube_config', Mock())
@patch('kubernetes.client.api_client.ThreadPool', Mock(), create=True)
@@ -142,24 +154,20 @@ class TestKubernetesWatcher(unittest.TestCase):
def setUp(self):
self.k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10, 'labels': {'f': 'b'}})
def test_leader_update(self, mock_stream):
mock_stream.return_value = [
{'raw_object': {'type': 'MODIFIED',
'metadata': {'name': self.k.leader_path, 'annotations': {self.k._LEADER: 'foo'}}}},
{'raw_object': {'type': 'MODIFIED',
'metadata': {'name': self.k.leader_path, 'annotations': {self.k._LEADER: 'bar'}}}}
]
self.assertRaises(SleepException, self.k._watcher.run)
@patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map)
@patch('patroni.dcs.kubernetes.ObjectCache._watch')
def test__build_cache(self, mock_response):
mock_response.return_value.read_chunked.return_value = json.dumps(
{'type': 'MODIFIED', 'object': {'metadata': {
'name': self.k.config_path, 'resourceVersion': '2', 'annotations': {self.k._CONFIG: 'foo'}}}}
) + '\n' + json.dumps(
{'type': 'DELETED', 'object': {'metadata': {
'name': self.k.config_path, 'resourceVersion': '3'}}}
) + '\n' + json.dumps(
{'type': 'MDIFIED', 'object': {'metadata': {'name': self.k.config_path}}}
) + '\n' + json.dumps({'object': {'code': 410}}) + '\n'
self.k._kinds._build_cache()
def test_config_update(self, mock_stream):
mock_stream.return_value = [
{'raw_object': {'type': 'MODIFIED',
'metadata': {'name': self.k.config_path, 'annotations': {self.k._CONFIG: 'foo'}}}},
{'raw_object': {'type': 'MODIFIED',
'metadata': {'name': self.k.config_path, 'annotations': {self.k._CONFIG: 'bar'}}}}
]
self.assertRaises(SleepException, self.k._watcher.run)
def test_run(self, mock_stream):
mock_stream.side_effect = Exception
self.assertRaises(SleepException, self.k._watcher.run)
@patch('patroni.dcs.kubernetes.logger.error', Mock(side_effect=SleepException))
def test_run(self):
self.assertRaises(SleepException, self.k._pods.run)