gitops/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-08-25 14:53:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make it possible to bypass kubernetes service (#1614)

Browse Source 
When running on K8s Patroni is communicating with API via the `kubernetes` service, which is address is exposed via the
`KUBERNETES_SERVICE_HOST` environment variable. Like any other service, the `kubernetes` service is handled by `kube-proxy`, that depending on configuration is either relying on userspace program or `iptables` for traffic routing.

During K8s upgrade, when master nodes are replaced, it is possible that `kube-proxy` doesn't update the service configuration in time and as a result Patroni fails to update the leader lock and demotes postgres.

In order to improve the user experience and get more control on the problem we make it possible to bypass the `kubernetes` service and connect directly to API nodes.
The strategy is very simple:
1. Resolve list IPs of API nodes from the kubernetes endpoint on every iteration of HA loop.
2. Stick to one of these IPs for API requests
3. Switch to a different IP if connected to IP is not from the list
4. If the request fails, switch to another IP and retry

Such a strategy is already used for Etcd and proven to work quite well.

In order to enable the feature, you need either to set to `true` `kubernetes.bypass_api_service` in the Patroni configuration file or `PATRONI_KUBERNETES_BYPASS_API_SERVICE` environment variable.

If for some reason `GET /default/endpoints/kubernetes` isn't allowed Patroni will disable the feature.
This commit is contained in:
Alexander Kukushkin
2020-08-14 12:39:47 +02:00
committed by GitHub
parent 1ab709c5f0
commit 23dcfaab49
9 changed files with 333 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_kubernetes.py
+40 -5
View File
@@ -1,10 +1,12 @@
import json
import socket
import time
import unittest
from mock import Mock, mock_open, patch
from patroni.dcs.kubernetes import Kubernetes, KubernetesError, K8sConfig, K8sObject, RetryFailedError,\
k8s_client, k8s_config, SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME
from patroni.dcs.kubernetes import k8s_client, k8s_config, K8sConfig, K8sConnectionFailed,\
K8sException, K8sObject, Kubernetes, KubernetesError, KubernetesRetriableException,\
Retry, RetryFailedError, SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME
from six.moves import builtins
from threading import Thread
from . import MockResponse, SleepException
@@ -93,12 +95,43 @@ class TestK8sConfig(unittest.TestCase):
self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer token')
@patch('urllib3.PoolManager.request')
class TestApiClient(unittest.TestCase):
@patch.object(K8sConfig, '_server', '', create=True)
@patch('urllib3.PoolManager.request', Mock())
def setUp(self):
self.a = k8s_client.ApiClient(True)
self.mock_get_ep = MockResponse()
self.mock_get_ep.content = '{"subsets":[{"ports":[{"name":"https","protocol":"TCP","port":443}],' +\
'"addresses":[{"ip":"127.0.0.1"},{"ip":"127.0.0.2"}]}]}'
def test__do_http_request(self, mock_request):
mock_request.side_effect = [self.mock_get_ep] + [socket.timeout]
self.assertRaises(K8sException, self.a.call_api, 'GET', 'f')
@patch('time.sleep', Mock())
def test_request(self, mock_request):
retry = Retry(deadline=10, max_delay=1, max_tries=1, retry_exceptions=KubernetesRetriableException)
mock_request.side_effect = [self.mock_get_ep] + 3 * [socket.timeout] + [k8s_client.rest.ApiException(500, '')]
self.assertRaises(k8s_client.rest.ApiException, retry, self.a.call_api, 'GET', 'f', _retry=retry)
mock_request.side_effect = [self.mock_get_ep, socket.timeout, Mock(), self.mock_get_ep]
self.assertRaises(k8s_client.rest.ApiException, retry, self.a.call_api, 'GET', 'f', _retry=retry)
retry.deadline = 0.0001
mock_request.side_effect = [socket.timeout, socket.timeout, self.mock_get_ep]
self.assertRaises(K8sConnectionFailed, retry, self.a.call_api, 'GET', 'f', _retry=retry)
def test__refresh_api_servers_cache(self, mock_request):
mock_request.side_effect = k8s_client.rest.ApiException(403, '')
self.a.refresh_api_servers_cache()
class TestCoreV1Api(unittest.TestCase):
@patch('urllib3.PoolManager.request', Mock())
@patch.object(K8sConfig, '_server', '', create=True)
def setUp(self):
self.a = k8s_client.CoreV1Api()
self.a._api_client.set_read_timeout(10)
self.a._api_client.pool_manager.request = Mock(return_value=MockResponse())
def test_create_namespaced_service(self):
@@ -126,6 +159,7 @@ class TestCoreV1Api(unittest.TestCase):
class BaseTestKubernetes(unittest.TestCase):
@patch('urllib3.PoolManager.request', Mock())
@patch('socket.TCP_KEEPIDLE', 4, create=True)
@patch('socket.TCP_KEEPINTVL', 5, create=True)
@patch('socket.TCP_KEEPCNT', 6, create=True)
@@ -136,7 +170,8 @@ class BaseTestKubernetes(unittest.TestCase):
@patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map, create=True)
def setUp(self, config=None):
config = config or {}
config.update(ttl=30, scope='test', name='p-0', loop_wait=10, retry_timeout=10, labels={'f': 'b'})
config.update(ttl=30, scope='test', name='p-0', loop_wait=10,
retry_timeout=10, labels={'f': 'b'}, bypass_api_service=True)
self.k = Kubernetes(config)
self.assertRaises(AttributeError, self.k._pods._build_cache)
self.k._pods._is_ready = True
@@ -152,7 +187,7 @@ class TestKubernetesConfigMaps(BaseTestKubernetes):
def test__wait_caches(self):
self.k._pods._is_ready = False
with self.k._condition:
self.assertRaises(RetryFailedError, self.k._wait_caches)
self.assertRaises(RetryFailedError, self.k._wait_caches, time.time() + 10)
@patch('time.time', Mock(return_value=time.time() + 100))
def test_get_cluster(self):