Add health check on user defined tags (#1964)

Close #1958
This commit is contained in:
Arman Jafari Tehrani
2021-06-23 08:30:10 +02:00
committed by GitHub
parent 00bf546848
commit e48df9987d
4 changed files with 127 additions and 1 deletions
+9
View File
@@ -373,3 +373,12 @@ Tags
- **noloadbalance**: ``true`` or ``false``. If set to ``true`` the node will return HTTP Status Code 503 for the ``GET /replica`` REST API health-check and therefore will be excluded from the load-balancing. Defaults to ``false``.
- **replicatefrom**: The IP address/hostname of another replica. Used to support cascading replication.
- **nosync**: ``true`` or ``false``. If set to ``true`` the node will never be selected as a synchronous replica.
In addition to these predefined tags, you can also add your own ones:
- **key1**: ``true``
- **key2**: ``false``
- **key3**: ``1.4``
- **key4**: ``"RandomString"``
Tags are visible in the :ref:`REST API <rest_api>` and ``patronictl list`` You can also check for an instance health using these tags. If the tag isn't defined for an instance, or if the respective value doesn't match the querying value, it will return HTTP Status Code 503.
+11
View File
@@ -29,6 +29,17 @@ For all health check ``GET`` requests Patroni returns a JSON document with the s
- ``GET /replica?lag=10MB``
- ``GET /replica?lag=1GB``
- ``GET /replica?tag_key1=value1&tag_key2=value2``: replica check endpoint. In addition, It will also check for user defined tags ``key1`` and ``key2`` and their respective values in the **tags** section of the yaml configuration management. If the tag isn't defined for an instance, or if the value in the yaml configuration doesn't match the querying value, it will return HTTP Status Code 503.
In the following requests, since we are checking for the leader or standby-leader status, Patroni doesn't apply any of the user defined tags and they will be ignored.
- ``GET /?tag_key1=value1&tag_key2=value2``
- ``GET /master?tag_key1=value1&tag_key2=value2``
- ``GET /leader?tag_key1=value1&tag_key2=value2``
- ``GET /primary?tag_key1=value1&tag_key2=value2``
- ``GET /read-write?tag_key1=value1&tag_key2=value2``
- ``GET /standby_leader?tag_key1=value1&tag_key2=value2``
- ``GET /standby-leader?tag_key1=value1&tag_key2=value2``
- ``GET /read-only``: like the above endpoint, but also includes the primary.
- ``GET /synchronous`` or ``GET /sync``: returns HTTP status code **200** only when the Patroni node is running as a synchronous standby.
+23
View File
@@ -124,12 +124,16 @@ class RestApiHandler(BaseHTTPRequestHandler):
status_code = 503
ignore_tags = False
if 'standby_leader' in path or 'standby-leader' in path:
status_code = standby_leader_status_code
ignore_tags = True
elif 'leader' in path:
status_code = leader_status_code
ignore_tags = True
elif 'master' in path or 'primary' in path or 'read-write' in path:
status_code = primary_status_code
ignore_tags = True
elif 'replica' in path:
status_code = replica_status_code
elif 'read-only' in path:
@@ -144,6 +148,25 @@ class RestApiHandler(BaseHTTPRequestHandler):
elif path in ('/async', '/asynchronous') and not is_synchronous:
status_code = replica_status_code
# check for user defined tags in query params
if not ignore_tags and status_code == 200:
qs_tag_prefix = "tag_"
for qs_key, qs_value in self.path_query.items():
if not qs_key.startswith(qs_tag_prefix):
continue
qs_key = qs_key[len(qs_tag_prefix):]
qs_value = qs_value[0]
instance_tag_value = patroni.tags.get(qs_key)
# tag not registered for instance
if instance_tag_value is None:
status_code = 503
break
if not isinstance(instance_tag_value, six.string_types):
instance_tag_value = str(instance_tag_value).lower()
if instance_tag_value != qs_value:
status_code = 503
break
if write_status_code_only: # when haproxy sends OPTIONS request it reads only status code and nothing more
self._write_status_code_only(status_code)
else:
+84 -1
View File
@@ -118,7 +118,7 @@ class MockPatroni(object):
postgresql = ha.state_handler
dcs = Mock()
logger = MockLogger()
tags = {}
tags = {"key1": True, "key2": False, "key3": 1, "key4": 1.4, "key5": "RandomTag"}
version = '0.00'
noloadbalance = PropertyMock(return_value=False)
scheduled_restart = {'schedule': future_restart_time,
@@ -197,6 +197,89 @@ class TestRestApiHandler(unittest.TestCase):
with patch.object(MockHa, 'is_standby_cluster', Mock(return_value=True)):
MockRestApiServer(RestApiHandler, 'GET /standby_leader')
# test tags
#
MockRestApiServer(RestApiHandler, 'GET /master?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /master?lag=1M&'
'tag_key1=true&tag_key2=False&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /master?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /master?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2')
#
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
MockRestApiServer(RestApiHandler, 'GET /master?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /master?lag=1M&'
'tag_key1=true&tag_key2=False&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /master?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /master?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2')
#
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'standby_leader'})):
MockRestApiServer(RestApiHandler, 'GET /standby_leader?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /standby_leader?lag=1M&'
'tag_key1=true&tag_key2=False&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /standby_leader?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /standby_leader?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2')
#
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
'tag_key1=true&tag_key2=False&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2')
#
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
'tag_key1=true&tag_key2=False&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2')
#
MockRestApiServer(RestApiHandler, 'GET /read-write?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /read-write?lag=1M&'
'tag_key1=true&tag_key2=False&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /read-write?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag')
MockRestApiServer(RestApiHandler, 'GET /read-write?lag=1M&'
'tag_key1=true&tag_key2=false&'
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2')
def test_do_OPTIONS(self):
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'OPTIONS / HTTP/1.0'))