From 0e19e3e98e89f1aba0584e55cc53489d98553b30 Mon Sep 17 00:00:00 2001 From: Waynerv Date: Tue, 25 Jul 2023 16:29:04 +0800 Subject: [PATCH] Make pod role label configurable (#2659) Close #2495 --- docs/ENVIRONMENT.rst | 6 ++++- docs/kubernetes.rst | 49 +++++++++++++++++++++++++++++++++++++ docs/yaml_configuration.rst | 6 ++++- patroni/config.py | 3 ++- patroni/dcs/kubernetes.py | 23 ++++++++++++++--- patroni/validator.py | 4 +++ tests/test_kubernetes.py | 22 +++++++++++++++++ 7 files changed, 106 insertions(+), 7 deletions(-) diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index 6fbfe4e0..00595cf7 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -112,7 +112,11 @@ Kubernetes - **PATRONI\_KUBERNETES\_NAMESPACE**: (optional) Kubernetes namespace where the Patroni pod is running. Default value is `default`. - **PATRONI\_KUBERNETES\_LABELS**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates. - **PATRONI\_KUBERNETES\_SCOPE\_LABEL**: (optional) name of the label containing cluster name. Default value is `cluster-name`. -- **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing Postgres role (`master` or `replica`). Patroni will set this label on the pod it is running in. Default value is `role`. +- **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing role (master or replica or other custom value). Patroni will set this label on the pod it runs in. Default value is ``role``. +- **PATRONI\_KUBERNETES\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is `master`. Default value is `master`. +- **PATRONI\_KUBERNETES\_FOLLOWER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is `replica`. Default value is `replica`. +- **PATRONI\_KUBERNETES\_STANDBY\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is ``standby-leader``. Default value is ``standby-leader``. +- **PATRONI\_KUBERNETES\_TMP\_ROLE\_LABEL**: (optional) name of the temporary label containing role (master or replica). Value of this label will always use the default of corresponding role. Set only when necessary. - **PATRONI\_KUBERNETES\_USE\_ENDPOINTS**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state. - **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted. - **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='[{"name": "postgresql", "port": 5432}]'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set. diff --git a/docs/kubernetes.rst b/docs/kubernetes.rst index 699321fa..ca7313bf 100644 --- a/docs/kubernetes.rst +++ b/docs/kubernetes.rst @@ -32,6 +32,55 @@ Configuration Patroni Kubernetes :ref:`settings ` and :ref:`environment variables ` are described in the general chapters of the documentation. +Customize role label +^^^^^^^^^^^^^^^^^^^^ +By default, Patroni will set corresponding labels on the pod it runs in based on node's role, such as ``role=master``. +The key and value of label can be customized by `kubernetes.role_label`, `kubernetes.leader_label_value`, `kubernetes.follower_label_value` and `kubernetes.standby_leader_label_value`. + +Note that if you migrate from default role labels to custom ones, you can reduce downtime by following migration steps: + +1. Add a temporary label using original role value for the pod with `kubernetes.tmp_role_label` (like ``tmp_role``). Once pods are restarted they will get following labels set by Patroni: + + .. code:: YAML + + labels: + cluster-name: foo + role: master + tmp_role: master + +2. After all pods have been updated, modify the service selector to select the temporary label. + + .. code:: YAML + + selector: + cluster-name: foo + tmp_role: master + +3. Add your custom role label (e.g., set `kubernetes.leader_label_value=primary`). Once pods are restarted they will get following new labels set by Patroni: + + .. code:: YAML + + labels: + cluster-name: foo + role: primary + tmp_role: master + +4. After all pods have been updated again, modify the service selector to use new role value. + + .. code:: YAML + + selector: + cluster-name: foo + role: primary + +5. Finally, remove the temporary label from your configuration and update all pods. + + .. code:: YAML + + labels: + cluster-name: foo + role: primary + Examples -------- diff --git a/docs/yaml_configuration.rst b/docs/yaml_configuration.rst index fbb2de68..ab5b5587 100644 --- a/docs/yaml_configuration.rst +++ b/docs/yaml_configuration.rst @@ -155,7 +155,11 @@ Kubernetes - **namespace**: (optional) Kubernetes namespace where Patroni pod is running. Default value is `default`. - **labels**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates. - **scope\_label**: (optional) name of the label containing cluster name. Default value is `cluster-name`. -- **role\_label**: (optional) name of the label containing role (master or replica). Patroni will set this label on the pod it runs in. Default value is ``role``. +- **role\_label**: (optional) name of the label containing role (master or replica or other custom value). Patroni will set this label on the pod it runs in. Default value is ``role``. +- **leader\_label\_value**: (optional) value of the pod label when Postgres role is ``master``. Default value is ``master``. +- **follower\_label\_value**: (optional) value of the pod label when Postgres role is ``replica``. Default value is ``replica``. +- **standby\_leader\_label\_value**: (optional) value of the pod label when Postgres role is ``standby-leader``. Default value is ``standby-leader``. +- **tmp_\role\_label**: (optional) name of the temporary label containing role (master or replica). Value of this label will always use the default of corresponding role. Set only when necessary. - **use\_endpoints**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state. - **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted. - **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: [{"name": "postgresql", "port": 5432}]`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set. diff --git a/patroni/config.py b/patroni/config.py index 365abc24..66ffd891 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -475,7 +475,8 @@ class Config(object): 'REGISTER_SERVICE', 'SERVICE_CHECK_INTERVAL', 'SERVICE_CHECK_TLS_SERVER_NAME', 'SERVICE_TAGS', 'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP', 'PORTS', 'LABELS', 'BYPASS_API_SERVICE', 'RETRIABLE_HTTP_CODES', 'KEY_PASSWORD', - 'USE_SSL', 'SET_ACLS', 'GROUP', 'DATABASE') and name: + 'USE_SSL', 'SET_ACLS', 'GROUP', 'DATABASE', 'LEADER_LABEL_VALUE', 'FOLLOWER_LABEL_VALUE', + 'STANDBY_LEADER_LABEL_VALUE', 'TMP_ROLE_LABEL') and name: value = os.environ.pop(param) if name == 'CITUS': if suffix == 'GROUP': diff --git a/patroni/dcs/kubernetes.py b/patroni/dcs/kubernetes.py index f770ac64..bda4bdc8 100644 --- a/patroni/dcs/kubernetes.py +++ b/patroni/dcs/kubernetes.py @@ -752,6 +752,10 @@ class Kubernetes(AbstractDCS): self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items()) self._namespace = config.get('namespace') or 'default' self._role_label = config.get('role_label', 'role') + self._leader_label_value = config.get('leader_label_value', 'master') + self._follower_label_value = config.get('follower_label_value', 'replica') + self._standby_leader_label_value = config.get('standby_leader_label_value', 'standby-leader') + self._tmp_role_label = config.get('tmp_role_label') self._ca_certs = os.environ.get('PATRONI_KUBERNETES_CACERT', config.get('cacert')) or SERVICE_CERT_FILENAME super(Kubernetes, self).__init__({**config, 'namespace': ''}) if self._citus_group: @@ -1263,19 +1267,30 @@ class Kubernetes(AbstractDCS): def touch_member(self, data: Dict[str, Any]) -> bool: cluster = self.cluster if cluster and cluster.leader and cluster.leader.name == self._name: - role = 'master' + role = self._leader_label_value + tmp_role = 'master' elif data['state'] == 'running' and data['role'] not in ('master', 'primary'): - role = data['role'] + role = { + 'replica': self._follower_label_value, + 'standby-leader': self._standby_leader_label_value, + }.get(data['role'], data['role']) + tmp_role = data['role'] else: role = None + tmp_role = None + + role_labels = {self._role_label: role} + if self._tmp_role_label: + role_labels[self._tmp_role_label] = tmp_role member = cluster and cluster.get_member(self._name, fallback_to_leader=False) pod_labels = member and member.data.pop('pod_labels', None) ret = member and pod_labels is not None\ - and pod_labels.get(self._role_label) == role and deep_compare(data, member.data) + and all(pod_labels.get(k) == v for k, v in role_labels.items())\ + and deep_compare(data, member.data) if not ret: - metadata = {'namespace': self._namespace, 'name': self._name, 'labels': {self._role_label: role}, + metadata = {'namespace': self._namespace, 'name': self._name, 'labels': role_labels, 'annotations': {'status': json.dumps(data, separators=(',', ':'))}} body = k8s_client.V1Pod(metadata=k8s_client.V1ObjectMeta(**metadata)) ret = self._api.patch_namespaced_pod(self._name, self._namespace, body) diff --git a/patroni/validator.py b/patroni/validator.py index e9afcc13..b68f9654 100644 --- a/patroni/validator.py +++ b/patroni/validator.py @@ -995,6 +995,10 @@ schema = Schema({ Optional("namespace"): str, Optional("scope_label"): str, Optional("role_label"): str, + Optional("leader_label_value"): str, + Optional("follower_label_value"): str, + Optional("standby_leader_label_value"): str, + Optional("tmp_role_label"): str, Optional("use_endpoints"): bool, Optional("pod_ip"): Or(is_ipv4_address, is_ipv6_address), Optional("ports"): [{"name": str, "port": int}], diff --git a/tests/test_kubernetes.py b/tests/test_kubernetes.py index cfed1559..f79539a7 100644 --- a/tests/test_kubernetes.py +++ b/tests/test_kubernetes.py @@ -298,6 +298,28 @@ class TestKubernetesConfigMaps(BaseTestKubernetes): self.k.touch_member({'state': 'running', 'role': 'replica'}) self.k.touch_member({'state': 'stopped', 'role': 'primary'}) + self.k._role_label = 'isMaster' + self.k._leader_label_value = 'true' + self.k._follower_label_value = 'false' + self.k._standby_leader_label_value = 'false' + self.k._tmp_role_label = 'tmp_role' + + self.k.touch_member({'state': 'running', 'role': 'replica'}) + mock_patch_namespaced_pod.assert_called() + self.assertEqual(mock_patch_namespaced_pod.call_args.args[2].metadata.labels['isMaster'], 'false') + self.assertEqual(mock_patch_namespaced_pod.call_args.args[2].metadata.labels['tmp_role'], 'replica') + + self.k.touch_member({'state': 'running', 'role': 'standby-leader'}) + mock_patch_namespaced_pod.assert_called() + self.assertEqual(mock_patch_namespaced_pod.call_args.args[2].metadata.labels['isMaster'], 'false') + self.assertEqual(mock_patch_namespaced_pod.call_args.args[2].metadata.labels['tmp_role'], 'standby-leader') + + self.k._name = 'p-0' + self.k.touch_member({'role': 'primary'}) + mock_patch_namespaced_pod.assert_called() + self.assertEqual(mock_patch_namespaced_pod.call_args.args[2].metadata.labels['isMaster'], 'true') + self.assertEqual(mock_patch_namespaced_pod.call_args.args[2].metadata.labels['tmp_role'], 'master') + def test_initialize(self): self.k.initialize()