From 559b4c3d24ab584a38fc01f4f96111c6c0e36475 Mon Sep 17 00:00:00 2001
From: Vegard Hagen <vegard@stonegarden.dev>
Date: Fri, 25 Jul 2025 22:08:31 +0200
Subject: [PATCH] feat(cilium): add grafana dashboards

Signed-off-by: Vegard Hagen <vegard@stonegarden.dev>
---
 .../monitoring/prometheus-stack/values.yaml   | 27 +++++++++
 k8s/infra/network/application-set.yaml        |  3 +
 k8s/infra/network/cilium/values.yaml          | 59 +++++++++++++++++++
 tofu/kubernetes/talos/config.tofu             |  3 +-
 4 files changed, 91 insertions(+), 1 deletion(-)

diff --git a/k8s/infra/monitoring/prometheus-stack/values.yaml b/k8s/infra/monitoring/prometheus-stack/values.yaml
index 03db838..11ac99a 100644
--- a/k8s/infra/monitoring/prometheus-stack/values.yaml
+++ b/k8s/infra/monitoring/prometheus-stack/values.yaml
@@ -1,5 +1,12 @@
+# https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml
 prometheus:
   prometheusSpec:
+    podMonitorNamespaceSelector: { }
+    podMonitorSelectorNilUsesHelmValues: false
+    podMonitorSelector: { }
+    serviceMonitorNamespaceSelector: { }
+    serviceMonitorSelectorNilUsesHelmValues: false
+    serviceMonitorSelector: { }
     nodeSelector:
       topology.kubernetes.io/zone: abel
     storageSpec:
@@ -18,3 +25,23 @@ prometheus:
 
 nodeExporter:
   enabled: false
+
+grafana:
+  enabled: true
+  defaultDashboardsEnabled: true
+  defaultDashboardsTimezone: Europe/Oslo
+  # https://github.com/grafana/helm-charts/issues/527#issuecomment-982319638
+  sidecar:
+    dashboards:
+      enabled: true
+      searchNamespace: ALL
+      folder: /tmp/dashboards
+      folderAnnotation: grafana_folder
+      annotations:
+        grafana_folder: "Kubernetes"
+      provider:
+        allowUiUpdates: false
+        foldersFromFilesStructure: true
+
+    datasources:
+      enabled: true
diff --git a/k8s/infra/network/application-set.yaml b/k8s/infra/network/application-set.yaml
index b601b4b..7aa7060 100644
--- a/k8s/infra/network/application-set.yaml
+++ b/k8s/infra/network/application-set.yaml
@@ -32,3 +32,6 @@ spec:
         automated:
           selfHeal: true
           prune: true
+        syncOptions:
+          - ApplyOutOfSyncOnly=true
+          - ServerSideApply=true
diff --git a/k8s/infra/network/cilium/values.yaml b/k8s/infra/network/cilium/values.yaml
index d852e17..819e94d 100644
--- a/k8s/infra/network/cilium/values.yaml
+++ b/k8s/infra/network/cilium/values.yaml
@@ -30,6 +30,16 @@ ipam:
 
 operator:
   rollOutPods: true
+  prometheus:
+    metricsService: true
+    enabled: true
+    port: 9963
+    serviceMonitor:
+      enabled: true
+  dashboards:
+    enabled: true
+    annotations:
+      grafana_folder: "Cilium"
   resources:
     limits:
       cpu: 500m
@@ -72,7 +82,13 @@ gatewayAPI:
   enabled: true
 #  enableAlpn: true
 #  enableAppProtocol: true
+
 envoy:
+  prometheus:
+    enabled: true
+    port: "9964"
+    serviceMonitor:
+      enabled: true
   securityContext:
     capabilities:
       keepCapNetBindService: true
@@ -80,15 +96,45 @@ envoy:
 
 hubble:
   enabled: true
+  metrics:
+    enabled:
+      - dns
+      - drop
+      - tcp
+      - flow
+      - port-distribution
+      - icmp
+      - "httpV2:exemplars=true;labelsContext=source_ip,source_namespace,source_workload,destination_ip,destination_namespace,destination_workload,traffic_direction;sourceContext=workload-name|reserved-identity;destinationContext=workload-name|reserved-identity"
+    enableOpenMetrics: true
+    port: 9965
+    serviceMonitor:
+      enabled: true
+    dashboards:
+      enabled: true
+      annotations:
+        grafana_folder: "Hubble"
   relay:
     enabled: true
     rollOutPods: true
+    prometheus:
+      enabled: true
+      port: 9966
+      serviceMonitor:
+        enabled: true
   ui:
     enabled: true
     rollOutPods: true
 
 ingressController: { enabled: false }
 
+clustermesh:
+  apiserver:
+    metrics:
+      enabled: true
+      port: 9962
+      serviceMonitor:
+        enabled: true
+
 # mTLS
 authentication:
   enabled: false
@@ -99,3 +145,16 @@ authentication:
         server:
           dataStorage:
             storageClass: cilium-spire-sc
+
+prometheus:
+  metricsService: true
+  enabled: true
+  port: 9962
+  serviceMonitor:
+    enabled: true
+    trustCRDsExist: true
+
+dashboards:
+  enabled: true
+  annotations:
+    grafana_folder: "Cilium"
diff --git a/tofu/kubernetes/talos/config.tofu b/tofu/kubernetes/talos/config.tofu
index 584ec03..e587677 100644
--- a/tofu/kubernetes/talos/config.tofu
+++ b/tofu/kubernetes/talos/config.tofu
@@ -3,7 +3,8 @@ locals {
   kubernetes_endpoint         = coalesce(var.cluster.vip, local.first_control_plane_node_ip)
   extra_manifests = concat(var.cluster.extra_manifests, [
     "https://github.com/kubernetes-sigs/gateway-api/releases/download/${var.cluster.gateway_api_version}/standard-install.yaml",
-    "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/${var.cluster.gateway_api_version}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml"
+    "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/${var.cluster.gateway_api_version}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml",
+    "https://raw.githubusercontent.com/prometheus-community/helm-charts/refs/heads/main/charts/kube-prometheus-stack/charts/crds/crds/crd-servicemonitors.yaml"
   ])
 }