From c9ffd698c8fe9ec504b653babdbe7c3c484ed6c4 Mon Sep 17 00:00:00 2001
From: Vegard Hagen <vegard@stonegarden.dev>
Date: Sun, 5 Jan 2025 13:10:26 +0100
Subject: [PATCH] fix(netbird-agent): allow read root fs and add sysctl

The agent seems to assume it's running with full root privileges,
something we don't want. Accommodate by changing manually.
---
 k8s/infra/vpn/netbird/agent/daemon-set.yaml     | 17 ++++++++++++-----
 k8s/infra/vpn/netbird/agent/kustomization.yaml  |  2 +-
 .../machine-config/control-plane.yaml.tftpl     |  4 ++++
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/k8s/infra/vpn/netbird/agent/daemon-set.yaml b/k8s/infra/vpn/netbird/agent/daemon-set.yaml
index 541b9be..0680eff 100644
--- a/k8s/infra/vpn/netbird/agent/daemon-set.yaml
+++ b/k8s/infra/vpn/netbird/agent/daemon-set.yaml
@@ -15,23 +15,30 @@ spec:
       securityContext:
         seccompProfile:
           type: RuntimeDefault
+        sysctls:
+          # Considered unsafe, need to enable with kubelet argument.
+          # see https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls
+          - name: net.ipv4.conf.all.src_valid_mark
+            value: "1"
       containers:
         - name: agent
           image: docker.io/netbirdio/netbird:0.35.2 # renovate: docker=docker.io/netbirdio/netbird
           imagePullPolicy: IfNotPresent
           securityContext:
             allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
+            readOnlyRootFilesystem: false
+            runAsNonRoot: false
             capabilities:
-              add: [ NET_ADMIN, PERFMON, BPF ]
+              drop: [ ALL ]
+              add: [ NET_ADMIN, NET_RAW, PERFMON, BPF ]
           envFrom:
             - configMapRef:
                 name: agent-config
             - secretRef:
                 name: agent-setup-key
           volumeMounts:
-            - name: config
-              mountPath: /config
+            - name: netbird
+              mountPath: /var/lib/netbird
           resources:
             requests:
               memory: 192Mi
@@ -40,5 +47,5 @@ spec:
               memory: 256Mi
               cpu: 4000m
       volumes:
-        - name: config
+        - name: netbird
           emptyDir: { }
diff --git a/k8s/infra/vpn/netbird/agent/kustomization.yaml b/k8s/infra/vpn/netbird/agent/kustomization.yaml
index dcbd668..9668c6c 100644
--- a/k8s/infra/vpn/netbird/agent/kustomization.yaml
+++ b/k8s/infra/vpn/netbird/agent/kustomization.yaml
@@ -9,7 +9,7 @@ configMapGenerator:
       - NB_LOG_LEVEL="warn"
       - NB_MANAGEMENT_URL="https://netbird.stonegarden.dev"
       - NB_ADMIN_URL="https://netbird.stonegarden.dev"
-      - NB_CONFIG="/config/config.json"
+      - NB_CONFIG="/var/lib/netbird/config.json"
 
 resources:
   - agent-setup-key.yaml
diff --git a/tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl b/tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl
index 539529e..7e01023 100644
--- a/tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl
+++ b/tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl
@@ -1,4 +1,8 @@
 machine:
+  kubelet:
+    extraArgs:
+      # Needed for Netbird agent https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls
+      allowed-unsafe-sysctls: net.ipv4.conf.all.src_valid_mark
   network:
     hostname: ${hostname}
   nodeLabels: