From ee41262cc256fe2927e26844b0818c57e9edf47a Mon Sep 17 00:00:00 2001 From: Vegard Hagen Date: Sun, 2 Nov 2025 20:10:02 +0100 Subject: [PATCH] feat(bgp): enable bgp in cilium and disable l2 announcements Change IP of all LB Services to a different subnet Use externalTrafficPolicy on some LB Services Signed-off-by: Vegard Hagen --- k8s/apps/dev/whoami/svc.yaml | 6 +++++- k8s/apps/external/proxmox/svc.yaml | 1 + k8s/apps/external/truenas/svc.yaml | 1 + k8s/apps/media/plex/kustomization.yaml | 2 +- k8s/apps/utils/torrent/svc-torrent.yaml | 6 +++++- k8s/apps/utils/unpoller/config/up.conf | 2 +- k8s/infra/auth/authelia/cnpg-db.yaml | 2 +- k8s/infra/auth/lldap/cnpg-db.yaml | 2 +- k8s/infra/auth/lldap/svc.yaml | 6 +++++- k8s/infra/network/cilium/announce.yaml | 8 -------- .../cilium/bgp-advertise-lb-services.yaml | 15 +++++++++++++++ .../network/cilium/bgp-cluster-config.yaml | 17 +++++++++++++++++ k8s/infra/network/cilium/bgp-ip-pool.yaml | 10 ++++++++++ .../network/cilium/bgp-peer-ucg-max-config.yaml | 17 +++++++++++++++++ .../cilium/bgp-peer-ucg-max-password.yaml | 12 ++++++++++++ k8s/infra/network/cilium/ip-pool.yaml | 8 -------- k8s/infra/network/cilium/kustomization.yaml | 7 +++++-- k8s/infra/network/cilium/values.yaml | 10 ++++------ .../network/dns/adguard/config/AdGuardHome.yaml | 10 +++++----- k8s/infra/network/dns/adguard/svc.yaml | 8 +++++++- .../network/dns/unbound/config/a-records.conf | 5 +++++ k8s/infra/network/dns/unbound/deployment.yaml | 1 - k8s/infra/network/dns/unbound/svc.yaml | 6 +++++- k8s/infra/network/gateway/gw-external.yaml | 6 +++++- k8s/infra/network/gateway/gw-internal.yaml | 6 +++++- k8s/infra/vpn/coturn/svc.yaml | 6 +++++- .../vpn/netbird/management/deployment.yaml | 5 +++-- 27 files changed, 141 insertions(+), 44 deletions(-) delete mode 100644 k8s/infra/network/cilium/announce.yaml create mode 100644 k8s/infra/network/cilium/bgp-advertise-lb-services.yaml create mode 100644 k8s/infra/network/cilium/bgp-cluster-config.yaml create mode 100644 k8s/infra/network/cilium/bgp-ip-pool.yaml create mode 100644 k8s/infra/network/cilium/bgp-peer-ucg-max-config.yaml create mode 100644 k8s/infra/network/cilium/bgp-peer-ucg-max-password.yaml delete mode 100644 k8s/infra/network/cilium/ip-pool.yaml diff --git a/k8s/apps/dev/whoami/svc.yaml b/k8s/apps/dev/whoami/svc.yaml index 77b3a28..2ebaf4e 100644 --- a/k8s/apps/dev/whoami/svc.yaml +++ b/k8s/apps/dev/whoami/svc.yaml @@ -4,9 +4,13 @@ metadata: name: whoami namespace: whoami annotations: - io.cilium/lb-ipam-ips: 192.168.1.223 + io.cilium/lb-ipam-ips: 172.20.10.200 + labels: + bgp.cilium.io/advertise-service: default + bgp.cilium.io/ip-pool: default spec: type: LoadBalancer + externalTrafficPolicy: Local selector: app: whoami ports: diff --git a/k8s/apps/external/proxmox/svc.yaml b/k8s/apps/external/proxmox/svc.yaml index 688be4e..3cd53a7 100644 --- a/k8s/apps/external/proxmox/svc.yaml +++ b/k8s/apps/external/proxmox/svc.yaml @@ -8,4 +8,5 @@ spec: - name: https protocol: TCP port: 443 + appProtocol: kubernetes.io/h2c targetPort: 8006 diff --git a/k8s/apps/external/truenas/svc.yaml b/k8s/apps/external/truenas/svc.yaml index 92a9cc5..14852e9 100644 --- a/k8s/apps/external/truenas/svc.yaml +++ b/k8s/apps/external/truenas/svc.yaml @@ -8,3 +8,4 @@ spec: - name: https protocol: TCP port: 443 + appProtocol: kubernetes.io/h2c diff --git a/k8s/apps/media/plex/kustomization.yaml b/k8s/apps/media/plex/kustomization.yaml index ac5d339..26b4583 100644 --- a/k8s/apps/media/plex/kustomization.yaml +++ b/k8s/apps/media/plex/kustomization.yaml @@ -8,7 +8,7 @@ configMapGenerator: namespace: plex literals: - TZ="Europe/Oslo" - - PLEX_ADVERTISE_URL=https://plex.stonegarden.dev:443,http://192.168.1.228:32400 + - PLEX_ADVERTISE_URL=https://plex.stonegarden.dev:443 resources: - ns.yaml diff --git a/k8s/apps/utils/torrent/svc-torrent.yaml b/k8s/apps/utils/torrent/svc-torrent.yaml index cdbdf9e..d577298 100644 --- a/k8s/apps/utils/torrent/svc-torrent.yaml +++ b/k8s/apps/utils/torrent/svc-torrent.yaml @@ -4,9 +4,13 @@ metadata: name: torrent-torrent namespace: torrent annotations: - io.cilium/lb-ipam-ips: 192.168.1.225 + io.cilium/lb-ipam-ips: 172.20.10.250 + labels: + bgp.cilium.io/advertise-service: default + bgp.cilium.io/ip-pool: default spec: type: LoadBalancer + externalTrafficPolicy: Local selector: app: torrent ports: diff --git a/k8s/apps/utils/unpoller/config/up.conf b/k8s/apps/utils/unpoller/config/up.conf index 7235686..04a6777 100644 --- a/k8s/apps/utils/unpoller/config/up.conf +++ b/k8s/apps/utils/unpoller/config/up.conf @@ -26,7 +26,7 @@ dynamic = false [unifi.defaults] - url = "https://192.168.1.1" + url = "https://172.20.10.1" user = "unifipoller" pass = "file:///mnt/secrets/password" diff --git a/k8s/infra/auth/authelia/cnpg-db.yaml b/k8s/infra/auth/authelia/cnpg-db.yaml index 254c908..d9ab188 100644 --- a/k8s/infra/auth/authelia/cnpg-db.yaml +++ b/k8s/infra/auth/authelia/cnpg-db.yaml @@ -27,7 +27,7 @@ spec: # metadata: # name: authelia-postgres-db # annotations: - # io.cilium/lb-ipam-ips: 192.168.1.240 + # io.cilium/lb-ipam-ips: 172.20.10.241 # spec: # type: LoadBalancer storage: diff --git a/k8s/infra/auth/lldap/cnpg-db.yaml b/k8s/infra/auth/lldap/cnpg-db.yaml index 61d1588..b02dc11 100644 --- a/k8s/infra/auth/lldap/cnpg-db.yaml +++ b/k8s/infra/auth/lldap/cnpg-db.yaml @@ -27,7 +27,7 @@ spec: # metadata: # name: lldap-postgres-db # annotations: - # io.cilium/lb-ipam-ips: 192.168.1.241 + # io.cilium/lb-ipam-ips: 172.20.10.240 # spec: # type: LoadBalancer storage: diff --git a/k8s/infra/auth/lldap/svc.yaml b/k8s/infra/auth/lldap/svc.yaml index d79ff4a..2888553 100644 --- a/k8s/infra/auth/lldap/svc.yaml +++ b/k8s/infra/auth/lldap/svc.yaml @@ -4,9 +4,13 @@ metadata: name: lldap namespace: lldap annotations: - io.cilium/lb-ipam-ips: 192.168.1.242 + io.cilium/lb-ipam-ips: 172.20.10.120 + labels: + bgp.cilium.io/advertise-service: default + bgp.cilium.io/ip-pool: default spec: type: LoadBalancer + externalTrafficPolicy: Local # https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/ clusterIP: 10.96.0.15 selector: diff --git a/k8s/infra/network/cilium/announce.yaml b/k8s/infra/network/cilium/announce.yaml deleted file mode 100644 index 14f52b2..0000000 --- a/k8s/infra/network/cilium/announce.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: cilium.io/v2alpha1 -kind: CiliumL2AnnouncementPolicy -metadata: - name: default-l2-announcement-policy - namespace: kube-system -spec: - externalIPs: true - loadBalancerIPs: true diff --git a/k8s/infra/network/cilium/bgp-advertise-lb-services.yaml b/k8s/infra/network/cilium/bgp-advertise-lb-services.yaml new file mode 100644 index 0000000..c40f370 --- /dev/null +++ b/k8s/infra/network/cilium/bgp-advertise-lb-services.yaml @@ -0,0 +1,15 @@ +apiVersion: cilium.io/v2 +kind: CiliumBGPAdvertisement +metadata: + name: loadbalancer-services + labels: + bgp.cilium.io/advertise: loadbalancer-services +spec: + advertisements: + - advertisementType: Service + service: + addresses: + - LoadBalancerIP + selector: + matchLabels: + bgp.cilium.io/advertise-service: default diff --git a/k8s/infra/network/cilium/bgp-cluster-config.yaml b/k8s/infra/network/cilium/bgp-cluster-config.yaml new file mode 100644 index 0000000..112a9d8 --- /dev/null +++ b/k8s/infra/network/cilium/bgp-cluster-config.yaml @@ -0,0 +1,17 @@ +apiVersion: cilium.io/v2 +kind: CiliumBGPClusterConfig +metadata: + name: cilium-unifi +spec: + nodeSelector: + matchLabels: + node-role.kubernetes.io/control-plane: "" + bgpInstances: + - name: "65200" + localASN: 65200 + peers: + - name: "ucg-max-65100" + peerASN: 65100 + peerAddress: 172.20.10.1 + peerConfigRef: + name: ucg-max diff --git a/k8s/infra/network/cilium/bgp-ip-pool.yaml b/k8s/infra/network/cilium/bgp-ip-pool.yaml new file mode 100644 index 0000000..c05723d --- /dev/null +++ b/k8s/infra/network/cilium/bgp-ip-pool.yaml @@ -0,0 +1,10 @@ +apiVersion: cilium.io/v2 +kind: CiliumLoadBalancerIPPool +metadata: + name: default-bgp-ip-pool +spec: + blocks: + - cidr: 172.20.10.0/24 + serviceSelector: + matchLabels: + bgp.cilium.io/ip-pool: default diff --git a/k8s/infra/network/cilium/bgp-peer-ucg-max-config.yaml b/k8s/infra/network/cilium/bgp-peer-ucg-max-config.yaml new file mode 100644 index 0000000..8802fe3 --- /dev/null +++ b/k8s/infra/network/cilium/bgp-peer-ucg-max-config.yaml @@ -0,0 +1,17 @@ +apiVersion: cilium.io/v2 +kind: CiliumBGPPeerConfig +metadata: + name: ucg-max +spec: + authSecretRef: bgp-peer-ucg-max-password + timers: + connectRetryTimeSeconds: 10 + gracefulRestart: + enabled: true + restartTimeSeconds: 15 + families: + - afi: ipv4 + safi: unicast + advertisements: + matchLabels: + bgp.cilium.io/advertise: loadbalancer-services diff --git a/k8s/infra/network/cilium/bgp-peer-ucg-max-password.yaml b/k8s/infra/network/cilium/bgp-peer-ucg-max-password.yaml new file mode 100644 index 0000000..c837502 --- /dev/null +++ b/k8s/infra/network/cilium/bgp-peer-ucg-max-password.yaml @@ -0,0 +1,12 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: bgp-peer-ucg-max-password + namespace: cilium-secrets +spec: + encryptedData: + password: AgAfFRgvvxVt/KTOs7Ko646DwXNDTgXf+UuL21TjywwtVU7frAMfFx7U3+EfiBmXAbWsq2zAek0PGB8qhbaw9Q3Vnn5OgNed16dVOuYkV4QKqbarai/U2HXySts8pA6OR5ICE4WqJN/ayLsbN62jbMfjJMRlwIwXoRgW+6xLuspp81E/VUOQ0VBBKVlONzUWQWKgckpGjXLi7E1EEYQ6JVcKnDVUacM5adF0CjKkkQw/2MK4RnYhZXzmGTV1AnPe+ZK9HgjlmN5fqZ1cks4N1mfWPIGsRJX4CmWw4deqZ/nuB+3dhrDsdYCjxVNuOf9gTlyNwfBjETlYXuR9UCo/YuDzautmo6NlOy1nzl1O/nZbN4Rat+mbgFmn1sv5TwDUawcz4tNhKlrA+5iyVhmhNSYMqkNpkXRBJNRm+fK4Yspdo2yRl5jOX/dfkoWOgk9NijhfoDI2XGGpb7VIrD945i+9EBVe0UwMhnrhRWQZCTerx/1/kNajw77CUuD87KR+akeWVpxO1sG+tDwIceafeKbORYvl5S3TI24NkpS16Fdxl6PoFPVtiZlLqVxWRsnzK6Vb6cvWNGrlehHxURPrL9oB7+R+AM4W10/vqvX2NTNdIbzc2aT8qEq8gUlkPaRjcfeS9DKgnwM3OgF9UDyxjbRiHJ92dMOi3Gvu1F80crUZ0W6fdKhdsWQNqv84aS0JL0yaDLfbqNWf + template: + metadata: + name: bgp-peer-ucg-max-password + namespace: cilium-secrets diff --git a/k8s/infra/network/cilium/ip-pool.yaml b/k8s/infra/network/cilium/ip-pool.yaml deleted file mode 100644 index a71964b..0000000 --- a/k8s/infra/network/cilium/ip-pool.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: cilium.io/v2 -kind: CiliumLoadBalancerIPPool -metadata: - name: first-pool -spec: - blocks: - - start: 192.168.1.220 - stop: 192.168.1.255 diff --git a/k8s/infra/network/cilium/kustomization.yaml b/k8s/infra/network/cilium/kustomization.yaml index 1111033..cfe8749 100644 --- a/k8s/infra/network/cilium/kustomization.yaml +++ b/k8s/infra/network/cilium/kustomization.yaml @@ -2,8 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - announce.yaml - - ip-pool.yaml + - bgp-ip-pool.yaml + - bgp-advertise-lb-services.yaml + - bgp-peer-ucg-max-config.yaml + - bgp-peer-ucg-max-password.yaml + - bgp-cluster-config.yaml - dashboards/cilium.yaml - dashboards/cilium-operator.yaml diff --git a/k8s/infra/network/cilium/values.yaml b/k8s/infra/network/cilium/values.yaml index a60d468..78493dc 100644 --- a/k8s/infra/network/cilium/values.yaml +++ b/k8s/infra/network/cilium/values.yaml @@ -59,13 +59,11 @@ resources: #debug: # enabled: true -# Increase rate limit when doing L2 announcements -k8sClientRateLimit: - qps: 20 - burst: 100 - -l2announcements: +# https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v2/ +bgpControlPlane: enabled: true + secretsNamespace: + name: cilium-secrets externalIPs: enabled: true diff --git a/k8s/infra/network/dns/adguard/config/AdGuardHome.yaml b/k8s/infra/network/dns/adguard/config/AdGuardHome.yaml index d275cbe..ec4dfd3 100644 --- a/k8s/infra/network/dns/adguard/config/AdGuardHome.yaml +++ b/k8s/infra/network/dns/adguard/config/AdGuardHome.yaml @@ -144,12 +144,12 @@ filtering: parental_block_host: family-block.dns.adguard.com safebrowsing_block_host: standard-block.dns.adguard.com rewrites: - - domain: '*.stonegarden.dev' - answer: 192.168.1.220 - - domain: stonegarden.dev - answer: 192.168.1.220 - domain: coturn.stonegarden.dev - answer: 192.168.1.226 + answer: 172.20.10.140 + - domain: '*.stonegarden.dev' + answer: 172.20.10.100 + - domain: stonegarden.dev + answer: 172.20.10.100 safe_fs_patterns: - /opt/adguardhome/work/data/userfilters/* safebrowsing_cache_size: 1048576 diff --git a/k8s/infra/network/dns/adguard/svc.yaml b/k8s/infra/network/dns/adguard/svc.yaml index 36c053c..47a7473 100644 --- a/k8s/infra/network/dns/adguard/svc.yaml +++ b/k8s/infra/network/dns/adguard/svc.yaml @@ -4,9 +4,15 @@ metadata: name: adguard namespace: dns annotations: - io.cilium/lb-ipam-ips: 192.168.1.253 + io.cilium/lb-ipam-ips: 172.20.10.153 + labels: + bgp.cilium.io/advertise-service: default + bgp.cilium.io/ip-pool: default spec: type: LoadBalancer + externalTrafficPolicy: Local + # https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/ + clusterIP: 10.96.0.12 ports: - name: http protocol: TCP diff --git a/k8s/infra/network/dns/unbound/config/a-records.conf b/k8s/infra/network/dns/unbound/config/a-records.conf index 319d934..2f99887 100644 --- a/k8s/infra/network/dns/unbound/config/a-records.conf +++ b/k8s/infra/network/dns/unbound/config/a-records.conf @@ -1,5 +1,10 @@ # A Record #local-data: "somecomputer.local. A 192.168.1.1" + local-data: "coturn.stonegarden.dev A 172.20.10.140" + local-data: "*.stonegarden.dev A 172.20.10.100" + local-data: "stonegarden.dev A 172.20.10.100" # PTR Record #local-data-ptr: "192.168.1.1 somecomputer.local." + local-data-ptr: "172.20.10.100 stonegarden.dev." + local-data-ptr: "172.20.10.140 coturn.stonegarden.dev." diff --git a/k8s/infra/network/dns/unbound/deployment.yaml b/k8s/infra/network/dns/unbound/deployment.yaml index 346795f..100d601 100644 --- a/k8s/infra/network/dns/unbound/deployment.yaml +++ b/k8s/infra/network/dns/unbound/deployment.yaml @@ -33,7 +33,6 @@ spec: cpu: 10m memory: 64Mi limits: - cpu: 500m memory: 128Mi startupProbe: exec: diff --git a/k8s/infra/network/dns/unbound/svc.yaml b/k8s/infra/network/dns/unbound/svc.yaml index 1c3f0f3..813182e 100644 --- a/k8s/infra/network/dns/unbound/svc.yaml +++ b/k8s/infra/network/dns/unbound/svc.yaml @@ -4,9 +4,13 @@ metadata: name: unbound namespace: dns annotations: - io.cilium/lb-ipam-ips: 192.168.1.252 + io.cilium/lb-ipam-ips: 172.20.10.150 + labels: + bgp.cilium.io/advertise-service: default + bgp.cilium.io/ip-pool: default spec: type: LoadBalancer + externalTrafficPolicy: Local # https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/ clusterIP: 10.96.0.11 ports: diff --git a/k8s/infra/network/gateway/gw-external.yaml b/k8s/infra/network/gateway/gw-external.yaml index cfe65df..12779fd 100644 --- a/k8s/infra/network/gateway/gw-external.yaml +++ b/k8s/infra/network/gateway/gw-external.yaml @@ -5,9 +5,13 @@ metadata: namespace: gateway spec: gatewayClassName: cilium + infrastructure: + labels: + bgp.cilium.io/advertise-service: default + bgp.cilium.io/ip-pool: default addresses: - type: IPAddress - value: 192.168.1.222 + value: 172.20.10.110 listeners: - protocol: HTTPS port: 443 diff --git a/k8s/infra/network/gateway/gw-internal.yaml b/k8s/infra/network/gateway/gw-internal.yaml index 342d356..f97a316 100644 --- a/k8s/infra/network/gateway/gw-internal.yaml +++ b/k8s/infra/network/gateway/gw-internal.yaml @@ -5,9 +5,13 @@ metadata: namespace: gateway spec: gatewayClassName: cilium + infrastructure: + labels: + bgp.cilium.io/advertise-service: default + bgp.cilium.io/ip-pool: default addresses: - type: IPAddress - value: 192.168.1.220 + value: 172.20.10.100 listeners: - protocol: TLS port: 443 diff --git a/k8s/infra/vpn/coturn/svc.yaml b/k8s/infra/vpn/coturn/svc.yaml index 5e4f6d9..a2b043b 100644 --- a/k8s/infra/vpn/coturn/svc.yaml +++ b/k8s/infra/vpn/coturn/svc.yaml @@ -4,9 +4,13 @@ metadata: name: coturn namespace: coturn annotations: - io.cilium/lb-ipam-ips: 192.168.1.226 + io.cilium/lb-ipam-ips: 172.20.10.140 + labels: + bgp.cilium.io/advertise-service: default + bgp.cilium.io/ip-pool: default spec: type: LoadBalancer + externalTrafficPolicy: Local selector: app.kubernetes.io/name: coturn ports: diff --git a/k8s/infra/vpn/netbird/management/deployment.yaml b/k8s/infra/vpn/netbird/management/deployment.yaml index 0215cd0..4be1d92 100644 --- a/k8s/infra/vpn/netbird/management/deployment.yaml +++ b/k8s/infra/vpn/netbird/management/deployment.yaml @@ -19,9 +19,10 @@ spec: nodeSelector: topology.kubernetes.io/zone: abel dnsConfig: - # Internal AdGuard Home DNS nameservers: - - 192.168.1.253 + - 172.20.10.153 # AdGuard Home + - 10.96.0.12 # AdGuard Home + - 10.96.0.11 # Unbound dnsPolicy: None securityContext: seccompProfile: