feat(bgp): enable bgp in cilium and disable l2 announcements

Change IP of all LB Services to a different subnet

Use externalTrafficPolicy on some LB Services

Signed-off-by: Vegard Hagen <[email protected]>
This commit is contained in:
Vegard Hagen
2025-11-16 18:59:51 +01:00
parent 3a77de4332
commit ee41262cc2
27 changed files with 141 additions and 44 deletions
+5 -1
View File
@@ -4,9 +4,13 @@ metadata:
name: whoami
namespace: whoami
annotations:
io.cilium/lb-ipam-ips: 192.168.1.223
io.cilium/lb-ipam-ips: 172.20.10.200
labels:
bgp.cilium.io/advertise-service: default
bgp.cilium.io/ip-pool: default
spec:
type: LoadBalancer
externalTrafficPolicy: Local
selector:
app: whoami
ports:
+1
View File
@@ -8,4 +8,5 @@ spec:
- name: https
protocol: TCP
port: 443
appProtocol: kubernetes.io/h2c
targetPort: 8006
+1
View File
@@ -8,3 +8,4 @@ spec:
- name: https
protocol: TCP
port: 443
appProtocol: kubernetes.io/h2c
+1 -1
View File
@@ -8,7 +8,7 @@ configMapGenerator:
namespace: plex
literals:
- TZ="Europe/Oslo"
- PLEX_ADVERTISE_URL=https://plex.stonegarden.dev:443,http://192.168.1.228:32400
- PLEX_ADVERTISE_URL=https://plex.stonegarden.dev:443
resources:
- ns.yaml
+5 -1
View File
@@ -4,9 +4,13 @@ metadata:
name: torrent-torrent
namespace: torrent
annotations:
io.cilium/lb-ipam-ips: 192.168.1.225
io.cilium/lb-ipam-ips: 172.20.10.250
labels:
bgp.cilium.io/advertise-service: default
bgp.cilium.io/ip-pool: default
spec:
type: LoadBalancer
externalTrafficPolicy: Local
selector:
app: torrent
ports:
+1 -1
View File
@@ -26,7 +26,7 @@
dynamic = false
[unifi.defaults]
url = "https://192.168.1.1"
url = "https://172.20.10.1"
user = "unifipoller"
pass = "file:///mnt/secrets/password"
+1 -1
View File
@@ -27,7 +27,7 @@ spec:
# metadata:
# name: authelia-postgres-db
# annotations:
# io.cilium/lb-ipam-ips: 192.168.1.240
# io.cilium/lb-ipam-ips: 172.20.10.241
# spec:
# type: LoadBalancer
storage:
+1 -1
View File
@@ -27,7 +27,7 @@ spec:
# metadata:
# name: lldap-postgres-db
# annotations:
# io.cilium/lb-ipam-ips: 192.168.1.241
# io.cilium/lb-ipam-ips: 172.20.10.240
# spec:
# type: LoadBalancer
storage:
+5 -1
View File
@@ -4,9 +4,13 @@ metadata:
name: lldap
namespace: lldap
annotations:
io.cilium/lb-ipam-ips: 192.168.1.242
io.cilium/lb-ipam-ips: 172.20.10.120
labels:
bgp.cilium.io/advertise-service: default
bgp.cilium.io/ip-pool: default
spec:
type: LoadBalancer
externalTrafficPolicy: Local
# https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/
clusterIP: 10.96.0.15
selector:
-8
View File
@@ -1,8 +0,0 @@
apiVersion: cilium.io/v2alpha1
kind: CiliumL2AnnouncementPolicy
metadata:
name: default-l2-announcement-policy
namespace: kube-system
spec:
externalIPs: true
loadBalancerIPs: true
@@ -0,0 +1,15 @@
apiVersion: cilium.io/v2
kind: CiliumBGPAdvertisement
metadata:
name: loadbalancer-services
labels:
bgp.cilium.io/advertise: loadbalancer-services
spec:
advertisements:
- advertisementType: Service
service:
addresses:
- LoadBalancerIP
selector:
matchLabels:
bgp.cilium.io/advertise-service: default
@@ -0,0 +1,17 @@
apiVersion: cilium.io/v2
kind: CiliumBGPClusterConfig
metadata:
name: cilium-unifi
spec:
nodeSelector:
matchLabels:
node-role.kubernetes.io/control-plane: ""
bgpInstances:
- name: "65200"
localASN: 65200
peers:
- name: "ucg-max-65100"
peerASN: 65100
peerAddress: 172.20.10.1
peerConfigRef:
name: ucg-max
+10
View File
@@ -0,0 +1,10 @@
apiVersion: cilium.io/v2
kind: CiliumLoadBalancerIPPool
metadata:
name: default-bgp-ip-pool
spec:
blocks:
- cidr: 172.20.10.0/24
serviceSelector:
matchLabels:
bgp.cilium.io/ip-pool: default
@@ -0,0 +1,17 @@
apiVersion: cilium.io/v2
kind: CiliumBGPPeerConfig
metadata:
name: ucg-max
spec:
authSecretRef: bgp-peer-ucg-max-password
timers:
connectRetryTimeSeconds: 10
gracefulRestart:
enabled: true
restartTimeSeconds: 15
families:
- afi: ipv4
safi: unicast
advertisements:
matchLabels:
bgp.cilium.io/advertise: loadbalancer-services
@@ -0,0 +1,12 @@
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
name: bgp-peer-ucg-max-password
namespace: cilium-secrets
spec:
encryptedData:
password: AgAfFRgvvxVt/KTOs7Ko646DwXNDTgXf+UuL21TjywwtVU7frAMfFx7U3+EfiBmXAbWsq2zAek0PGB8qhbaw9Q3Vnn5OgNed16dVOuYkV4QKqbarai/U2HXySts8pA6OR5ICE4WqJN/ayLsbN62jbMfjJMRlwIwXoRgW+6xLuspp81E/VUOQ0VBBKVlONzUWQWKgckpGjXLi7E1EEYQ6JVcKnDVUacM5adF0CjKkkQw/2MK4RnYhZXzmGTV1AnPe+ZK9HgjlmN5fqZ1cks4N1mfWPIGsRJX4CmWw4deqZ/nuB+3dhrDsdYCjxVNuOf9gTlyNwfBjETlYXuR9UCo/YuDzautmo6NlOy1nzl1O/nZbN4Rat+mbgFmn1sv5TwDUawcz4tNhKlrA+5iyVhmhNSYMqkNpkXRBJNRm+fK4Yspdo2yRl5jOX/dfkoWOgk9NijhfoDI2XGGpb7VIrD945i+9EBVe0UwMhnrhRWQZCTerx/1/kNajw77CUuD87KR+akeWVpxO1sG+tDwIceafeKbORYvl5S3TI24NkpS16Fdxl6PoFPVtiZlLqVxWRsnzK6Vb6cvWNGrlehHxURPrL9oB7+R+AM4W10/vqvX2NTNdIbzc2aT8qEq8gUlkPaRjcfeS9DKgnwM3OgF9UDyxjbRiHJ92dMOi3Gvu1F80crUZ0W6fdKhdsWQNqv84aS0JL0yaDLfbqNWf
template:
metadata:
name: bgp-peer-ucg-max-password
namespace: cilium-secrets
-8
View File
@@ -1,8 +0,0 @@
apiVersion: cilium.io/v2
kind: CiliumLoadBalancerIPPool
metadata:
name: first-pool
spec:
blocks:
- start: 192.168.1.220
stop: 192.168.1.255
+5 -2
View File
@@ -2,8 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- announce.yaml
- ip-pool.yaml
- bgp-ip-pool.yaml
- bgp-advertise-lb-services.yaml
- bgp-peer-ucg-max-config.yaml
- bgp-peer-ucg-max-password.yaml
- bgp-cluster-config.yaml
- dashboards/cilium.yaml
- dashboards/cilium-operator.yaml
+4 -6
View File
@@ -59,13 +59,11 @@ resources:
#debug:
# enabled: true
# Increase rate limit when doing L2 announcements
k8sClientRateLimit:
qps: 20
burst: 100
l2announcements:
# https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v2/
bgpControlPlane:
enabled: true
secretsNamespace:
name: cilium-secrets
externalIPs:
enabled: true
@@ -144,12 +144,12 @@ filtering:
parental_block_host: family-block.dns.adguard.com
safebrowsing_block_host: standard-block.dns.adguard.com
rewrites:
- domain: '*.stonegarden.dev'
answer: 192.168.1.220
- domain: stonegarden.dev
answer: 192.168.1.220
- domain: coturn.stonegarden.dev
answer: 192.168.1.226
answer: 172.20.10.140
- domain: '*.stonegarden.dev'
answer: 172.20.10.100
- domain: stonegarden.dev
answer: 172.20.10.100
safe_fs_patterns:
- /opt/adguardhome/work/data/userfilters/*
safebrowsing_cache_size: 1048576
+7 -1
View File
@@ -4,9 +4,15 @@ metadata:
name: adguard
namespace: dns
annotations:
io.cilium/lb-ipam-ips: 192.168.1.253
io.cilium/lb-ipam-ips: 172.20.10.153
labels:
bgp.cilium.io/advertise-service: default
bgp.cilium.io/ip-pool: default
spec:
type: LoadBalancer
externalTrafficPolicy: Local
# https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/
clusterIP: 10.96.0.12
ports:
- name: http
protocol: TCP
@@ -1,5 +1,10 @@
# A Record
#local-data: "somecomputer.local. A 192.168.1.1"
local-data: "coturn.stonegarden.dev A 172.20.10.140"
local-data: "*.stonegarden.dev A 172.20.10.100"
local-data: "stonegarden.dev A 172.20.10.100"
# PTR Record
#local-data-ptr: "192.168.1.1 somecomputer.local."
local-data-ptr: "172.20.10.100 stonegarden.dev."
local-data-ptr: "172.20.10.140 coturn.stonegarden.dev."
@@ -33,7 +33,6 @@ spec:
cpu: 10m
memory: 64Mi
limits:
cpu: 500m
memory: 128Mi
startupProbe:
exec:
+5 -1
View File
@@ -4,9 +4,13 @@ metadata:
name: unbound
namespace: dns
annotations:
io.cilium/lb-ipam-ips: 192.168.1.252
io.cilium/lb-ipam-ips: 172.20.10.150
labels:
bgp.cilium.io/advertise-service: default
bgp.cilium.io/ip-pool: default
spec:
type: LoadBalancer
externalTrafficPolicy: Local
# https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/
clusterIP: 10.96.0.11
ports:
+5 -1
View File
@@ -5,9 +5,13 @@ metadata:
namespace: gateway
spec:
gatewayClassName: cilium
infrastructure:
labels:
bgp.cilium.io/advertise-service: default
bgp.cilium.io/ip-pool: default
addresses:
- type: IPAddress
value: 192.168.1.222
value: 172.20.10.110
listeners:
- protocol: HTTPS
port: 443
+5 -1
View File
@@ -5,9 +5,13 @@ metadata:
namespace: gateway
spec:
gatewayClassName: cilium
infrastructure:
labels:
bgp.cilium.io/advertise-service: default
bgp.cilium.io/ip-pool: default
addresses:
- type: IPAddress
value: 192.168.1.220
value: 172.20.10.100
listeners:
- protocol: TLS
port: 443
+5 -1
View File
@@ -4,9 +4,13 @@ metadata:
name: coturn
namespace: coturn
annotations:
io.cilium/lb-ipam-ips: 192.168.1.226
io.cilium/lb-ipam-ips: 172.20.10.140
labels:
bgp.cilium.io/advertise-service: default
bgp.cilium.io/ip-pool: default
spec:
type: LoadBalancer
externalTrafficPolicy: Local
selector:
app.kubernetes.io/name: coturn
ports:
@@ -19,9 +19,10 @@ spec:
nodeSelector:
topology.kubernetes.io/zone: abel
dnsConfig:
# Internal AdGuard Home DNS
nameservers:
- 192.168.1.253
- 172.20.10.153 # AdGuard Home
- 10.96.0.12 # AdGuard Home
- 10.96.0.11 # Unbound
dnsPolicy: None
securityContext:
seccompProfile: