feat: Added provision to provide a VIP for talos cluster

refactor: Seperated machine configs refactor: Made subnet configurable refactor: Made domain name for oidc configurable docs: Detailed comments in the code about VIP, endpoints, etc Signed-off-by: Karteek <[email protected]>
2026-08-25 14:53:19 +00:00 · 2025-03-23 12:19:17 +01:00
parent 2e77f47aec
commit 47142aad99
9 changed files with 97 additions and 32 deletions
@@ -20,11 +20,19 @@ module "talos" {

  cluster = {
    name            = "talos"
+    # This should point to the vip as below(if nodes on layer 2) or one of the nodes (if nodes not on layer 2)
+    # Note: Nodes are not on layer 2 if there is a router between them (even a mesh router)
+    #       Not sure how it works if connected to the same router via ethernet (does it act as a switch then???)
+    # Ref: https://www.talos.dev/v1.9/talos-guides/network/vip/#requirements
+    # Note This is Kubernetes API endpoint. Different from all mentions of Talos endpoints.
    endpoint        = "192.168.1.102"
+    # Omit this if devices are not connected on layer 2
+    vip             = "192.168.1.99"
    gateway         = "192.168.1.1"
    talos_version   = "v1.8"
    proxmox_cluster = "homelab"
    kubernetes_version = "1.32.0"  # renovate: github-releases=kubernetes/kubernetes
+    base_domain     = "stonegarden.dev"
  }

  nodes = {
@@ -6,31 +6,46 @@ data "talos_client_configuration" "this" {
  cluster_name         = var.cluster.name
  client_configuration = talos_machine_secrets.this.client_configuration
  nodes                = [for k, v in var.nodes : v.ip]
+  # Don't use vip in talosconfig endpoints
+  # ref - https://www.talos.dev/v1.9/talos-guides/network/vip/#caveats
  endpoints            = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"]
 }

 data "talos_machine_configuration" "this" {
  for_each         = var.nodes
  cluster_name     = var.cluster.name
+  # This is the Kubernetes API Server endpoint.
+  # ref - https://www.talos.dev/v1.9/introduction/prodnotes/#decide-the-kubernetes-endpoint
  cluster_endpoint = "https://${var.cluster.endpoint}:6443"
  talos_version    = var.cluster.talos_version
  machine_type     = each.value.machine_type
  machine_secrets  = talos_machine_secrets.this.machine_secrets
-  kubernetes_version = var.cluster.kubernetes_version
-  config_patches   = each.value.machine_type == "controlplane" ? [
-    templatefile("${path.module}/machine-config/control-plane.yaml.tftpl", {
-      hostname       = each.key
+  config_patches   = [
+    templatefile("${path.module}/machine-config/common.yaml.tftpl", {
      node_name      = each.value.host_node
      cluster_name   = var.cluster.proxmox_cluster
-      cilium_values  = var.cilium.values
-      cilium_install = var.cilium.install
-    })
-  ] : [
-    templatefile("${path.module}/machine-config/worker.yaml.tftpl", {
-      hostname     = each.key
-      node_name    = each.value.host_node
-      cluster_name = var.cluster.proxmox_cluster
    })
+    , each.value.machine_type == "controlplane" ?
+        templatefile("${path.module}/machine-config/control-plane.yaml.tftpl", {
+          cilium_values  = var.cilium.values
+          cilium_install = var.cilium.install
+          base_domain    = var.cluster.base_domain
+        }) : ""
+    , (each.value.machine_type == "controlplane" && var.cluster.vip != null)
+        ? templatefile("${path.module}/machine-config/network-vip.yaml.tftpl", {
+            hostname       = each.key
+            ip             = each.value.ip
+            mac_address    = lower(each.value.mac_address)
+            gateway        = var.cluster.gateway
+            subnet_mask    = var.cluster.subnet_mask
+            vip            = var.cluster.vip
+          }) : templatefile("${path.module}/machine-config/network-no-vip.yaml.tftpl", {
+            hostname       = each.key
+            ip             = each.value.ip
+            mac_address    = lower(each.value.mac_address)
+            gateway        = var.cluster.gateway
+            subnet_mask    = var.cluster.subnet_mask
+          })
  ]
 }

@@ -47,11 +62,10 @@ resource "talos_machine_configuration_apply" "this" {
 }

 resource "talos_machine_bootstrap" "this" {
-  depends_on = [talos_machine_configuration_apply.this]
-  //for_each             = var.nodes
-  //node                 = each.value.ip
-  node                 = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"][2]
-  endpoint             = var.cluster.endpoint
+  depends_on           = [talos_machine_configuration_apply.this]
+  # Bootstrap with the first node. VIP not yet available at this stage, so cant use var.cluster.endpoint as it may be set to VIP
+  # ref - https://www.talos.dev/v1.9/talos-guides/network/vip/#caveats
+  node                 = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"][0]
  client_configuration = talos_machine_secrets.this.client_configuration
 }

@@ -75,8 +89,11 @@ resource "talos_cluster_kubeconfig" "this" {
    talos_machine_bootstrap.this,
    data.talos_cluster_health.this
  ]
-  node                 = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"][2]
-  endpoint             = var.cluster.endpoint
+  # If using VIP, it should be up by now, but to be safer retrive from one of the nodes
+  # As mentioned don't use talosctl on vip
+  # ref - https://www.talos.dev/v1.9/talos-guides/network/vip/#caveats
+  # In kubeconfig endpoint will be polulated by cluster_endpoint from machine-config
+  node                 = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"][0]
  client_configuration = talos_machine_secrets.this.client_configuration
  timeouts = {
    read = "1m"
@@ -0,0 +1,17 @@
+machine:
+  sysctls:
+    fs.inotify.max_user_watches: 1048576   # Watchdog
+    fs.inotify.max_user_instances: 8192    # Watchdog
+    net.core.default_qdisc: fq             # 10Gb/s
+    net.core.rmem_max: 67108864            # 10Gb/s | Cloudflared / QUIC
+    net.core.wmem_max: 67108864            # 10Gb/s | Cloudflared / QUIC
+    net.ipv4.tcp_congestion_control: bbr   # 10Gb/s
+    net.ipv4.tcp_fastopen: 3               # Send and accept data in the opening SYN packet
+    net.ipv4.tcp_mtu_probing: 1            # 10Gb/s | Jumbo frames
+    net.ipv4.tcp_rmem: 4096 87380 33554432 # 10Gb/s
+    net.ipv4.tcp_wmem: 4096 65536 33554432 # 10Gb/s
+    net.ipv4.tcp_window_scaling: 1         # 10Gb/s
+    vm.nr_hugepages: 1024                  # PostgreSQL
+  nodeLabels:
+    topology.kubernetes.io/region: ${cluster_name}
+    topology.kubernetes.io/zone: ${node_name}
@@ -3,17 +3,12 @@ machine:
    extraArgs:
      # Needed for Netbird agent https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls
      allowed-unsafe-sysctls: net.ipv4.conf.all.src_valid_mark
-  network:
-    hostname: ${hostname}
-  nodeLabels:
-    topology.kubernetes.io/region: ${cluster_name}
-    topology.kubernetes.io/zone: ${node_name}

 cluster:
  allowSchedulingOnControlPlanes: true
  apiServer:
    extraArgs:
-      oidc-issuer-url: https://authelia.stonegarden.dev
+      oidc-issuer-url: https://authelia.${base_domain}
      oidc-client-id: kubectl
      oidc-username-claim: preferred_username
      oidc-username-prefix: 'authelia:'
@@ -24,6 +19,11 @@ cluster:
      name: none
  proxy:
    disabled: true
+  discovery:
+    enabled: true
+    registries:
+      service:
+        disabled: false
  extraManifests:
    - https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml
    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.1/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml
@@ -0,0 +1,12 @@
+machine:
+  network:
+    hostname: ${hostname}
+    interfaces:
+      - deviceSelector:
+          hardwareAddr: ${mac_address}
+        addresses:
+          - ${ip}/${subnet_mask}
+        routes:
+          - network: 0.0.0.0/0
+            gateway: ${gateway}
+        dhcp: false
@@ -0,0 +1,14 @@
+machine:
+  network:
+    hostname: ${hostname}
+    interfaces:
+      - deviceSelector:
+          hardwareAddr: ${mac_address}
+        addresses:
+          - ${ip}/${subnet_mask}
+        routes:
+          - network: 0.0.0.0/0
+            gateway: ${gateway}
+        dhcp: false
+        vip:
+          ip: ${vip}
@@ -1,6 +0,0 @@
-machine:
-  network:
-    hostname: ${hostname}
-  nodeLabels:
-    topology.kubernetes.io/region: ${cluster_name}
-    topology.kubernetes.io/zone: ${node_name}
@@ -17,10 +17,13 @@ variable "cluster" {
  type = object({
    name            = string
    endpoint        = string
+    vip             = optional(string)
    gateway         = string
+    subnet_mask     = optional(string, "24")
    talos_version   = string
    proxmox_cluster = string
    kubernetes_version = string
+    base_domain     = string
  })
 }

@@ -53,7 +53,7 @@ resource "proxmox_virtual_environment_vm" "this" {
    datastore_id = each.value.datastore_id
    ip_config {
      ipv4 {
-        address = "${each.value.ip}/24"
+        address = "${each.value.ip}/${var.cluster.subnet_mask}"
        gateway = var.cluster.gateway
      }
    }