fix(tofu): allow multiple nodes on the same host again

A bug reported in #299 resulted in it not being possible to schedule two Talos nodes on the same host machine with the same schematic ID and image version

This commit fixes #299

Signed-off-by: Vegard Hagen <[email protected]>

tofu/kubernetes/talos/config.tofu

@@ -1,6 +1,6 @@
 locals {
   first_control_plane_node_ip = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"][0]
-  kubernetes_endpoint         = coalesce(var.cluster.vip, local.first_control_plane_node_ip)
+  kubernetes_endpoint = coalesce(var.cluster.vip, local.first_control_plane_node_ip)
   extra_manifests = concat(var.cluster.extra_manifests, [
     "https://github.com/kubernetes-sigs/gateway-api/releases/download/${var.cluster.gateway_api_version}/standard-install.yaml",
     "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/${var.cluster.gateway_api_version}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml"
@@ -21,7 +21,7 @@ data "talos_client_configuration" "this" {
 resource "terraform_data" "cilium_bootstrap_inline_manifests" {
   input = [
     {
-      name     = "cilium-bootstrap"
+      name = "cilium-bootstrap"
       contents = file("${path.root}/${var.cluster.cilium.bootstrap_manifest_path}")
     },
     {
@@ -42,8 +42,8 @@ resource "terraform_data" "cilium_bootstrap_inline_manifests" {
 }
 
 data "talos_machine_configuration" "this" {
-  for_each     = var.nodes
+  for_each        = var.nodes
-  cluster_name = var.cluster.name
+  cluster_name    = var.cluster.name
   # This is the Kubernetes API Server endpoint.
   # ref - https://www.talos.dev/latest/introduction/prodnotes/#decide-the-kubernetes-endpoint
   cluster_endpoint = "https://${local.kubernetes_endpoint}:6443"
@@ -58,23 +58,29 @@ data "talos_machine_configuration" "this" {
       cluster_name       = var.cluster.proxmox_cluster
       kubernetes_version = var.cluster.kubernetes_version
       hostname           = each.key
-      ip                 = each.value.ip
+      kubelet            = var.cluster.kubelet
-      mac_address        = lower(each.value.mac_address)
-      gateway            = var.cluster.gateway
-      subnet_mask        = var.cluster.subnet_mask
-      vip                = var.cluster.vip
     }), each.value.machine_type == "controlplane" ?
-    templatefile("${path.module}/machine-config/control-plane.yaml.tftpl", {
+      templatefile("${path.module}/machine-config/control-plane.yaml.tftpl", {
-      kubelet          = var.cluster.kubelet
+        ip          = each.value.ip
-      extra_manifests  = jsonencode(local.extra_manifests)
+        mac_address = lower(each.value.mac_address)
-      api_server       = var.cluster.api_server
+        gateway     = var.cluster.gateway
-      inline_manifests = jsonencode(terraform_data.cilium_bootstrap_inline_manifests.output)
+        subnet_mask = var.cluster.subnet_mask
-    }) : ""
+        vip         = var.cluster.vip
+        extra_manifests = jsonencode(local.extra_manifests)
+        api_server  = var.cluster.api_server
+        inline_manifests = jsonencode(terraform_data.cilium_bootstrap_inline_manifests.output)
+      }) :
+      templatefile("${path.module}/machine-config/worker.yaml.tftpl", {
+        ip          = each.value.ip
+        mac_address = lower(each.value.mac_address)
+        gateway     = var.cluster.gateway
+        subnet_mask = var.cluster.subnet_mask
+      })
   ]
 }
 
 resource "talos_machine_configuration_apply" "this" {
-  depends_on                  = [proxmox_virtual_environment_vm.this]
+  depends_on = [proxmox_virtual_environment_vm.this]
   for_each                    = var.nodes
   node                        = each.value.ip
   client_configuration        = talos_machine_secrets.this.client_configuration

tofu/kubernetes/talos/image.tofu

@@ -1,12 +1,12 @@
 locals {
   version      = var.image.version
-  schematic    = file("${path.root}/${var.image.schematic_path}")
+  schematic = file("${path.root}/${var.image.schematic_path}")
   schematic_id = jsondecode(data.http.schematic_id.response_body)["id"]
 
-  update_version        = coalesce(var.image.update_version, var.image.version)
+  update_version = coalesce(var.image.update_version, var.image.version)
   update_schematic_path = coalesce(var.image.update_schematic_path, var.image.schematic_path)
-  update_schematic      = file("${path.root}/${local.update_schematic_path}")
+  update_schematic = file("${path.root}/${local.update_schematic_path}")
-  update_schematic_id   = jsondecode(data.http.updated_schematic_id.response_body)["id"]
+  update_schematic_id = jsondecode(data.http.updated_schematic_id.response_body)["id"]
 
   image_id        = "${local.schematic_id}_${local.version}"
   update_image_id = "${local.update_schematic_id}_${local.update_version}"
@@ -37,22 +37,26 @@ resource "talos_image_factory_schematic" "updated" {
   schematic = local.update_schematic
 }
 
+# Note the ellipsis (...) after the for-loop. This collects values with same keys into a list.
+# The key is purposefully made up of the values (image_id contains both schematic id and version),
+# since all values under a key therefore are the same, we can simply select the first element of the value list.
+# Improvements are welcome!
 resource "proxmox_virtual_environment_download_file" "this" {
   for_each = {
     for k, v in var.nodes :
     "${v.host_node}_${v.update == true ? local.update_image_id : local.image_id}" => {
       host_node = v.host_node
-      version   = v.update == true ? local.update_version : local.version
       schematic = v.update == true ? talos_image_factory_schematic.updated.id : talos_image_factory_schematic.this.id
-    }
+      version   = v.update == true ? local.update_version : local.version
+    }...
   }
 
-  node_name    = each.value.host_node
+  node_name    = each.value[0].host_node
   content_type = "iso"
   datastore_id = var.image.proxmox_datastore
 
-  file_name               = "talos-${each.value.schematic}-${each.value.version}-${var.image.platform}-${var.image.arch}.img"
+  file_name               = "talos-${each.value[0].schematic}-${each.value[0].version}-${var.image.platform}-${var.image.arch}.img"
-  url                     = "${var.image.factory_url}/image/${each.value.schematic}/${each.value.version}/${var.image.platform}-${var.image.arch}.raw.gz"
+  url                     = "${var.image.factory_url}/image/${each.value[0].schematic}/${each.value[0].version}/${var.image.platform}-${var.image.arch}.raw.gz"
   decompression_algorithm = "gz"
   overwrite               = false
 }

tofu/kubernetes/talos/machine-config/common.yaml.tftpl

@@ -1,24 +1,12 @@
 machine:
+  kubelet:
+    image: ghcr.io/siderolabs/kubelet:${kubernetes_version}
+    ${indent(4, kubelet)}
   nodeLabels:
     topology.kubernetes.io/region: ${cluster_name}
     topology.kubernetes.io/zone: ${node_name}
-  kubelet:
-    image: ghcr.io/siderolabs/kubelet:${kubernetes_version}
   network:
     hostname: ${hostname}
-    interfaces:
-      - deviceSelector:
-          hardwareAddr: ${mac_address}
-        addresses:
-          - ${ip}/${subnet_mask}
-        routes:
-          - network: 0.0.0.0/0
-            gateway: ${gateway}
-        dhcp: false
-%{ if vip != null }
-        vip:
-          ip: ${vip}
-%{ endif }
   sysctls:
     fs.inotify.max_user_watches: 1048576   # Watchdog
     fs.inotify.max_user_instances: 8192    # Watchdog

tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl

@@ -1,6 +1,18 @@
 machine:
-  kubelet:
+  network:
-    ${indent(4, kubelet)}
+    interfaces:
+      - deviceSelector:
+          hardwareAddr: ${mac_address}
+        addresses:
+          - ${ip}/${subnet_mask}
+        routes:
+          - network: 0.0.0.0/0
+            gateway: ${gateway}
+        dhcp: false
+%{ if vip != null }
+        vip:
+          ip: ${vip}
+%{ endif }
 
 cluster:
   allowSchedulingOnControlPlanes: true

tofu/kubernetes/talos/machine-config/worker.yaml.tftpl

@@ -0,0 +1,11 @@
+machine:
+  network:
+    interfaces:
+      - deviceSelector:
+          hardwareAddr: ${mac_address}
+        addresses:
+          - ${ip}/${subnet_mask}
+        routes:
+          - network: 0.0.0.0/0
+            gateway: ${gateway}
+        dhcp: false

tofu/kubernetes/talos_nodes.auto.tfvars

@@ -18,7 +18,7 @@ talos_nodes = {
     cpu           = 4
     ram_dedicated = 20480
     igpu          = true
-#    update        = true
+    #update        = true
   }
   "ctrl-02" = {
     host_node     = "cantor"
@@ -28,16 +28,16 @@ talos_nodes = {
     vm_id         = 802
     cpu           = 4
     ram_dedicated = 6144
-#    update        = true
+    #update        = true
+  }
+  "work-00" = {
+    host_node     = "abel"
+    machine_type  = "worker"
+    ip            = "192.168.1.110"
+    dns           = ["1.1.1.1", "8.8.8.8"] # Optional Value.
+    mac_address   = "BC:24:11:2E:A8:00"
+    vm_id         = 810
+    cpu           = 8
+    ram_dedicated = 4096
   }
-  #    "work-00" = {
-  #      host_node     = "abel"
-  #      machine_type  = "worker"
-  #      ip            = "192.168.1.110"
-  #      dns           = ["1.1.1.1", "8.8.8.8"] # Optional Value.
-  #      mac_address   = "BC:24:11:2E:A8:00"
-  #      vm_id         = 810
-  #      cpu           = 8
-  #      ram_dedicated = 4096
-  #    }
 }