diff --git a/tofu/kubernetes/talos/config.tofu b/tofu/kubernetes/talos/config.tofu index 67ca7f6..30a6479 100644 --- a/tofu/kubernetes/talos/config.tofu +++ b/tofu/kubernetes/talos/config.tofu @@ -1,6 +1,6 @@ locals { first_control_plane_node_ip = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"][0] - kubernetes_endpoint = coalesce(var.cluster.vip, local.first_control_plane_node_ip) + kubernetes_endpoint = coalesce(var.cluster.vip, local.first_control_plane_node_ip) extra_manifests = concat(var.cluster.extra_manifests, [ "https://github.com/kubernetes-sigs/gateway-api/releases/download/${var.cluster.gateway_api_version}/standard-install.yaml", "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/${var.cluster.gateway_api_version}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml" @@ -21,7 +21,7 @@ data "talos_client_configuration" "this" { resource "terraform_data" "cilium_bootstrap_inline_manifests" { input = [ { - name = "cilium-bootstrap" + name = "cilium-bootstrap" contents = file("${path.root}/${var.cluster.cilium.bootstrap_manifest_path}") }, { @@ -42,8 +42,8 @@ resource "terraform_data" "cilium_bootstrap_inline_manifests" { } data "talos_machine_configuration" "this" { - for_each = var.nodes - cluster_name = var.cluster.name + for_each = var.nodes + cluster_name = var.cluster.name # This is the Kubernetes API Server endpoint. # ref - https://www.talos.dev/latest/introduction/prodnotes/#decide-the-kubernetes-endpoint cluster_endpoint = "https://${local.kubernetes_endpoint}:6443" @@ -58,23 +58,29 @@ data "talos_machine_configuration" "this" { cluster_name = var.cluster.proxmox_cluster kubernetes_version = var.cluster.kubernetes_version hostname = each.key - ip = each.value.ip - mac_address = lower(each.value.mac_address) - gateway = var.cluster.gateway - subnet_mask = var.cluster.subnet_mask - vip = var.cluster.vip + kubelet = var.cluster.kubelet }), each.value.machine_type == "controlplane" ? - templatefile("${path.module}/machine-config/control-plane.yaml.tftpl", { - kubelet = var.cluster.kubelet - extra_manifests = jsonencode(local.extra_manifests) - api_server = var.cluster.api_server - inline_manifests = jsonencode(terraform_data.cilium_bootstrap_inline_manifests.output) - }) : "" + templatefile("${path.module}/machine-config/control-plane.yaml.tftpl", { + ip = each.value.ip + mac_address = lower(each.value.mac_address) + gateway = var.cluster.gateway + subnet_mask = var.cluster.subnet_mask + vip = var.cluster.vip + extra_manifests = jsonencode(local.extra_manifests) + api_server = var.cluster.api_server + inline_manifests = jsonencode(terraform_data.cilium_bootstrap_inline_manifests.output) + }) : + templatefile("${path.module}/machine-config/worker.yaml.tftpl", { + ip = each.value.ip + mac_address = lower(each.value.mac_address) + gateway = var.cluster.gateway + subnet_mask = var.cluster.subnet_mask + }) ] } resource "talos_machine_configuration_apply" "this" { - depends_on = [proxmox_virtual_environment_vm.this] + depends_on = [proxmox_virtual_environment_vm.this] for_each = var.nodes node = each.value.ip client_configuration = talos_machine_secrets.this.client_configuration diff --git a/tofu/kubernetes/talos/image.tofu b/tofu/kubernetes/talos/image.tofu index c9278a4..9532653 100644 --- a/tofu/kubernetes/talos/image.tofu +++ b/tofu/kubernetes/talos/image.tofu @@ -1,12 +1,12 @@ locals { version = var.image.version - schematic = file("${path.root}/${var.image.schematic_path}") + schematic = file("${path.root}/${var.image.schematic_path}") schematic_id = jsondecode(data.http.schematic_id.response_body)["id"] - update_version = coalesce(var.image.update_version, var.image.version) + update_version = coalesce(var.image.update_version, var.image.version) update_schematic_path = coalesce(var.image.update_schematic_path, var.image.schematic_path) - update_schematic = file("${path.root}/${local.update_schematic_path}") - update_schematic_id = jsondecode(data.http.updated_schematic_id.response_body)["id"] + update_schematic = file("${path.root}/${local.update_schematic_path}") + update_schematic_id = jsondecode(data.http.updated_schematic_id.response_body)["id"] image_id = "${local.schematic_id}_${local.version}" update_image_id = "${local.update_schematic_id}_${local.update_version}" @@ -37,22 +37,26 @@ resource "talos_image_factory_schematic" "updated" { schematic = local.update_schematic } +# Note the ellipsis (...) after the for-loop. This collects values with same keys into a list. +# The key is purposefully made up of the values (image_id contains both schematic id and version), +# since all values under a key therefore are the same, we can simply select the first element of the value list. +# Improvements are welcome! resource "proxmox_virtual_environment_download_file" "this" { for_each = { for k, v in var.nodes : "${v.host_node}_${v.update == true ? local.update_image_id : local.image_id}" => { host_node = v.host_node - version = v.update == true ? local.update_version : local.version schematic = v.update == true ? talos_image_factory_schematic.updated.id : talos_image_factory_schematic.this.id - } + version = v.update == true ? local.update_version : local.version + }... } - node_name = each.value.host_node + node_name = each.value[0].host_node content_type = "iso" datastore_id = var.image.proxmox_datastore - file_name = "talos-${each.value.schematic}-${each.value.version}-${var.image.platform}-${var.image.arch}.img" - url = "${var.image.factory_url}/image/${each.value.schematic}/${each.value.version}/${var.image.platform}-${var.image.arch}.raw.gz" + file_name = "talos-${each.value[0].schematic}-${each.value[0].version}-${var.image.platform}-${var.image.arch}.img" + url = "${var.image.factory_url}/image/${each.value[0].schematic}/${each.value[0].version}/${var.image.platform}-${var.image.arch}.raw.gz" decompression_algorithm = "gz" overwrite = false } diff --git a/tofu/kubernetes/talos/machine-config/common.yaml.tftpl b/tofu/kubernetes/talos/machine-config/common.yaml.tftpl index b0a6f2f..c254200 100644 --- a/tofu/kubernetes/talos/machine-config/common.yaml.tftpl +++ b/tofu/kubernetes/talos/machine-config/common.yaml.tftpl @@ -1,24 +1,12 @@ machine: + kubelet: + image: ghcr.io/siderolabs/kubelet:${kubernetes_version} + ${indent(4, kubelet)} nodeLabels: topology.kubernetes.io/region: ${cluster_name} topology.kubernetes.io/zone: ${node_name} - kubelet: - image: ghcr.io/siderolabs/kubelet:${kubernetes_version} network: hostname: ${hostname} - interfaces: - - deviceSelector: - hardwareAddr: ${mac_address} - addresses: - - ${ip}/${subnet_mask} - routes: - - network: 0.0.0.0/0 - gateway: ${gateway} - dhcp: false -%{ if vip != null } - vip: - ip: ${vip} -%{ endif } sysctls: fs.inotify.max_user_watches: 1048576 # Watchdog fs.inotify.max_user_instances: 8192 # Watchdog diff --git a/tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl b/tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl index ffc5a04..08c8693 100644 --- a/tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl +++ b/tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl @@ -1,6 +1,18 @@ machine: - kubelet: - ${indent(4, kubelet)} + network: + interfaces: + - deviceSelector: + hardwareAddr: ${mac_address} + addresses: + - ${ip}/${subnet_mask} + routes: + - network: 0.0.0.0/0 + gateway: ${gateway} + dhcp: false +%{ if vip != null } + vip: + ip: ${vip} +%{ endif } cluster: allowSchedulingOnControlPlanes: true diff --git a/tofu/kubernetes/talos/machine-config/worker.yaml.tftpl b/tofu/kubernetes/talos/machine-config/worker.yaml.tftpl new file mode 100644 index 0000000..cbb8e52 --- /dev/null +++ b/tofu/kubernetes/talos/machine-config/worker.yaml.tftpl @@ -0,0 +1,11 @@ +machine: + network: + interfaces: + - deviceSelector: + hardwareAddr: ${mac_address} + addresses: + - ${ip}/${subnet_mask} + routes: + - network: 0.0.0.0/0 + gateway: ${gateway} + dhcp: false diff --git a/tofu/kubernetes/talos_nodes.auto.tfvars b/tofu/kubernetes/talos_nodes.auto.tfvars index a3002cd..4ae9261 100644 --- a/tofu/kubernetes/talos_nodes.auto.tfvars +++ b/tofu/kubernetes/talos_nodes.auto.tfvars @@ -18,7 +18,7 @@ talos_nodes = { cpu = 4 ram_dedicated = 20480 igpu = true -# update = true + #update = true } "ctrl-02" = { host_node = "cantor" @@ -28,16 +28,16 @@ talos_nodes = { vm_id = 802 cpu = 4 ram_dedicated = 6144 -# update = true + #update = true + } + "work-00" = { + host_node = "abel" + machine_type = "worker" + ip = "192.168.1.110" + dns = ["1.1.1.1", "8.8.8.8"] # Optional Value. + mac_address = "BC:24:11:2E:A8:00" + vm_id = 810 + cpu = 8 + ram_dedicated = 4096 } - # "work-00" = { - # host_node = "abel" - # machine_type = "worker" - # ip = "192.168.1.110" - # dns = ["1.1.1.1", "8.8.8.8"] # Optional Value. - # mac_address = "BC:24:11:2E:A8:00" - # vm_id = 810 - # cpu = 8 - # ram_dedicated = 4096 - # } }