diff --git a/tofu/kubernetes/main.tf b/tofu/kubernetes/main.tf index 7286c85..3efdd1b 100644 --- a/tofu/kubernetes/main.tf +++ b/tofu/kubernetes/main.tf @@ -20,11 +20,19 @@ module "talos" { cluster = { name = "talos" + # This should point to the vip as below(if nodes on layer 2) or one of the nodes (if nodes not on layer 2) + # Note: Nodes are not on layer 2 if there is a router between them (even a mesh router) + # Not sure how it works if connected to the same router via ethernet (does it act as a switch then???) + # Ref: https://www.talos.dev/v1.9/talos-guides/network/vip/#requirements + # Note This is Kubernetes API endpoint. Different from all mentions of Talos endpoints. endpoint = "192.168.1.102" + # Omit this if devices are not connected on layer 2 + vip = "192.168.1.99" gateway = "192.168.1.1" talos_version = "v1.8" proxmox_cluster = "homelab" kubernetes_version = "1.32.0" # renovate: github-releases=kubernetes/kubernetes + base_domain = "stonegarden.dev" } nodes = { diff --git a/tofu/kubernetes/talos/config.tf b/tofu/kubernetes/talos/config.tf index 4d1c696..8f79095 100644 --- a/tofu/kubernetes/talos/config.tf +++ b/tofu/kubernetes/talos/config.tf @@ -6,31 +6,46 @@ data "talos_client_configuration" "this" { cluster_name = var.cluster.name client_configuration = talos_machine_secrets.this.client_configuration nodes = [for k, v in var.nodes : v.ip] + # Don't use vip in talosconfig endpoints + # ref - https://www.talos.dev/v1.9/talos-guides/network/vip/#caveats endpoints = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"] } data "talos_machine_configuration" "this" { for_each = var.nodes cluster_name = var.cluster.name + # This is the Kubernetes API Server endpoint. + # ref - https://www.talos.dev/v1.9/introduction/prodnotes/#decide-the-kubernetes-endpoint cluster_endpoint = "https://${var.cluster.endpoint}:6443" talos_version = var.cluster.talos_version machine_type = each.value.machine_type machine_secrets = talos_machine_secrets.this.machine_secrets - kubernetes_version = var.cluster.kubernetes_version - config_patches = each.value.machine_type == "controlplane" ? [ - templatefile("${path.module}/machine-config/control-plane.yaml.tftpl", { - hostname = each.key + config_patches = [ + templatefile("${path.module}/machine-config/common.yaml.tftpl", { node_name = each.value.host_node cluster_name = var.cluster.proxmox_cluster - cilium_values = var.cilium.values - cilium_install = var.cilium.install - }) - ] : [ - templatefile("${path.module}/machine-config/worker.yaml.tftpl", { - hostname = each.key - node_name = each.value.host_node - cluster_name = var.cluster.proxmox_cluster }) + , each.value.machine_type == "controlplane" ? + templatefile("${path.module}/machine-config/control-plane.yaml.tftpl", { + cilium_values = var.cilium.values + cilium_install = var.cilium.install + base_domain = var.cluster.base_domain + }) : "" + , (each.value.machine_type == "controlplane" && var.cluster.vip != null) + ? templatefile("${path.module}/machine-config/network-vip.yaml.tftpl", { + hostname = each.key + ip = each.value.ip + mac_address = lower(each.value.mac_address) + gateway = var.cluster.gateway + subnet_mask = var.cluster.subnet_mask + vip = var.cluster.vip + }) : templatefile("${path.module}/machine-config/network-no-vip.yaml.tftpl", { + hostname = each.key + ip = each.value.ip + mac_address = lower(each.value.mac_address) + gateway = var.cluster.gateway + subnet_mask = var.cluster.subnet_mask + }) ] } @@ -47,11 +62,10 @@ resource "talos_machine_configuration_apply" "this" { } resource "talos_machine_bootstrap" "this" { - depends_on = [talos_machine_configuration_apply.this] - //for_each = var.nodes - //node = each.value.ip - node = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"][2] - endpoint = var.cluster.endpoint + depends_on = [talos_machine_configuration_apply.this] + # Bootstrap with the first node. VIP not yet available at this stage, so cant use var.cluster.endpoint as it may be set to VIP + # ref - https://www.talos.dev/v1.9/talos-guides/network/vip/#caveats + node = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"][0] client_configuration = talos_machine_secrets.this.client_configuration } @@ -75,8 +89,11 @@ resource "talos_cluster_kubeconfig" "this" { talos_machine_bootstrap.this, data.talos_cluster_health.this ] - node = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"][2] - endpoint = var.cluster.endpoint + # If using VIP, it should be up by now, but to be safer retrive from one of the nodes + # As mentioned don't use talosctl on vip + # ref - https://www.talos.dev/v1.9/talos-guides/network/vip/#caveats + # In kubeconfig endpoint will be polulated by cluster_endpoint from machine-config + node = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"][0] client_configuration = talos_machine_secrets.this.client_configuration timeouts = { read = "1m" diff --git a/tofu/kubernetes/talos/machine-config/common.yaml.tftpl b/tofu/kubernetes/talos/machine-config/common.yaml.tftpl new file mode 100644 index 0000000..5fb0204 --- /dev/null +++ b/tofu/kubernetes/talos/machine-config/common.yaml.tftpl @@ -0,0 +1,17 @@ +machine: + sysctls: + fs.inotify.max_user_watches: 1048576 # Watchdog + fs.inotify.max_user_instances: 8192 # Watchdog + net.core.default_qdisc: fq # 10Gb/s + net.core.rmem_max: 67108864 # 10Gb/s | Cloudflared / QUIC + net.core.wmem_max: 67108864 # 10Gb/s | Cloudflared / QUIC + net.ipv4.tcp_congestion_control: bbr # 10Gb/s + net.ipv4.tcp_fastopen: 3 # Send and accept data in the opening SYN packet + net.ipv4.tcp_mtu_probing: 1 # 10Gb/s | Jumbo frames + net.ipv4.tcp_rmem: 4096 87380 33554432 # 10Gb/s + net.ipv4.tcp_wmem: 4096 65536 33554432 # 10Gb/s + net.ipv4.tcp_window_scaling: 1 # 10Gb/s + vm.nr_hugepages: 1024 # PostgreSQL + nodeLabels: + topology.kubernetes.io/region: ${cluster_name} + topology.kubernetes.io/zone: ${node_name} diff --git a/tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl b/tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl index 97a9595..87306e3 100644 --- a/tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl +++ b/tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl @@ -3,17 +3,12 @@ machine: extraArgs: # Needed for Netbird agent https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls allowed-unsafe-sysctls: net.ipv4.conf.all.src_valid_mark - network: - hostname: ${hostname} - nodeLabels: - topology.kubernetes.io/region: ${cluster_name} - topology.kubernetes.io/zone: ${node_name} cluster: allowSchedulingOnControlPlanes: true apiServer: extraArgs: - oidc-issuer-url: https://authelia.stonegarden.dev + oidc-issuer-url: https://authelia.${base_domain} oidc-client-id: kubectl oidc-username-claim: preferred_username oidc-username-prefix: 'authelia:' @@ -24,6 +19,11 @@ cluster: name: none proxy: disabled: true + discovery: + enabled: true + registries: + service: + disabled: false extraManifests: - https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.1/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml diff --git a/tofu/kubernetes/talos/machine-config/network-no-vip.yaml.tftpl b/tofu/kubernetes/talos/machine-config/network-no-vip.yaml.tftpl new file mode 100644 index 0000000..5cc4191 --- /dev/null +++ b/tofu/kubernetes/talos/machine-config/network-no-vip.yaml.tftpl @@ -0,0 +1,12 @@ +machine: + network: + hostname: ${hostname} + interfaces: + - deviceSelector: + hardwareAddr: ${mac_address} + addresses: + - ${ip}/${subnet_mask} + routes: + - network: 0.0.0.0/0 + gateway: ${gateway} + dhcp: false diff --git a/tofu/kubernetes/talos/machine-config/network-vip.yaml.tftpl b/tofu/kubernetes/talos/machine-config/network-vip.yaml.tftpl new file mode 100644 index 0000000..32c3e5f --- /dev/null +++ b/tofu/kubernetes/talos/machine-config/network-vip.yaml.tftpl @@ -0,0 +1,14 @@ +machine: + network: + hostname: ${hostname} + interfaces: + - deviceSelector: + hardwareAddr: ${mac_address} + addresses: + - ${ip}/${subnet_mask} + routes: + - network: 0.0.0.0/0 + gateway: ${gateway} + dhcp: false + vip: + ip: ${vip} diff --git a/tofu/kubernetes/talos/machine-config/worker.yaml.tftpl b/tofu/kubernetes/talos/machine-config/worker.yaml.tftpl deleted file mode 100644 index 6d2480e..0000000 --- a/tofu/kubernetes/talos/machine-config/worker.yaml.tftpl +++ /dev/null @@ -1,6 +0,0 @@ -machine: - network: - hostname: ${hostname} - nodeLabels: - topology.kubernetes.io/region: ${cluster_name} - topology.kubernetes.io/zone: ${node_name} diff --git a/tofu/kubernetes/talos/variables.tf b/tofu/kubernetes/talos/variables.tf index 86d3bf2..0751748 100644 --- a/tofu/kubernetes/talos/variables.tf +++ b/tofu/kubernetes/talos/variables.tf @@ -17,10 +17,13 @@ variable "cluster" { type = object({ name = string endpoint = string + vip = optional(string) gateway = string + subnet_mask = optional(string, "24") talos_version = string proxmox_cluster = string kubernetes_version = string + base_domain = string }) } diff --git a/tofu/kubernetes/talos/virtual-machines.tf b/tofu/kubernetes/talos/virtual-machines.tf index 034febd..9dde288 100644 --- a/tofu/kubernetes/talos/virtual-machines.tf +++ b/tofu/kubernetes/talos/virtual-machines.tf @@ -53,7 +53,7 @@ resource "proxmox_virtual_environment_vm" "this" { datastore_id = each.value.datastore_id ip_config { ipv4 { - address = "${each.value.ip}/24" + address = "${each.value.ip}/${var.cluster.subnet_mask}" gateway = var.cluster.gateway } }