diff --git a/.gitignore b/.gitignore index a1950ca..3668a32 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,5 @@ override.tf.json *_override.tf *_override.tf.json -*.qcow2 \ No newline at end of file +*.qcow2 +*.raw \ No newline at end of file diff --git a/machines/cantor/main.tf b/machines/cantor/main.tf new file mode 100644 index 0000000..3cc76e5 --- /dev/null +++ b/machines/cantor/main.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "0.50.0" + } + } +} + +provider "proxmox" { + alias = "cantor" + endpoint = var.cantor.endpoint + insecure = var.cantor.insecure + + api_token = var.cantor_auth.api_token + ssh { + agent = true + username = var.cantor_auth.username + } + + tmp_dir = "/var/tmp" +} \ No newline at end of file diff --git a/machines/cantor/truenas-scale-vm.tf b/machines/cantor/truenas-scale-vm.tf new file mode 100644 index 0000000..142330d --- /dev/null +++ b/machines/cantor/truenas-scale-vm.tf @@ -0,0 +1,141 @@ +resource "proxmox_virtual_environment_download_file" "truenas-scale-23" { + provider = proxmox.cantor + node_name = var.cantor.node_name + content_type = "iso" + datastore_id = "local" + + file_name = "TrueNAS-SCALE-23.10.2.iso" + url = "https://download.sys.truenas.net/TrueNAS-SCALE-Cobia/23.10.2/TrueNAS-SCALE-23.10.2.iso" + checksum = "c2b0d6ef6ca6a9bf53a0ee9c50f8d0461fd5f12b962a8800e95d0bc3ef629edb" + checksum_algorithm = "sha256" +} + +resource "proxmox_virtual_environment_vm" "truenas-scale" { + provider = proxmox.cantor + node_name = var.cantor.node_name + + name = "truenas-scale" + description = "True NAS scale" + tags = ["nas"] + on_boot = true + vm_id = 1000 + + machine = "q35" + scsi_hardware = "virtio-scsi-single" + bios = "ovmf" + + cpu { + cores = 4 + type = "host" + } + + memory { + dedicated = 24576 + } + + network_device { + bridge = "vmbr0" + } + + efi_disk { + datastore_id = "local-zfs" + file_format = "raw" // To support qcow2 format + type = "4m" + } + + disk { + datastore_id = "local-zfs" + file_id = proxmox_virtual_environment_download_file.truenas-scale-23.id + iothread = true + interface = "scsi0" + cache = "writethrough" + discard = "on" + ssd = true + size = 16 + } + + disk { + datastore_id = "local-zfs" + iothread = true + file_format = "raw" + interface = "scsi1" + cache = "writethrough" + discard = "on" + ssd = true + size = 128 + } + + boot_order = ["scsi1", "scsi0"] + + agent { + enabled = true + } + + operating_system { + type = "l26" # Linux Kernel 2.6 - 6.X. + } + + initialization { + dns { + domain = var.vm_dns.domain + servers = var.vm_dns.servers + } + ip_config { + ipv4 { + address = "192.168.1.55/24" + gateway = "192.168.1.1" + } + } + + datastore_id = "local-zfs" + # user_data_file_id = proxmox_virtual_environment_file.cloud-init-work-01.id + } + + hostpci { + device = "hostpci0" + mapping = "ASM1166-0" + pcie = true + rombar = true + xvga = false + } + + // hostpci { + // device = "hostpci1" + // mapping = "ASM1182e-0" + // pcie = true + // rombar = true + // xvga = false + // } + // + // hostpci { + // device = "hostpci2" + // mapping = "ASM1182e-1" + // pcie = true + // rombar = true + // xvga = false + // } + // + // hostpci { + // device = "hostpci3" + // mapping = "ASM1182e-2" + // pcie = true + // rombar = true + // xvga = false + // } + // + // hostpci { + // device = "hostpci4" + // mapping = "I226-V-0" + // pcie = true + // rombar = true + // xvga = false + // } + // + // hostpci { + // device = "hostpci5" + // mapping = "I226-V-1" + // pcie = true + // rombar = true + // xvga = false + // } +} diff --git a/machines/cantor/variables.tf b/machines/cantor/variables.tf new file mode 100644 index 0000000..5004e8c --- /dev/null +++ b/machines/cantor/variables.tf @@ -0,0 +1,51 @@ +variable "cantor" { + description = "Proxmox server configuration for Cantor" + type = object({ + node_name = string + endpoint = string + insecure = bool + }) +} + +variable "cantor_auth" { + description = "Cantor Proxmox server auth" + type = object({ + username = string + api_token = string + }) + sensitive = true +} + +variable "vm_dns" { + description = "DNS config for VMs" + type = object({ + domain = string + servers = list(string) + }) +} + +variable "vm_user" { + description = "VM username" + type = string +} + +variable "vm_password" { + description = "VM password" + type = string + sensitive = true +} + +variable "host_pub-key" { + description = "Host public key" + type = string +} + +variable "k8s-version" { + description = "Kubernetes version" + type = string +} + +variable "cilium-cli-version" { + description = "Cilium CLI version" + type = string +} diff --git a/machines/euclid/images/download.sh b/machines/euclid/images/download.sh index 033372f..d6db208 100755 --- a/machines/euclid/images/download.sh +++ b/machines/euclid/images/download.sh @@ -1,2 +1,4 @@ wget https://github.com/home-assistant/operating-system/releases/download/12.1/haos_ova-12.1.qcow2.xz -xz -d haos_ova-12.1.qcow2.xz \ No newline at end of file +xz -d haos_ova-12.1.qcow2.xz + +wget https://cloud.debian.org/images/cloud/bookworm-backports/20240429-1732/debian-12-backports-generic-amd64-20240429-1732.qcow2 \ No newline at end of file diff --git a/machines/euclid/k8s-config.tf b/machines/euclid/k8s-config.tf index 852d2ec..ded7c3e 100644 --- a/machines/euclid/k8s-config.tf +++ b/machines/euclid/k8s-config.tf @@ -1,4 +1,4 @@ -resource "proxmox_virtual_environment_download_file" "debian_12_generic_image" { +resource "proxmox_virtual_environment_download_file" "debian_12_bookworm" { provider = proxmox.euclid node_name = var.euclid.node_name content_type = "iso" @@ -10,6 +10,66 @@ resource "proxmox_virtual_environment_download_file" "debian_12_generic_image" { checksum_algorithm = "sha512" } +resource "proxmox_virtual_environment_download_file" "debian_12_bpo" { + provider = proxmox.euclid + node_name = var.euclid.node_name + content_type = "iso" + datastore_id = "local" + + file_name = "debian-12-backports-generic-amd64-20240429-1732.img" + url = "https://cloud.debian.org/images/cloud/bookworm-backports/20240429-1732/debian-12-backports-generic-amd64-20240429-1732.qcow2" +# checksum = "b679398972ba45a60574d9202c4f97ea647dd3577e857407138b73b71a3c3c039804e40aac2f877f3969676b6c8a1ebdb4f2d67a4efa6301c21e349e37d43ef5" +# checksum_algorithm = "sha512" +} + +resource "proxmox_virtual_environment_download_file" "ubuntu_jammy_cloud_amd64" { + provider = proxmox.euclid + node_name = var.euclid.node_name + content_type = "iso" + datastore_id = "local" + + file_name = "jammy-server-cloudimg-amd64.img" + url = "https://cloud-images.ubuntu.com/jammy/20240514/jammy-server-cloudimg-amd64.img" + checksum = "1718f177dde4c461148ab7dcbdcf2f410c1f5daa694567f6a8bbb239d864b525" + checksum_algorithm = "sha256" +} + +resource "proxmox_virtual_environment_download_file" "ubuntu_mantic-cloud-amd64" { + provider = proxmox.euclid + node_name = var.euclid.node_name + content_type = "iso" + datastore_id = "local" + + file_name = "mantic-server-cloudimg-amd64.img" + url = "https://cloud-images.ubuntu.com/mantic/20240514/mantic-server-cloudimg-amd64.img" +# checksum = "1718f177dde4c461148ab7dcbdcf2f410c1f5daa694567f6a8bbb239d864b525" +# checksum_algorithm = "sha256" +} + +resource "proxmox_virtual_environment_download_file" "ubuntu_noble-cloud-amd64" { + provider = proxmox.euclid + node_name = var.euclid.node_name + content_type = "iso" + datastore_id = "local" + + file_name = "noble-server-cloudimg-amd64.img" + url = "https://cloud-images.ubuntu.com/noble/20240505/noble-server-cloudimg-amd64.img" + # checksum = "1718f177dde4c461148ab7dcbdcf2f410c1f5daa694567f6a8bbb239d864b525" + # checksum_algorithm = "sha256" +} + +#resource "proxmox_virtual_environment_file" "debian_12_backports_image" { +# provider = proxmox.euclid +# node_name = var.euclid.node_name +# content_type = "iso" +# datastore_id = "local" +# +# source_file { +# path = "images/debian-12-backports-generic-amd64-20240429-1732.qcow2" +# file_name = "debian-12-backports-generic-amd64-20240429-1732.img" +# } +#} + # Make sure the "Snippets" content type is enabled on the target datastore in Proxmox before applying the configuration below. # https://github.com/bpg/terraform-provider-proxmox/blob/main/docs/guides/cloud-init.md resource "proxmox_virtual_environment_file" "cloud-init-ctrl-01" { diff --git a/machines/euclid/k8s-vm-control-plane.tf b/machines/euclid/k8s-vm-control-plane.tf index 29bfdc4..bdc6d64 100644 --- a/machines/euclid/k8s-vm-control-plane.tf +++ b/machines/euclid/k8s-vm-control-plane.tf @@ -34,7 +34,7 @@ resource "proxmox_virtual_environment_vm" "k8s-ctrl-01" { disk { datastore_id = "local-zfs" - file_id = proxmox_virtual_environment_download_file.debian_12_generic_image.id + file_id = proxmox_virtual_environment_download_file.debian_12_bpo.id interface = "scsi0" cache = "writethrough" discard = "on" diff --git a/machines/euclid/k8s-vm-worker.tf b/machines/euclid/k8s-vm-worker.tf index 3f1ffbd..dc5961a 100644 --- a/machines/euclid/k8s-vm-worker.tf +++ b/machines/euclid/k8s-vm-worker.tf @@ -34,7 +34,7 @@ resource "proxmox_virtual_environment_vm" "k8s-work-01" { disk { datastore_id = "local-zfs" - file_id = proxmox_virtual_environment_download_file.debian_12_generic_image.id + file_id = proxmox_virtual_environment_download_file.debian_12_bpo.id iothread = true interface = "scsi0" cache = "writethrough" @@ -54,16 +54,16 @@ resource "proxmox_virtual_environment_vm" "k8s-work-01" { size = 64 } - disk { - datastore_id = "local-zfs" - iothread = true - file_format = "raw" - interface = "scsi2" - cache = "writethrough" - discard = "on" - ssd = true - size = 512 - } +# disk { +# datastore_id = "local-zfs" +# iothread = true +# file_format = "raw" +# interface = "scsi2" +# cache = "writethrough" +# discard = "on" +# ssd = true +# size = 512 +# } boot_order = ["scsi0"] diff --git a/machines/euclid/main.tf b/machines/euclid/main.tf index c03a3d1..38f0725 100644 --- a/machines/euclid/main.tf +++ b/machines/euclid/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { proxmox = { source = "bpg/proxmox" - version = "0.50.0" + version = "0.57.0" } } } diff --git a/tofu/home-assistant/images.tf b/tofu/home-assistant/images.tf new file mode 100644 index 0000000..05141e8 --- /dev/null +++ b/tofu/home-assistant/images.tf @@ -0,0 +1,11 @@ +resource "proxmox_virtual_environment_file" "haos_generic_image" { + provider = proxmox.euclid + node_name = var.euclid.node_name + content_type = "iso" + datastore_id = "local" + + source_file { + path = "images/haos_ova-12.3.qcow2" + file_name = "haos_ova-12.3.img" + } +} diff --git a/tofu/home-assistant/images/download.sh b/tofu/home-assistant/images/download.sh new file mode 100755 index 0000000..3a0aaec --- /dev/null +++ b/tofu/home-assistant/images/download.sh @@ -0,0 +1,2 @@ +wget https://github.com/home-assistant/operating-system/releases/download/12.3/haos_ova-12.3.qcow2.xz +xz -d haos_ova-12.3.qcow2.xz diff --git a/tofu/home-assistant/main.tf b/tofu/home-assistant/main.tf new file mode 100644 index 0000000..3146601 --- /dev/null +++ b/tofu/home-assistant/main.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = ">= 0.57.0" + } + } +} + diff --git a/tofu/home-assistant/pve_euclid.tf b/tofu/home-assistant/pve_euclid.tf new file mode 120000 index 0000000..fcf4531 --- /dev/null +++ b/tofu/home-assistant/pve_euclid.tf @@ -0,0 +1 @@ +../machines/euclid.tf \ No newline at end of file diff --git a/tofu/home-assistant/vm-haos.tf b/tofu/home-assistant/vm-haos.tf new file mode 100644 index 0000000..246da0b --- /dev/null +++ b/tofu/home-assistant/vm-haos.tf @@ -0,0 +1,57 @@ +resource "proxmox_virtual_environment_vm" "home_assistant" { + provider = proxmox.euclid + node_name = var.euclid.node_name + + name = "Home-Assistant" + description = "Managed by OpenTofu" + tags = ["home-assistant"] + on_boot = true + bios = "ovmf" + scsi_hardware = "virtio-scsi-single" + + vm_id = 1000 + + tablet_device = false + + cpu { + cores = 2 + type = "host" + } + + memory { + dedicated = 4096 + } + + network_device { + bridge = "vmbr0" + mac_address = "BC:24:11:50:A6:33" + } + + agent { + enabled = true + } + + efi_disk { + datastore_id = "local-zfs" + file_format = "raw" // To support qcow2 format + type = "4m" + } + + disk { + datastore_id = "local-zfs" + file_id = proxmox_virtual_environment_file.haos_generic_image.id + interface = "scsi0" + cache = "writethrough" + discard = "on" + ssd = true + size = 64 + } + + operating_system { + type = "l26" # Linux Kernel 2.6 - 5.X. + } + + lifecycle { + prevent_destroy = true + } +} diff --git a/tofu/k8s/bootstrap/cilium-install.yaml b/tofu/k8s/bootstrap/cilium-install.yaml new file mode 100644 index 0000000..29618a3 --- /dev/null +++ b/tofu/k8s/bootstrap/cilium-install.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-install +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: cilium-install + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cilium-install + namespace: kube-system +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: cilium-install + namespace: kube-system +spec: + backoffLimit: 10 + template: + metadata: + labels: + app: cilium-install + spec: + restartPolicy: OnFailure + tolerations: + - operator: Exists + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - effect: PreferNoSchedule + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoExecute + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: PreferNoSchedule + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + serviceAccountName: cilium-install + hostNetwork: true + containers: + - name: cilium-install + image: quay.io/cilium/cilium-cli-ci:latest + env: + - name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: KUBERNETES_SERVICE_PORT + value: "6443" + command: + - cilium + - install + - --version=v1.15.5 # renovate: github-releases=cilium/cilium + - --helm-set=ipam.mode=kubernetes + - --set + - kubeProxyReplacement=true + - --helm-set=securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID} + - --helm-set=securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE} + - --helm-set=cgroup.autoMount.enabled=false + - --helm-set=cgroup.hostRoot=/sys/fs/cgroup + - --helm-set=k8sServiceHost=localhost + - --helm-set=k8sServicePort=7445 diff --git a/tofu/k8s/cloud-init/k8s-common.yaml.tftpl b/tofu/k8s/cloud-init/k8s-common.yaml.tftpl new file mode 100644 index 0000000..aaf3477 --- /dev/null +++ b/tofu/k8s/cloud-init/k8s-common.yaml.tftpl @@ -0,0 +1,70 @@ +users: + - name: ${username} + passwd: ${password} + lock_passwd: false + groups: [ adm, cdrom, dip, plugdev, lxd, sudo ] + shell: /bin/bash + ssh_authorized_keys: + - ${pub-key} + #sudo: ALL=(ALL) NOPASSWD:ALL + +hostname: ${hostname} +package_update: true +package_upgrade: true +timezone: Europe/Oslo + +write_files: + - path: /etc/ssh/sshd_config.d/01-harden-ssh.conf + content: | + PermitRootLogin no + PasswordAuthentication no + ChallengeResponseAuthentication no + UsePAM no + + - path: /etc/modules-load.d/k8s.conf + content: | + overlay + br_netfilter + + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + net.ipv4.ip_forward = 1 + +packages: + - qemu-guest-agent + - net-tools + - vim + - apt-transport-https + - ca-certificates + - curl + - gpg + - open-iscsi + - jq + +power_state: + delay: now + mode: reboot + message: Rebooting after cloud-init completion + condition: true + +runcmd: + - systemctl enable qemu-guest-agent + - localectl set-locale LANG=en_US.UTF-8 + - sed -i '/Components/s/$/ non-free non-free-firmware/' /etc/apt/sources.list.d/debian.sources + - curl -fsSL https://pkgs.k8s.io/core:/stable:/v${k8s-version}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + - echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${k8s-version}/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list + - apt update + - apt install -y nfs-common + - apt install -y firmware-misc-nonfree intel-media-va-driver-non-free intel-gpu-tools + - apt install -y kubelet kubeadm kubectl + - apt-mark hold kubelet kubeadm kubectl + - apt install -y runc containerd + - containerd config default | tee /etc/containerd/config.toml + - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml + - modprobe overlay + - modprobe br_netfilter + - sysctl --system + - systemctl restart containerd + - ${kubeadm-cmd} \ No newline at end of file diff --git a/tofu/k8s/cloud-init/k8s-control-plane.yaml.tftpl b/tofu/k8s/cloud-init/k8s-control-plane.yaml.tftpl new file mode 100644 index 0000000..0f0817d --- /dev/null +++ b/tofu/k8s/cloud-init/k8s-control-plane.yaml.tftpl @@ -0,0 +1,9 @@ +#cloud-config +${common-config} + - mkdir -p /home/${username}/.kube + - cp /etc/kubernetes/admin.conf /home/${username}/.kube/config + - chown -R ${username}:${username} /home/${username}/.kube + - curl -sfLO https://github.com/cilium/cilium-cli/releases/download/v${cilium-cli-version}/cilium-linux-amd64.tar.gz + - tar xzvfC cilium-linux-amd64.tar.gz /usr/local/bin + - rm cilium-linux-amd64.tar.gz + - ${cilium-cli-cmd} diff --git a/tofu/k8s/cloud-init/k8s-worker.yaml.tftpl b/tofu/k8s/cloud-init/k8s-worker.yaml.tftpl new file mode 100644 index 0000000..23bcdb2 --- /dev/null +++ b/tofu/k8s/cloud-init/k8s-worker.yaml.tftpl @@ -0,0 +1,2 @@ +#cloud-config +${common-config} \ No newline at end of file diff --git a/tofu/k8s/config.tf b/tofu/k8s/config.tf new file mode 100644 index 0000000..c59504d --- /dev/null +++ b/tofu/k8s/config.tf @@ -0,0 +1,132 @@ +resource "talos_machine_secrets" "machine_secrets" { + talos_version = "v1.7" +} + +data "talos_client_configuration" "talosconfig" { + cluster_name = var.cluster.name + client_configuration = talos_machine_secrets.machine_secrets.client_configuration + endpoints = [for k, v in var.node_data.controlplanes : v.ip] +} + +data "talos_machine_configuration" "control-plane" { + for_each = var.node_data.controlplanes + cluster_name = var.cluster.name + cluster_endpoint = var.cluster.endpoint + machine_type = "controlplane" + machine_secrets = talos_machine_secrets.machine_secrets.machine_secrets + talos_version = "v1.7" + config_patches = [ + templatefile("talos/control-plane.yaml.tftpl", { + hostname = each.key + }) + ] +} + + +resource "proxmox_virtual_environment_file" "controlplane-config" { + provider = proxmox.abel + for_each = var.node_data.controlplanes + + node_name = each.value.host_node + content_type = "snippets" + datastore_id = "local" + + source_raw { + data = data.talos_machine_configuration.control-plane[each.key].machine_configuration + file_name = "talos-${each.key}.cloud-config.yaml" + } +} + +resource "talos_machine_configuration_apply" "ctrl_config_apply" { + depends_on = [proxmox_virtual_environment_vm.controlplane] + for_each = var.node_data.controlplanes + node = each.value.ip + client_configuration = talos_machine_secrets.machine_secrets.client_configuration + machine_configuration_input = data.talos_machine_configuration.control-plane[each.key].machine_configuration +} + +data "talos_machine_configuration" "worker" { + for_each = var.node_data.workers + cluster_name = var.cluster.name + cluster_endpoint = var.cluster.endpoint + machine_type = "worker" + machine_secrets = talos_machine_secrets.machine_secrets.machine_secrets + talos_version = "v1.7" + config_patches = [ + templatefile("talos/worker.yaml.tftpl", { + hostname = each.key + }) + ] +} + +resource "proxmox_virtual_environment_file" "worker-config" { + provider = proxmox.abel + for_each = var.node_data.workers + + node_name = each.value.host_node + content_type = "snippets" + datastore_id = "local" + + source_raw { + data = data.talos_machine_configuration.worker[each.key].machine_configuration + file_name = "talos-${each.key}.cloud-config.yaml" + } +} + +resource "talos_machine_configuration_apply" "worker_config_apply" { + depends_on = [proxmox_virtual_environment_vm.workers] + for_each = var.node_data.workers + node = each.value.ip + client_configuration = talos_machine_secrets.machine_secrets.client_configuration + machine_configuration_input = data.talos_machine_configuration.worker[each.key].machine_configuration + config_patches = [ + templatefile("talos/worker.yaml.tftpl", { + hostname = each.key + }) + ] +} + +resource "talos_machine_bootstrap" "bootstrap" { + depends_on = [talos_machine_configuration_apply.ctrl_config_apply] + client_configuration = talos_machine_secrets.machine_secrets.client_configuration + node = [for k, v in var.node_data.controlplanes : v.ip][0] +} + +data "talos_cluster_health" "health" { + depends_on = [talos_machine_configuration_apply.ctrl_config_apply] + client_configuration = data.talos_client_configuration.talosconfig.client_configuration + control_plane_nodes = [for k, v in var.node_data.controlplanes : v.ip] + worker_nodes = [for k, v in var.node_data.workers : v.ip] + endpoints = data.talos_client_configuration.talosconfig.endpoints + timeouts = { + read = "10m" + } +} + +data "talos_cluster_kubeconfig" "kubeconfig" { + depends_on = [talos_machine_bootstrap.bootstrap, data.talos_cluster_health.health] + client_configuration = talos_machine_secrets.machine_secrets.client_configuration + node = [for k, v in var.node_data.controlplanes : v.ip][0] +} + +output "talosconfig" { + value = data.talos_client_configuration.talosconfig.talos_config + sensitive = true +} + +output "kubeconfig" { + value = data.talos_cluster_kubeconfig.kubeconfig.kubeconfig_raw + sensitive = true +} + +resource "local_file" "taloc-client-config" { + content = data.talos_client_configuration.talosconfig.talos_config + filename = "output/talos-config.yaml" + file_permission = "0600" +} + +resource "local_file" "kube-config" { + content = data.talos_cluster_kubeconfig.kubeconfig.kubeconfig_raw + filename = "output/kube-config.yaml" + file_permission = "0600" +} diff --git a/tofu/k8s/images.tf b/tofu/k8s/images.tf new file mode 100644 index 0000000..9dffa2e --- /dev/null +++ b/tofu/k8s/images.tf @@ -0,0 +1,20 @@ +locals { + talos = { + version = "v1.7.4" # renovate: github-releases=siderolabs/talos + checksum = "26e23f1bf44eecb0232d0aa221223b44f4e40806b7d12cf1a72626927da9a8a4" + } +} + +resource "proxmox_virtual_environment_file" "talos_nocloud_image" { + provider = proxmox.abel + for_each = toset(var.host_machines) + + node_name = each.key + content_type = "iso" + datastore_id = "local" + + source_file { + path = "images/talos-${local.talos.version}-nocloud-amd64.raw" + file_name = "talos-${local.talos.version}-nocloud-amd64.img" + } +} \ No newline at end of file diff --git a/tofu/k8s/images/download.sh b/tofu/k8s/images/download.sh new file mode 100755 index 0000000..795ed85 --- /dev/null +++ b/tofu/k8s/images/download.sh @@ -0,0 +1,4 @@ +#wget https://github.com/siderolabs/talos/releases/download/v1.7.4/nocloud-amd64.raw.xz +wget https://factory.talos.dev/image/dcac6b92c17d1d8947a0cee5e0e6b6904089aa878c70d66196bb1138dbd05d1a/v1.7.4/nocloud-amd64.raw.xz +xz -d nocloud-amd64.raw.xz +mv nocloud-amd64.raw talos-v1.7.4-nocloud-amd64.raw diff --git a/tofu/k8s/main.tf b/tofu/k8s/main.tf new file mode 100644 index 0000000..87129a4 --- /dev/null +++ b/tofu/k8s/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = ">= 0.57.0" + } + talos = { + source = "siderolabs/talos" + version = "0.5.0" + } + } +} + diff --git a/tofu/k8s/pve_abel.tf b/tofu/k8s/pve_abel.tf new file mode 120000 index 0000000..cae63c0 --- /dev/null +++ b/tofu/k8s/pve_abel.tf @@ -0,0 +1 @@ +../machines/abel.tf \ No newline at end of file diff --git a/tofu/k8s/pve_cantor.tf b/tofu/k8s/pve_cantor.tf new file mode 120000 index 0000000..483d946 --- /dev/null +++ b/tofu/k8s/pve_cantor.tf @@ -0,0 +1 @@ +../machines/cantor.tf \ No newline at end of file diff --git a/tofu/k8s/pve_euclid.tf b/tofu/k8s/pve_euclid.tf new file mode 120000 index 0000000..fcf4531 --- /dev/null +++ b/tofu/k8s/pve_euclid.tf @@ -0,0 +1 @@ +../machines/euclid.tf \ No newline at end of file diff --git a/tofu/k8s/rbac.tf b/tofu/k8s/rbac.tf new file mode 100644 index 0000000..72e1c38 --- /dev/null +++ b/tofu/k8s/rbac.tf @@ -0,0 +1,21 @@ +resource "proxmox_virtual_environment_role" "csi" { + provider = proxmox.abel + role_id = "csi" + privileges = [ + "VM.Audit", + "VM.Config.Disk", + "Datastore.Allocate", + "Datastore.AllocateSpace", + "Datastore.Audit" + ] +} + +resource "proxmox_virtual_environment_user" "kubernetes-csi" { + provider = proxmox.abel + user_id = "kubernetes-csi@pve" + acl { + path = "/" + propagate = true + role_id = proxmox_virtual_environment_role.csi.role_id + } +} \ No newline at end of file diff --git a/tofu/k8s/talos/control-plane.yaml.tftpl b/tofu/k8s/talos/control-plane.yaml.tftpl new file mode 100644 index 0000000..9b029df --- /dev/null +++ b/tofu/k8s/talos/control-plane.yaml.tftpl @@ -0,0 +1,14 @@ +machine: + network: + hostname: ${hostname} +cluster: + allowSchedulingOnControlPlanes: true + network: + cni: + name: none + proxy: + disabled: true + inlineManifests: + - name: cilium-install + contents: | + ${indent(8, file("bootstrap/cilium-install.yaml"))} diff --git a/tofu/k8s/talos/machine-config.yaml b/tofu/k8s/talos/machine-config.yaml new file mode 100644 index 0000000..9095e46 --- /dev/null +++ b/tofu/k8s/talos/machine-config.yaml @@ -0,0 +1,7 @@ +machine: + customization: + systemExtensions: + officialExtensions: + - siderolabs/i915-ucode + - siderolabs/intel-ucode + - siderolabs/qemu-guest-agent \ No newline at end of file diff --git a/tofu/k8s/talos/worker.yaml.tftpl b/tofu/k8s/talos/worker.yaml.tftpl new file mode 100644 index 0000000..faebaf1 --- /dev/null +++ b/tofu/k8s/talos/worker.yaml.tftpl @@ -0,0 +1,3 @@ +machine: + network: + hostname: ${hostname} diff --git a/tofu/k8s/variables.tf b/tofu/k8s/variables.tf new file mode 100644 index 0000000..60bd93a --- /dev/null +++ b/tofu/k8s/variables.tf @@ -0,0 +1,34 @@ +variable "cluster" { + type = object({ + name = string + endpoint = string + talos_version = string + }) +} + +variable "host_machines" { + type = list(string) +} + +variable "node_data" { + description = "A map of node data" + type = object({ + controlplanes = map(object({ + ip = string + mac_address = string + host_node = string + vm_id = number + cpu = number + ram_dedicated = number + igpu = optional(bool, false) + })) + workers = map(object({ + ip = string + mac_address = string + host_node = string + vm_id = number + cpu = number + ram_dedicated = number + })) + }) +} diff --git a/tofu/k8s/vm-controlplane.tf b/tofu/k8s/vm-controlplane.tf new file mode 100644 index 0000000..0d5f25b --- /dev/null +++ b/tofu/k8s/vm-controlplane.tf @@ -0,0 +1,80 @@ +resource "proxmox_virtual_environment_vm" "controlplane" { + provider = proxmox.abel + + for_each = var.node_data.controlplanes + + node_name = each.value.host_node + + name = each.key + description = "Talos Kubernetes Control Plane" + tags = ["k8s", "control-plane"] + on_boot = true + vm_id = each.value.vm_id + + machine = "q35" + scsi_hardware = "virtio-scsi-single" + bios = "seabios" + + agent { + enabled = true + } + + cpu { + cores = each.value.cpu + type = "host" + } + + memory { + dedicated = each.value.ram_dedicated + } + + network_device { + bridge = "vmbr0" + mac_address = each.value.mac_address + } + + disk { + datastore_id = "local-zfs" + interface = "scsi0" + iothread = true + cache = "writethrough" + discard = "on" + ssd = true + # file_id = proxmox_virtual_environment_download_file.talos_nocloud_image.id + file_id = proxmox_virtual_environment_file.talos_nocloud_image[each.value.host_node].id + file_format = "raw" + size = 20 + } + + boot_order = ["scsi0"] + + operating_system { + type = "l26" # Linux Kernel 2.6 - 6.X. + } + + initialization { + datastore_id = "local-zfs" +# meta_data_file_id = proxmox_virtual_environment_file.controlplane-config[each.key].id + ip_config { + ipv4 { + address = "${each.value.ip}/24" + gateway = "192.168.1.1" + } + ipv6 { + address = "dhcp" + } + } + } + + dynamic "hostpci" { + for_each = each.value.igpu ? [1] : [] + content { + # Passthrough iGPU + device = "hostpci0" + mapping = "iGPU" + pcie = true + rombar = true + xvga = false + } + } +} diff --git a/tofu/k8s/vm-workers.tf b/tofu/k8s/vm-workers.tf new file mode 100644 index 0000000..b57b601 --- /dev/null +++ b/tofu/k8s/vm-workers.tf @@ -0,0 +1,78 @@ +resource "proxmox_virtual_environment_vm" "workers" { + provider = proxmox.abel + + for_each = var.node_data.workers + + node_name = each.value.host_node + + name = each.key + description = "Talos Kubernetes Worker" + tags = ["k8s", "worker"] + on_boot = true + vm_id = each.value.vm_id + + machine = "q35" + scsi_hardware = "virtio-scsi-single" + bios = "seabios" + + agent { + enabled = true + } + + cpu { + cores = each.value.cpu + type = "host" + } + + memory { + dedicated = each.value.ram_dedicated + } + + network_device { + bridge = "vmbr0" + mac_address = each.value.mac_address + } + + disk { + datastore_id = "local-zfs" + interface = "scsi0" + iothread = true + cache = "writethrough" + discard = "on" + ssd = true + # file_id = proxmox_virtual_environment_download_file.talos_nocloud_image.id + file_id = proxmox_virtual_environment_file.talos_nocloud_image[each.value.host_node].id + file_format = "raw" + size = 20 + } + + boot_order = ["scsi0"] + + operating_system { + type = "l26" # Linux Kernel 2.6 - 6.X. + } + + initialization { + datastore_id = "local-zfs" +# meta_data_file_id = proxmox_virtual_environment_file.worker-config[each.key].id + ip_config { + ipv4 { + address = "${each.value.ip}/24" + gateway = "192.168.1.1" + } + ipv6 { + address = "dhcp" + } + } + } + +# hostpci { +# # Passthrough iGPU +# device = "hostpci0" +# #id = "0000:00:02" +# mapping = "iGPU" +# pcie = true +# rombar = true +# xvga = false +# } +} diff --git a/tofu/machines/abel.tf b/tofu/machines/abel.tf new file mode 100644 index 0000000..6f52058 --- /dev/null +++ b/tofu/machines/abel.tf @@ -0,0 +1,24 @@ +variable "abel" { + description = "Abel Proxmox server auth" + type = object({ + node_name = string + username = string + api_token = string + }) + sensitive = true +} + +provider "proxmox" { + alias = "abel" + #endpoint = "https://proxmox.abel.stonegarden.dev" + endpoint = "https://192.168.1.62:8006" + insecure = true + + api_token = var.abel.api_token + ssh { + agent = true + username = var.abel.username + } + + tmp_dir = "/var/tmp" +} diff --git a/tofu/machines/cantor.tf b/tofu/machines/cantor.tf new file mode 100644 index 0000000..57b2931 --- /dev/null +++ b/tofu/machines/cantor.tf @@ -0,0 +1,24 @@ +variable "cantor" { + description = "Cantor Proxmox server auth" + type = object({ + node_name = string + username = string + api_token = string + }) + sensitive = true +} + +provider "proxmox" { + alias = "cantor" + #endpoint = "https://proxmox.cantor.stonegarden.dev" + endpoint = "https://192.168.1.52:8006" + insecure = true + + api_token = var.cantor.api_token + ssh { + agent = true + username = var.cantor.username + } + + tmp_dir = "/var/tmp" +} diff --git a/tofu/machines/euclid.tf b/tofu/machines/euclid.tf new file mode 100644 index 0000000..76b1e73 --- /dev/null +++ b/tofu/machines/euclid.tf @@ -0,0 +1,24 @@ +variable "euclid" { + description = "Euclid Proxmox server auth" + type = object({ + node_name = string + username = string + api_token = string + }) + sensitive = true +} + +provider "proxmox" { + alias = "euclid" + #endpoint = "https://proxmox.euclid.stonegarden.dev" + endpoint = "https://192.168.1.42:8006" + insecure = true + + api_token = var.euclid.api_token + ssh { + agent = true + username = var.euclid.username + } + + tmp_dir = "/var/tmp" +}