mirror of
https://github.com/outbackdingo/homelab-v.git
synced 2026-08-25 14:53:19 +00:00
feat(tofu): refactor tofu-code
Tidying code and clarifying some sections Signed-off-by: Vegard Hagen <[email protected]>
This commit is contained in:
Vegard Hagen
+2
-1
@@ -16,7 +16,8 @@
|
||||
"\\.tf$",
|
||||
"\\.tftpl$",
|
||||
"\\.yaml$",
|
||||
"\\.sh$"
|
||||
"\\.sh$",
|
||||
"\\.tfvars"
|
||||
],
|
||||
"matchStrings": [
|
||||
"(?<currentValue>[\\w+\\.\\-]*)['\",;]*\\s*#\\s?renovate: (?<datasource>\\S+)=(?<depName>\\S+)\\s?(registry=(?<registryUrl>\\S+))?\\s?(versioning=(?<versioning>\\S+))?"
|
||||
|
||||
+30
-20
@@ -1,11 +1,16 @@
|
||||
# Kubernetes Tofu
|
||||
|
||||
Read [Talos Kubernetes on Proxmox using OpenTofu](https://blog.stonegarden.dev/articles/2024/08/talos-proxmox-tofu/) for
|
||||
a more thorough explanation of how everything works.
|
||||
|
||||
## Install pre-requisites
|
||||
|
||||
1. [tofu](https://opentofu.org/docs/intro/install/)
|
||||
1. [talosctl](https://www.talos.dev/v1.9/talos-guides/install/talosctl/)
|
||||
1. [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
|
||||
2. [talosctl](https://www.talos.dev/v1.9/talos-guides/install/talosctl/)
|
||||
3. [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
|
||||
|
||||
## Initialize tofu
|
||||
|
||||
```shell
|
||||
tofu init
|
||||
```
|
||||
@@ -21,11 +26,12 @@ export TF_VAR_proxmox_api_token="<YOUR_API_TOKEN>"
|
||||
### Optional External Secrets Manager / Other methods
|
||||
|
||||
**Bitwarden Secrets Manager** - Name your secret TF_VAR_proxmox_api_token in bws.
|
||||
|
||||
```shell
|
||||
bws run -- tofu ...
|
||||
```
|
||||
Note: By default the shell is sh. Change with --shell if required.
|
||||
|
||||
Note: By default, the shell is sh. Change with --shell if required.
|
||||
|
||||
## Sealed-secrets
|
||||
|
||||
@@ -52,29 +58,33 @@ tofu output -raw talos_config
|
||||
```
|
||||
|
||||
## Upgrading Talos and Kubernetes
|
||||
[Upgrade](https://blog.stonegarden.dev/articles/2024/08/talos-proxmox-tofu/#upgrading-the-cluster) talos nodes one by one.
|
||||
|
||||
[Upgrade](https://blog.stonegarden.dev/articles/2024/08/talos-proxmox-tofu/#upgrading-the-cluster) talos nodes one by
|
||||
one.
|
||||
|
||||
1. Set talos_image.auto.tfvars -> image -> update_version to the required update version.
|
||||
1. Set talos_cluster.auto.tfvars -> talos_cluster_config -> kubernetes_version to the required kubernetes version.
|
||||
1. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_1 -> update = true and run tofu apply.
|
||||
1. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_2 -> update = true, leave the previous nodes update = true and run tofu apply.
|
||||
1. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_3 -> update = true, leave the previous nodes update = true and run tofu apply.
|
||||
1. ...
|
||||
1. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_n -> update = true, leave the previous nodes update = true and run tofu apply.
|
||||
1. After upgrading all nodes, Set talos_image.auto.tfvars -> image -> version to match the update version and set update = false for all nodes.
|
||||
2. Set talos_cluster.auto.tfvars -> talos_cluster_config -> kubernetes_version to the required kubernetes version.
|
||||
3. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_1 -> update = true and run tofu apply.
|
||||
4. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_2 -> update = true, leave the previous nodes update = true and
|
||||
run tofu apply.
|
||||
5. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_3 -> update = true, leave the previous nodes update = true and
|
||||
run tofu apply.
|
||||
6. ...
|
||||
7. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_n -> update = true, leave the previous nodes update = true and
|
||||
run tofu apply.
|
||||
8. After upgrading all nodes, Set talos_image.auto.tfvars -> image -> version to match the update version and set
|
||||
update = false for all nodes.
|
||||
|
||||
## Upgrading Talos Schematic
|
||||
|
||||
1. Create a new schematic file.
|
||||
1. Same process as above instead of image.version and image.update_version, change image.schematic and image.update_schematic, in talos_image.auto.tfvars.
|
||||
2. Same process as above instead of `image.version` and `image.update_version`, change `image.schematic` and
|
||||
`image.update_schematic`, in `talos_image.auto.tfvars`.
|
||||
|
||||
## Upgrading Kubernetes Only
|
||||
## Reuse machine secrets
|
||||
|
||||
Dry Run
|
||||
```shell
|
||||
sh upgrade-k8s.sh $CONTROLPLANE_NODE_IP --dry-run # For testing
|
||||
```
|
||||
|
||||
Upgrade
|
||||
```shell
|
||||
sh upgrade-k8s.sh $CONTROLPLANE_NODE_IP
|
||||
tofu state rm module.talos.talos_machine_secrets.this
|
||||
tofu import module.talos.talos_machine_secrets.this output/talos-machine-secrets.yaml
|
||||
tofu apply --refresh=false
|
||||
```
|
||||
@@ -1,9 +1,12 @@
|
||||
## GCS Remote
|
||||
1. Create a [Service Account](https://cloud.google.com/iam/docs/service-accounts-create) named tofu (after enabling the IAM API if needed). Leave the permissions blank.
|
||||
1. Create and download the [service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating).
|
||||
1. Create a GCS bucket for tofu state with public access prevention and versioning as necessary.
|
||||
1. In the permissions tab of the bucket, give **Storage Object Admin** access to the service account.
|
||||
1. Copy backend.tf.sample to backend.tf and make necessary changes.
|
||||
|
||||
1. Create a [Service Account](https://cloud.google.com/iam/docs/service-accounts-create) named tofu (after enabling the
|
||||
IAM API if needed). Leave the permissions blank.
|
||||
2. Create and download the [service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating).
|
||||
3. Create a GCS bucket for tofu state with public access prevention and versioning as necessary.
|
||||
4. In the permissions tab of the bucket, give **Storage Object Admin** access to the service account.
|
||||
5. Copy backend.tf.sample to backend.tf and make necessary changes.
|
||||
|
||||
```shell
|
||||
cp remote_backend.tf.sample remote_backend.tf
|
||||
```
|
||||
@@ -16,7 +19,7 @@ Generate the encryption key
|
||||
python3 -c 'import os;import base64;print(base64.b64encode(os.urandom(32)).decode("utf-8"))'
|
||||
```
|
||||
|
||||
 `Without the encryption key, your state would not be recoverable. Store in a password manager, if not using any kms like bws.`
|
||||
`Without the encryption key, your state would not be recoverable. Store in a password manager, if not using any kms like bws.`
|
||||
|
||||
### Environment variables
|
||||
|
||||
@@ -29,10 +32,11 @@ Run tofu init / plan / apply as usual.
|
||||
|
||||
### Bitwarden Secrets Manager
|
||||
|
||||
Store the downloaded key contents and generated encryption key into GOOGLE_CREDENTIALS and GOOGLE_ENCRYPTION_KEY respectively in bws.
|
||||
Store the downloaded key contents and generated encryption key into GOOGLE_CREDENTIALS and GOOGLE_ENCRYPTION_KEY
|
||||
respectively in bws.
|
||||
|
||||
Run bws run -- tofu init / plan / apply as usual.
|
||||
|
||||
### Beta Notice
|
||||
|
||||
 `Please treat this as beta and only use for air-gapped installations as of now. Will remove the beta tag after testing it in due course.`
|
||||
`Please treat this as beta and only use for air-gapped installations as of now. Will remove the beta tag after testing it in due course.`
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
|
||||
|
||||
```shell
|
||||
pvesm alloc local-zfs 8000 vm-8000-app-config 1G
|
||||
```
|
||||
|
||||
@@ -1,4 +1,34 @@
|
||||
resource "local_file" "machine_configs" {
|
||||
resource "local_file" "talos_machine_secrets" {
|
||||
content = yamlencode({
|
||||
cluster = module.talos.machine_secrets.cluster
|
||||
secrets = module.talos.machine_secrets.secrets
|
||||
trustdinfo = module.talos.machine_secrets.trustdinfo
|
||||
certs = {
|
||||
etcd = {
|
||||
crt = module.talos.machine_secrets.certs.etcd.cert
|
||||
key = module.talos.machine_secrets.certs.etcd.key
|
||||
}
|
||||
k8s = {
|
||||
crt = module.talos.machine_secrets.certs.k8s.cert
|
||||
key = module.talos.machine_secrets.certs.k8s.key
|
||||
}
|
||||
k8saggregator = {
|
||||
crt = module.talos.machine_secrets.certs.k8s_aggregator.cert
|
||||
key = module.talos.machine_secrets.certs.k8s_aggregator.key
|
||||
}
|
||||
k8sserviceaccount = {
|
||||
key = module.talos.machine_secrets.certs.k8s_serviceaccount.key
|
||||
}
|
||||
os = {
|
||||
crt = module.talos.machine_secrets.certs.os.cert
|
||||
key = module.talos.machine_secrets.certs.os.key
|
||||
}
|
||||
}
|
||||
})
|
||||
filename = "output/talos-machine-secrets.yaml"
|
||||
}
|
||||
|
||||
resource "local_file" "talos_machine_configs" {
|
||||
for_each = module.talos.machine_config
|
||||
content = each.value.machine_configuration
|
||||
filename = "output/talos-machine-config-${each.key}.yaml"
|
||||
|
||||
@@ -1,13 +1,20 @@
|
||||
locals {
|
||||
first_control_plane_node_ip = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"][0]
|
||||
kubernetes_endpoint = coalesce(var.cluster.vip, local.first_control_plane_node_ip)
|
||||
extra_manifests = concat(var.cluster.extra_manifests, [
|
||||
"https://github.com/kubernetes-sigs/gateway-api/releases/download/${var.cluster.gateway_api_version}/standard-install.yaml",
|
||||
"https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/${var.cluster.gateway_api_version}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml"
|
||||
])
|
||||
}
|
||||
|
||||
resource "talos_machine_secrets" "this" {
|
||||
talos_version = var.cluster.talos_machine_config_version != null ? var.cluster.talos_machine_config_version : var.image.update_version
|
||||
// Changing talos_version causes trouble as new certs are created
|
||||
}
|
||||
|
||||
data "talos_client_configuration" "this" {
|
||||
cluster_name = var.cluster.name
|
||||
client_configuration = talos_machine_secrets.this.client_configuration
|
||||
nodes = [for k, v in var.nodes : v.ip]
|
||||
# Don't use vip in talosconfig endpoints
|
||||
# ref - https://www.talos.dev/v1.9/talos-guides/network/vip/#caveats
|
||||
endpoints = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"]
|
||||
}
|
||||
|
||||
@@ -38,8 +45,8 @@ data "talos_machine_configuration" "this" {
|
||||
for_each = var.nodes
|
||||
cluster_name = var.cluster.name
|
||||
# This is the Kubernetes API Server endpoint.
|
||||
# ref - https://www.talos.dev/v1.9/introduction/prodnotes/#decide-the-kubernetes-endpoint
|
||||
cluster_endpoint = "https://${var.cluster.endpoint}:6443"
|
||||
# ref - https://www.talos.dev/latest/introduction/prodnotes/#decide-the-kubernetes-endpoint
|
||||
cluster_endpoint = "https://${local.kubernetes_endpoint}:6443"
|
||||
# @formatter:off
|
||||
talos_version = var.cluster.talos_machine_config_version != null ? var.cluster.talos_machine_config_version : (each.value.update == true ? var.image.update_version : var.image.version)
|
||||
# @formatter:on
|
||||
@@ -58,7 +65,7 @@ data "talos_machine_configuration" "this" {
|
||||
}), each.value.machine_type == "controlplane" ?
|
||||
templatefile("${path.module}/machine-config/control-plane.yaml.tftpl", {
|
||||
kubelet = var.cluster.kubelet
|
||||
extra_manifests = jsonencode(var.cluster.extra_manifests)
|
||||
extra_manifests = jsonencode(local.extra_manifests)
|
||||
api_server = var.cluster.api_server
|
||||
inline_manifests = jsonencode(terraform_data.cilium_bootstrap_inline_manifests.output)
|
||||
}) : ""
|
||||
@@ -79,9 +86,10 @@ resource "talos_machine_configuration_apply" "this" {
|
||||
|
||||
resource "talos_machine_bootstrap" "this" {
|
||||
depends_on = [talos_machine_configuration_apply.this]
|
||||
# Bootstrap with the first node. VIP not yet available at this stage, so cant use var.cluster.endpoint as it may be set to VIP
|
||||
# Bootstrap with the first control plane node.
|
||||
# VIP not yet available at this stage, so can't use var.cluster.vip
|
||||
# ref - https://www.talos.dev/v1.9/talos-guides/network/vip/#caveats
|
||||
node = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"][0]
|
||||
node = local.first_control_plane_node_ip
|
||||
client_configuration = talos_machine_secrets.this.client_configuration
|
||||
}
|
||||
|
||||
@@ -105,11 +113,8 @@ resource "talos_cluster_kubeconfig" "this" {
|
||||
talos_machine_bootstrap.this,
|
||||
data.talos_cluster_health.this
|
||||
]
|
||||
# If using VIP, it should be up by now, but to be safer retrive from one of the nodes
|
||||
# As mentioned don't use talosctl on vip
|
||||
# ref - https://www.talos.dev/v1.9/talos-guides/network/vip/#caveats
|
||||
# In kubeconfig endpoint will be polulated by cluster_endpoint from machine-config
|
||||
node = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"][0]
|
||||
# The kubeconfig endpoint will be populated from the talos_machine_configuration cluster_endpoint
|
||||
node = local.first_control_plane_node_ip
|
||||
client_configuration = talos_machine_secrets.this.client_configuration
|
||||
timeouts = {
|
||||
read = "1m"
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
output "machine_secrets" {
|
||||
value = talos_machine_secrets.this.machine_secrets
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
output "machine_config" {
|
||||
value = data.talos_machine_configuration.this
|
||||
}
|
||||
@@ -8,7 +13,6 @@ output "client_configuration" {
|
||||
}
|
||||
|
||||
output "kube_config" {
|
||||
#value = data.talos_cluster_kubeconfig.this
|
||||
value = talos_cluster_kubeconfig.this
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
@@ -16,13 +16,13 @@ variable "cluster" {
|
||||
description = "Cluster configuration"
|
||||
type = object({
|
||||
name = string
|
||||
endpoint = string
|
||||
vip = optional(string)
|
||||
gateway = string
|
||||
subnet_mask = optional(string, "24")
|
||||
talos_machine_config_version = optional(string)
|
||||
proxmox_cluster = string
|
||||
kubernetes_version = string
|
||||
gateway_api_version = string
|
||||
extra_manifests = optional(list(string))
|
||||
kubelet = optional(string)
|
||||
api_server = optional(string)
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
talos_cluster_config = {
|
||||
name = "talos"
|
||||
# This should point to the vip as below(if nodes on layer 2) or one of the nodes (if nodes not on layer 2)
|
||||
# Note: Nodes are not on layer 2 if there is a router between them (even a mesh router)
|
||||
# Not sure how it works if connected to the same router via ethernet (does it act as a switch then???)
|
||||
# Only use a VIP if the nodes share a layer 2 network
|
||||
# Ref: https://www.talos.dev/v1.9/talos-guides/network/vip/#requirements
|
||||
# Note This is Kubernetes API endpoint. Different from all mentions of Talos endpoints.
|
||||
endpoint = "192.168.1.102"
|
||||
vip = "192.168.1.99"
|
||||
gateway = "192.168.1.1"
|
||||
# The version of talos features to use in generated machine configuration. Generally the same as image version.
|
||||
@@ -18,10 +14,8 @@ talos_cluster_config = {
|
||||
bootstrap_manifest_path = "talos/inline-manifests/cilium-install.yaml"
|
||||
values_file_path = "../../k8s/infra/network/cilium/values.yaml"
|
||||
}
|
||||
extra_manifests = [
|
||||
"https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml",
|
||||
"https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.1/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml"
|
||||
]
|
||||
gateway_api_version = "v1.2.1" # renovate: github-releases=kubernetes-sigs/gateway-api
|
||||
extra_manifests = []
|
||||
kubelet = <<-EOT
|
||||
extraArgs:
|
||||
# Needed for Netbird agent https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls
|
||||
|
||||
@@ -3,7 +3,6 @@ talos_nodes = {
|
||||
host_node = "abel"
|
||||
machine_type = "controlplane"
|
||||
ip = "192.168.1.100"
|
||||
#dns = ["1.1.1.1", "8.8.8.8"] # Optional Value.
|
||||
mac_address = "BC:24:11:2E:C8:00"
|
||||
vm_id = 800
|
||||
cpu = 8
|
||||
@@ -28,13 +27,14 @@ talos_nodes = {
|
||||
mac_address = "BC:24:11:2E:C8:02"
|
||||
vm_id = 802
|
||||
cpu = 4
|
||||
ram_dedicated = 4096
|
||||
ram_dedicated = 6144
|
||||
#update = true
|
||||
}
|
||||
# "work-00" = {
|
||||
# host_node = "abel"
|
||||
# machine_type = "worker"
|
||||
# ip = "192.168.1.110"
|
||||
# dns = ["1.1.1.1", "8.8.8.8"] # Optional Value.
|
||||
# mac_address = "BC:24:11:2E:A8:00"
|
||||
# vm_id = 810
|
||||
# cpu = 8
|
||||
|
||||
@@ -33,13 +33,13 @@ variable "talos_cluster_config" {
|
||||
description = "Talos cluster configuration"
|
||||
type = object({
|
||||
name = string
|
||||
endpoint = string
|
||||
vip = optional(string)
|
||||
gateway = string
|
||||
talos_machine_config_version = optional(string)
|
||||
proxmox_cluster = string
|
||||
kubernetes_version = string
|
||||
extra_manifests = optional(list(string))
|
||||
gateway_api_version = string
|
||||
extra_manifests = optional(list(string), [])
|
||||
kubelet = optional(string)
|
||||
api_server = optional(string)
|
||||
cilium = object({
|
||||
|
||||
Reference in New Issue
Block a user