feat!: talos multinode initial support (#72)

nahsi · web-flow · commit f818568f1e4c · 2025-01-15T14:44:15.000+03:00
* Update

* Fix instance type

* Fix name

* F
diff --git a/ephemeral/instance.tf b/ephemeral/instance.tf
@@ -15,8 +15,9 @@ data "digitalocean_image" "talos" {
   name = "talos-v1.9.1"
 }
 
-resource "digitalocean_droplet" "talos" {
-  name     = "rnd-${local.prefix}-spectrum-cp"
+resource "digitalocean_droplet" "cp" {
+  count    = 1
+  name     = "rnd-${local.prefix}-spectrum-cp-${count.index}"
   size     = "s-8vcpu-16gb"
   image    = data.digitalocean_image.talos.id
   region   = "fra1"
@@ -32,6 +33,6 @@ resource "digitalocean_droplet" "talos" {
 }
 
 resource "digitalocean_reserved_ip" "l2" {
-  droplet_id = digitalocean_droplet.talos.id
-  region     = digitalocean_droplet.talos.region
+  droplet_id = digitalocean_droplet.cp[0].id
+  region     = digitalocean_droplet.cp[0].region
 }
diff --git a/ephemeral/main.tf b/ephemeral/main.tf
@@ -13,10 +13,15 @@ data "vault_generic_secret" "docker" {
 module "talos" {
   source       = "../terraform-modules/talos"
   cluster_name = local.prefix
-  server_ip    = digitalocean_droplet.talos.ipv4_address
 
-  config_patches = [
-    file("${path.root}/config_patch.yml"),
+  control_planes = [
+    {
+      name      = "cp-0"
+      server_ip = digitalocean_droplet.cp[0].ipv4_address
+      config_patches = [
+        file("${path.root}/patches/registry.yml"),
+      ]
+    },
   ]
 }
 
@@ -36,7 +41,7 @@ module "spectrum" {
     DOTOKEN         = base64encode(data.vault_generic_secret.spectrum.data.token)
     DOMAIN          = "${local.prefix}.fluence.dev"
     PREFIX          = local.prefix
-    LOADBALANCER_IP = digitalocean_droplet.talos.ipv4_address
+    LOADBALANCER_IP = digitalocean_droplet.cp[0].ipv4_address
     L2_IP           = digitalocean_reserved_ip.l2.ip_address
   }
 }
diff --git a/ephemeral/patches/registry.yml b/ephemeral/patches/registry.yml
diff --git a/examples/talos/README.md b/examples/talos/README.md
@@ -2,9 +2,9 @@
 
 ## Server customization
 
-In talos machine is configured from a single configuration file in yaml format. Talos terraform module allows to specify overlays of the main configuration file maintained by cloudless labs [here](https://github.com/fluencelabs/spectrum/blob/main/terraform-modules/talos/templates/controlplane_patch.yml) with `config_pathes` option.
+In talos machine is configured from a single configuration file in yaml format. Talos terraform module allows to specify overlays of the main configuration file maintained by cloudless labs [here](https://github.com/fluencelabs/spectrum/blob/main/terraform-modules/talos/base_config.yml) with `config_pathes` option.
 
-You can configure server specific things like layout of disks or network configuration. Checkout [talos documentation](https://www.talos.dev/v1.9/reference/configuration/v1alpha1/config/) and see `config_patch.yml` for an example of a `bond` interface configuration.
+You can configure server specific things like layout of disks or network configuration. Checkout [talos documentation](https://www.talos.dev/v1.9/reference/configuration/v1alpha1/config/) and see [`cp-0.yml`](https://github.com/fluencelabs/blob/main/examples/talos/patches/cp-0.yml) for an example of a `bond` interface configuration.
 
 ## Terraform state
 
diff --git a/examples/talos/main.tf b/examples/talos/main.tf
@@ -22,19 +22,16 @@ provider "helm" {
 
 module "talos" {
   source       = "git::https://github.com/fluencelabs/spectrum.git//terraform-modules/talos?ref=terraform-module-talos-v0.1.0" # x-release-please-version
-  cluster_name = var.cluster_name
-  server_ip    = var.server_ip
-  # config_patches = [
-  #   file("${path.root}/config_patch.yml"),
-  # ]
-}
-
-variable "server_ip" {
-  type        = string
-  description = "IP at which server is accessible"
-}
+  cluster_name = "my-cluster"
 
-variable "cluster_name" {
-  type        = string
-  description = "Name used in k8s and talos to distinguish between clusters"
+  control_planes = [
+    {
+      name      = "cp-0"
+      server_ip = "1.2.3.4"
+      config_patches = [
+        file("${path.root}/patches/base.yml"),
+        file("${path.root}/patches/cp-0.yml"),
+      ]
+    },
+  ]
 }
diff --git a/examples/talos/patches/base.yml b/examples/talos/patches/base.yml
@@ -0,0 +1,4 @@
+machine:
+  time:
+    servers:
+      - time.cloudflare.com
diff --git a/examples/talos/patches/cp-0.yml b/examples/talos/patches/cp-0.yml
@@ -30,7 +30,6 @@ machine:
       serial: Y0L0A031T5N8
 
   network:
-    hostname: foobar
     interfaces:
       - interface: bond0
         dhcp: false
diff --git a/examples/talos/variables.auto.tfvars b/examples/talos/variables.auto.tfvars
diff --git a/terraform-modules/talos/base_config.yml b/terraform-modules/talos/base_config.yml
@@ -1,7 +1,6 @@
 machine:
-  time:
-    servers:
-      - time.cloudflare.com
+  network:
+    hostname: ${hostname}
   install:
       diskSelector:
         size: '>= 100GB'
diff --git a/terraform-modules/talos/talos.tf b/terraform-modules/talos/talos.tf
@@ -1,61 +1,79 @@
 resource "talos_machine_secrets" "this" {
-  talos_version = "v1.9"
+  talos_version = var.talos_version
 }
 
-data "talos_machine_configuration" "this" {
+locals {
+  virtual_ip       = var.virtual_ip != "" ? var.virtual_ip : var.control_planes[0].server_ip
+  cluster_endpoint = "https://${local.virtual_ip}:6443"
+}
+
+data "talos_machine_configuration" "control_plane" {
+  for_each         = { for control_plane in var.control_planes : control_plane.name => control_plane }
+  talos_version    = var.talos_version
   cluster_name     = var.cluster_name
   machine_type     = "controlplane"
-  cluster_endpoint = "https://${var.server_ip}:6443"
+  cluster_endpoint = local.cluster_endpoint
   machine_secrets  = talos_machine_secrets.this.machine_secrets
-  talos_version    = "v1.9"
   config_patches = [
-    templatefile("${path.module}/templates/controlplane_patch.yml", {})
+    templatefile("${path.module}/base_config.yml", { hostname = each.value.name })
   ]
 }
 
-resource "talos_machine_configuration_apply" "this" {
+data "talos_machine_configuration" "worker" {
+  for_each         = { for worker in var.workers : worker.name => worker }
+  talos_version    = var.talos_version
+  cluster_name     = var.cluster_name
+  cluster_endpoint = local.cluster_endpoint
+  machine_type     = "worker"
+  machine_secrets  = talos_machine_secrets.this.machine_secrets
+  config_patches = [
+    templatefile("${path.module}/base_config.yml", { hostname = each.value.name })
+  ]
+}
+
+resource "talos_machine_configuration_apply" "control_plane" {
+  for_each                    = { for control_plane in var.control_planes : control_plane.name => control_plane }
+  client_configuration        = talos_machine_secrets.this.client_configuration
+  machine_configuration_input = data.talos_machine_configuration.control_plane[each.key].machine_configuration
+  node                        = each.value.server_ip
+  config_patches              = each.value.config_patches
+}
+
+resource "talos_machine_configuration_apply" "worker" {
+  for_each                    = { for worker in var.workers : worker.name => worker }
   client_configuration        = talos_machine_secrets.this.client_configuration
-  machine_configuration_input = data.talos_machine_configuration.this.machine_configuration
-  node                        = var.server_ip
-  config_patches              = var.config_patches
+  machine_configuration_input = data.talos_machine_configuration.worker[each.key].machine_configuration
+  node                        = each.value.server_ip
+  config_patches              = each.value.config_patches
 }
 
 data "talos_client_configuration" "this" {
   cluster_name         = var.cluster_name
   client_configuration = talos_machine_secrets.this.client_configuration
   endpoints = [
-    var.server_ip
+    for control_plane in var.control_planes : control_plane.server_ip
   ]
 }
 
 resource "talos_machine_bootstrap" "this" {
-  depends_on           = [talos_machine_configuration_apply.this]
+  depends_on           = [talos_machine_configuration_apply.control_plane]
   client_configuration = talos_machine_secrets.this.client_configuration
-  endpoint             = var.server_ip
-  node                 = var.server_ip
+  endpoint             = var.control_planes[0].server_ip
+  node                 = var.control_planes[0].server_ip
 }
 
 resource "talos_cluster_kubeconfig" "this" {
   client_configuration = talos_machine_secrets.this.client_configuration
-  node                 = var.server_ip
+  node                 = var.control_planes[0].server_ip
 }
 
-# data "talos_cluster_health" "this" {
-#   client_configuration   = data.talos_client_configuration.this.client_configuration
-#   control_plane_nodes    = [var.server_ip]
-#   endpoints              = data.talos_client_configuration.this.endpoints
-#   skip_kubernetes_checks = true
-# }
-
 data "http" "talos_health" {
-  url      = "https://${var.server_ip}:6443/version"
+  url      = "${local.cluster_endpoint}/version"
   insecure = true
   retry {
-    attempts     = 20
+    attempts     = 60
     min_delay_ms = 5000
     max_delay_ms = 5000
   }
-  depends_on = [
-    talos_machine_bootstrap.this,
-  ]
+  depends_on = [talos_machine_bootstrap.this]
 }
diff --git a/terraform-modules/talos/variables.tf b/terraform-modules/talos/variables.tf
@@ -1,12 +1,31 @@
-variable "server_ip" {
-  type = string
+variable "virtual_ip" {
+  type        = string
+  description = "IP shared between control-plane nodes"
+  default     = ""
 }
 
 variable "cluster_name" {
   type = string
 }
 
-variable "config_patches" {
-  type    = list(string)
+variable "talos_version" {
+  type    = string
+  default = "v1.9"
+}
+
+variable "control_planes" {
+  type = list(object({
+    name           = string
+    server_ip      = string
+    config_patches = list(string)
+  }))
+}
+
+variable "workers" {
+  type = list(object({
+    name           = string
+    server_ip      = string
+    config_patches = list(string)
+  }))
   default = []
 }

Original file line number	Diff line number	Diff line change
`@@ -15,8 +15,9 @@ data "digitalocean_image" "talos" {`
`15`	`15`	`name = "talos-v1.9.1"`
`16`	`16`	`}`
`17`	`17`
`18`		`-resource "digitalocean_droplet" "talos" {`
`19`		`- name = "rnd-${local.prefix}-spectrum-cp"`
	`18`	`+resource "digitalocean_droplet" "cp" {`
	`19`	`+ count = 1`
	`20`	`+ name = "rnd-${local.prefix}-spectrum-cp-${count.index}"`
`20`	`21`	`size = "s-8vcpu-16gb"`
`21`	`22`	`image = data.digitalocean_image.talos.id`
`22`	`23`	`region = "fra1"`
`@@ -32,6 +33,6 @@ resource "digitalocean_droplet" "talos" {`
`32`	`33`	`}`
`33`	`34`
`34`	`35`	`resource "digitalocean_reserved_ip" "l2" {`
`35`		`- droplet_id = digitalocean_droplet.talos.id`
`36`		`- region = digitalocean_droplet.talos.region`
	`36`	`+ droplet_id = digitalocean_droplet.cp[0].id`
	`37`	`+ region = digitalocean_droplet.cp[0].region`
`37`	`38`	`}`