feat: refactoring

Author: dmytro_velychko3

README.md

@@ -0,0 +1,37 @@
+resource "databricks_cluster" "cluster" {
+  for_each = { for cluster in var.clusters : cluster.cluster_name => cluster }
+
+  cluster_name            = each.value.cluster_name
+  spark_version           = each.value.spark_version
+  spark_conf              = each.value.spark_conf
+  spark_env_vars          = each.value.spark_env_vars
+  data_security_mode      = each.value.data_security_mode
+  node_type_id            = each.value.node_type_id
+  autotermination_minutes = each.value.autotermination_minutes
+
+  autoscale {
+    min_workers = each.value.min_workers
+    max_workers = each.value.max_workers
+  }
+
+  azure_attributes {
+    availability       = each.value.availability
+    first_on_demand    = each.value.first_on_demand
+    spot_bid_max_price = each.value.spot_bid_max_price
+  }
+
+  dynamic "cluster_log_conf" {
+    for_each = each.value.cluster_log_conf_destination != null ? [each.value.cluster_log_conf_destination] : []
+    content {
+      dbfs {
+        destination = cluster_log_conf.value
+      }
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [
+      state
+    ]
+  }
+}

cluster.tf

@@ -47,14 +47,14 @@ resource "databricks_sql_endpoint" "this" {
   for_each = { for endpoint in var.sql_endpoint : (endpoint.name) => endpoint }
 
   name                      = "${each.key}${local.suffix}"
-  cluster_size              = coalesce(each.value.cluster_size, "2X-Small")
-  min_num_clusters          = coalesce(each.value.min_num_clusters, 0)
-  max_num_clusters          = coalesce(each.value.max_num_clusters, 1)
-  auto_stop_mins            = coalesce(each.value.auto_stop_mins, "30")
-  enable_photon             = coalesce(each.value.enable_photon, false)
-  enable_serverless_compute = coalesce(each.value.enable_serverless_compute, false)
-  spot_instance_policy      = coalesce(each.value.spot_instance_policy, "COST_OPTIMIZED")
-  warehouse_type            = coalesce(each.value.warehouse_type, "PRO")
+  cluster_size              = each.value.cluster_size
+  min_num_clusters          = each.value.min_num_clusters
+  max_num_clusters          = each.value.max_num_clusters
+  auto_stop_mins            = each.value.auto_stop_mins
+  enable_photon             = each.value.enable_photon
+  enable_serverless_compute = each.value.enable_serverless_compute
+  spot_instance_policy      = each.value.spot_instance_policy
+  warehouse_type            = each.value.warehouse_type
 
   lifecycle {
     ignore_changes = [state, num_clusters]

main.tf

@@ -1,40 +1,39 @@
 locals {
-  secrets_acl_objects_list = flatten([for param in local.secret_scope_object : [
-  #secrets_acl_objects_list = flatten([for param in var.secret_scope_object : [
+  secrets_acl_objects_list = flatten([for param in var.secret_scope_object : [
     for permission in param.acl : {
       scope = param.scope_name, principal = permission.principal, permission = permission.permission
     }] if param.acl != null
   ])
-}
-
-resource "databricks_permissions" "default_cluster" {
-  for_each = coalesce(flatten([values(var.iam)[*].default_cluster_permission, "none"])...) != "none" ? var.default_cluster_id : {}
 
-  cluster_id = each.value
+  secret_scope_object = [for param in var.secret_scope : {
+    scope_name = databricks_secret_scope.this[param.scope_name].name
+    acl        = param.acl
+  } if param.acl != null]
+}
 
-  dynamic "access_control" {
-    for_each = { for k, v in var.iam : k => v.default_cluster_permission if v.default_cluster_permission != null }
-    content {
-      group_name       = databricks_group.this[access_control.key].display_name
-      permission_level = access_control.value
-    }
+resource "databricks_cluster_policy" "this" {
+  for_each = {
+    for param in var.custom_cluster_policies : (param.name) => param.definition
+    if param.definition != null
   }
+
+  name       = each.key
+  definition = jsonencode(each.value)
 }
 
-resource "databricks_permissions" "cluster_policy" {
+resource "databricks_permissions" "clusters" {
   for_each = {
-    for policy in var.cluster_policies_object : (policy.name) => policy
-    #for policy in var.cluster_policies_object : (policy.name) => policy
-    if policy.can_use != null
+    for v in var.clusters : (v.cluster_name) => v
+    if length(v.permissions) != 0
   }
 
-  cluster_policy_id = each.value.id
+  cluster_id = databricks_cluster.cluster[each.key].id
 
   dynamic "access_control" {
-    for_each = each.value.can_use
+    for_each = each.value.permissions
     content {
-      group_name       = databricks_group.this[access_control.value].display_name
-      permission_level = "CAN_USE"
+      group_name       = databricks_group.this[access_control.value.group_name].display_name
+      permission_level = access_control.value.permission_level
     }
   }
 }
@@ -56,7 +55,7 @@ resource "databricks_permissions" "unity_cluster" {
 resource "databricks_permissions" "sql_endpoint" {
   for_each = {
     for endpoint in var.sql_endpoint : (endpoint.name) => endpoint
-    if endpoint.permissions != null
+    if length(endpoint.permissions) != 0
   }
 
   sql_endpoint_id = databricks_sql_endpoint.this[each.key].id

permissions.tf

@@ -9,23 +9,12 @@ locals {
       scope_name = param.scope_name, key = secret.key, string_value = secret.string_value
     }] if param.secrets != null
   ])
-
-  secret_scope_object = [for param in var.secret_scope : {
-    scope_name = databricks_secret_scope.this[param.scope_name].name
-    acl        = param.acl
-  } if param.acl != null]
-
-  #cluster_policies_object = [for policy in var.custom_cluster_policies : {
-  #  id      = databricks_cluster_policy.this[policy.name].id
-  #  name    = databricks_cluster_policy.this[policy.name].name
-  #  can_use = policy.can_use
-  #} if policy.definition != null && var.sku == "premium"]
 }
 
 # Secret Scope with SP secrets for mounting Azure Data Lake Storage
 resource "databricks_secret_scope" "main" {
   name                     = "main"
-  initial_manage_principal = var.sku == "premium" ? null : "users"
+  initial_manage_principal = null
 }
 
 resource "databricks_secret" "main" {
@@ -44,7 +33,7 @@ resource "databricks_secret_scope" "this" {
   }
 
   name                     = each.key
-  initial_manage_principal = var.sku == "premium" ? null : "users"
+  initial_manage_principal = null
 }
 
 resource "databricks_secret" "this" {

secrets.tf

@@ -61,43 +61,22 @@ variable "iam" {
   }
 }
 
-# Default Cluster and Cluster Policy variables
-variable "default_cluster_id" {
-  type        = map(string)
-  description = "Single value of default Cluster id created by 'databricks-runtime' module"
-  default     = {}
-}
-
-variable "cluster_policies_object" {
-  type = list(object({
-    id      = string
-    name    = string
-    can_use = list(string)
-  }))
-  description = "List of objects that provides an ability to grant custom workspace group a permission to use(CAN_USE) cluster policy"
-  default = [{
-    id      = null
-    name    = null
-    can_use = null
-  }]
-}
-
 # SQL Endpoint variables
 variable "sql_endpoint" {
   type = set(object({
     name                      = string
-    cluster_size              = optional(string)
-    min_num_clusters          = optional(number)
-    max_num_clusters          = optional(number)
-    auto_stop_mins            = optional(string)
-    enable_photon             = optional(bool)
-    enable_serverless_compute = optional(bool)
-    spot_instance_policy      = optional(string)
-    warehouse_type            = optional(string)
+    cluster_size              = optional(string, "2X-Small")
+    min_num_clusters          = optional(number, 0)
+    max_num_clusters          = optional(number, 1)
+    auto_stop_mins            = optional(string, "30")
+    enable_photon             = optional(bool, false)
+    enable_serverless_compute = optional(bool, false)
+    spot_instance_policy      = optional(string, "COST_OPTIMIZED")
+    warehouse_type            = optional(string, "PRO")
     permissions = optional(set(object({
       group_name       = string
       permission_level = string
-    })))
+    })), [])
   }))
   description = "Set of objects with parameters to configure SQL Endpoint and assign permissions to it for certain custom groups"
   default     = []
@@ -254,12 +233,6 @@ EOT
   }]
 }
 
-variable "sku" {
-  type        = string
-  description = "The sku to use for the Databricks Workspace: [standard|premium|trial]"
-  default     = "premium"
-}
-
 variable "key_vault_id" {
   type        = string
   description = "ID of the Key Vault instance where the Secret resides"
@@ -277,4 +250,54 @@ variable "mountpoints" {
   }))
   description = "Mountpoints for databricks"
   default     = {}
-}
+}
+
+variable "custom_cluster_policies" {
+  type = list(object({
+    name       = string
+    can_use    = list(string)
+    definition = any
+    assigned   = bool
+  }))
+  description = <<-EOT
+Provides an ability to create custom cluster policy, assign it to cluster and grant CAN_USE permissions on it to certain custom groups
+name - name of custom cluster policy to create
+can_use - list of string, where values are custom group names, there groups have to be created with Terraform;
+definition - JSON document expressed in Databricks Policy Definition Language. No need to call 'jsonencode()' function on it when providing a value;
+assigned - boolean flag which assigns policy to default 'shared autoscaling' cluster, only single custom policy could be assigned;
+EOT
+  default = [{
+    name       = null
+    can_use    = null
+    definition = null
+    assigned   = false
+  }]
+  validation {
+    condition     = length([for policy in var.custom_cluster_policies : policy.assigned if policy.assigned]) <= 1
+    error_message = "Only single cluster policy assignment allowed. Please set 'assigned' parameter to 'true' for exact one or none policy"
+  }
+}
+
+variable "clusters" {
+  type = set(object({
+    cluster_name                 = string
+    spark_version                = optional(string, "11.3.x-scala2.12")
+    spark_conf                   = optional(map(any), {})
+    spark_env_vars               = optional(map(any), {})
+    data_security_mode           = optional(string, "USER_ISOLATION")
+    node_type_id                 = optional(string, "Standard_D3_v2")
+    autotermination_minutes      = optional(number, 30)
+    min_workers                  = optional(number, 1)
+    max_workers                  = optional(number, 2)
+    availability                 = optional(string, "ON_DEMAND_AZURE")
+    first_on_demand              = optional(number, 0)
+    spot_bid_max_price           = optional(number, 1)
+    cluster_log_conf_destination = optional(string, null)
+    permissions = optional(set(object({
+      group_name       = string
+      permission_level = string
+    })), [])
+  }))
+  description = "Set of objects with parameters to configure Databricks clusters and assign permissions to it for certain custom groups"
+  default     = []
+}