feat: redactor

Author: dmytro_velychko3

main.tf

@@ -3,6 +3,21 @@ locals {
   suffix   = length(var.suffix) == 0 ? "" : "-${var.suffix}"
 }
 
+data "azurerm_key_vault_secret" "sp_client_id" {
+  name         = var.sp_client_id_secret_name
+  key_vault_id = var.key_vault_id
+}
+
+data "azurerm_key_vault_secret" "sp_key" {
+  name         = var.sp_key_secret_name
+  key_vault_id = var.key_vault_id
+}
+
+data "azurerm_key_vault_secret" "tenant_id" {
+  name         = var.tenant_id_secret_name
+  key_vault_id = var.key_vault_id
+}
+
 resource "databricks_workspace_conf" "this" {
   count = local.ip_rules == null ? 0 : 1
 

mount.tf

@@ -0,0 +1,16 @@
+resource "databricks_mount" "adls" {
+  for_each = var.mountpoints
+
+  name = each.key
+  uri  = "abfss://${each.value["container_name"]}@${each.value["storage_account_name"]}.dfs.core.windows.net"
+  extra_configs = {
+    "fs.azure.account.auth.type" : "OAuth",
+    "fs.azure.account.oauth.provider.type" : "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
+    "fs.azure.account.oauth2.client.id" : data.azurerm_key_vault_secret.sp_client_id.value,
+    "fs.azure.account.oauth2.client.secret" : databricks_secret.main[data.azurerm_key_vault_secret.sp_key.name].config_reference,
+    "fs.azure.account.oauth2.client.endpoint" : "https://login.microsoftonline.com/${data.azurerm_key_vault_secret.tenant_id.value}/oauth2/token",
+    "fs.azure.createRemoteFileSystemDuringInitialization" : "false",
+    "spark.databricks.sqldw.jdbc.service.principal.client.id" : data.azurerm_key_vault_secret.sp_client_id.value,
+    "spark.databricks.sqldw.jdbc.service.principal.client.secret" : databricks_secret.main[data.azurerm_key_vault_secret.sp_key.name].config_reference
+  }
+}

permissions.tf

@@ -1,5 +1,6 @@
 locals {
-  secrets_acl_objects_list = flatten([for param in var.secret_scope_object : [
+  secrets_acl_objects_list = flatten([for param in local.secret_scope_object : [
+  #secrets_acl_objects_list = flatten([for param in var.secret_scope_object : [
     for permission in param.acl : {
       scope = param.scope_name, principal = permission.principal, permission = permission.permission
     }] if param.acl != null
@@ -23,6 +24,7 @@ resource "databricks_permissions" "default_cluster" {
 resource "databricks_permissions" "cluster_policy" {
   for_each = {
     for policy in var.cluster_policies_object : (policy.name) => policy
+    #for policy in var.cluster_policies_object : (policy.name) => policy
     if policy.can_use != null
   }
 
@@ -38,7 +40,7 @@ resource "databricks_permissions" "cluster_policy" {
 }
 
 resource "databricks_permissions" "unity_cluster" {
-  count = var.unity_cluster_config.permissions != null && var.unity_cluster_enabled ? 1 : 0
+  count = length(var.unity_cluster_config.permissions) != 0 && var.unity_cluster_enabled ? 1 : 0
 
   cluster_id = databricks_cluster.this[0].id
 

secrets.tf

@@ -0,0 +1,85 @@
+locals {
+  sp_secrets = {
+    (var.sp_client_id_secret_name) = { value = data.azurerm_key_vault_secret.sp_client_id.value }
+    (var.sp_key_secret_name)       = { value = data.azurerm_key_vault_secret.sp_key.value }
+  }
+
+  secrets_objects_list = flatten([for param in var.secret_scope : [
+    for secret in param.secrets : {
+      scope_name = param.scope_name, key = secret.key, string_value = secret.string_value
+    }] if param.secrets != null
+  ])
+
+  secret_scope_object = [for param in var.secret_scope : {
+    scope_name = databricks_secret_scope.this[param.scope_name].name
+    acl        = param.acl
+  } if param.acl != null]
+
+  #cluster_policies_object = [for policy in var.custom_cluster_policies : {
+  #  id      = databricks_cluster_policy.this[policy.name].id
+  #  name    = databricks_cluster_policy.this[policy.name].name
+  #  can_use = policy.can_use
+  #} if policy.definition != null && var.sku == "premium"]
+}
+
+# Secret Scope with SP secrets for mounting Azure Data Lake Storage
+resource "databricks_secret_scope" "main" {
+  name                     = "main"
+  initial_manage_principal = var.sku == "premium" ? null : "users"
+}
+
+resource "databricks_secret" "main" {
+  for_each = local.sp_secrets
+
+  key          = each.key
+  string_value = each.value["value"]
+  scope        = databricks_secret_scope.main.id
+}
+
+# Custom additional Databricks Secret Scope
+resource "databricks_secret_scope" "this" {
+  for_each = {
+    for param in var.secret_scope : (param.scope_name) => param
+    if param.scope_name != null
+  }
+
+  name                     = each.key
+  initial_manage_principal = var.sku == "premium" ? null : "users"
+}
+
+resource "databricks_secret" "this" {
+  for_each = { for entry in local.secrets_objects_list : "${entry.scope_name}.${entry.key}" => entry }
+
+  key          = each.value.key
+  string_value = each.value.string_value
+  scope        = databricks_secret_scope.this[each.value.scope_name].id
+}
+
+# At the nearest future, Azure will allow acquiring AAD tokens by service principals,
+# thus providing an ability to create Azure backed Key Vault with Terraform
+# https://github.com/databricks/terraform-provider-databricks/pull/1965
+
+## Azure Key Vault-backed Scope
+#resource "azurerm_key_vault_access_policy" "databricks" {
+#  count = var.key_vault_secret_scope.key_vault_id != null ? 1 : 0
+
+#  key_vault_id = var.key_vault_secret_scope.key_vault_id
+#  object_id    = "9b38785a-6e08-4087-a0c4-20634343f21f" # Global 'AzureDatabricks' SP object id
+#  tenant_id    = data.azurerm_key_vault_secret.tenant_id.value
+#
+#  secret_permissions = [
+#    "Get",
+#    "List",
+#  ]
+#}
+#
+#resource "databricks_secret_scope" "external" {
+#  count = var.key_vault_secret_scope.key_vault_id != null ? 1 : 0
+#
+#  name = "external"
+#  keyvault_metadata {
+#    resource_id = var.key_vault_secret_scope.key_vault_id
+#    dns_name    = var.key_vault_secret_scope.dns_name
+#  }
+#  depends_on = [azurerm_key_vault_access_policy.databricks]
+#}

variables.tf

@@ -216,3 +216,65 @@ variable "unity_cluster_config" {
   description = "Specifies the databricks unity cluster configuration"
   default     = {}
 }
+
+######
+variable "sp_client_id_secret_name" {
+  type        = string
+  description = "The name of Azure Key Vault secret that contains ClientID of Service Principal to access in Azure Key Vault"
+}
+
+variable "sp_key_secret_name" {
+  type        = string
+  description = "The name of Azure Key Vault secret that contains client secret of Service Principal to access in Azure Key Vault"
+}
+
+# Secret Scope variables
+variable "secret_scope" {
+  type = list(object({
+    scope_name = string
+    acl = optional(list(object({
+      principal  = string
+      permission = string
+    })))
+    secrets = optional(list(object({
+      key          = string
+      string_value = string
+    })))
+  }))
+  description = <<-EOT
+Provides an ability to create custom Secret Scope, store secrets in it and assigning ACL for access management
+scope_name - name of Secret Scope to create;
+acl - list of objects, where 'principal' custom group name, this group is created in 'Premium' module; 'permission' is one of "READ", "WRITE", "MANAGE";
+secrets - list of objects, where object's 'key' param is created key name and 'string_value' is a value for it;
+EOT
+  default = [{
+    scope_name = null
+    acl        = null
+    secrets    = null
+  }]
+}
+
+variable "sku" {
+  type        = string
+  description = "The sku to use for the Databricks Workspace: [standard|premium|trial]"
+  default     = "premium"
+}
+
+variable "key_vault_id" {
+  type        = string
+  description = "ID of the Key Vault instance where the Secret resides"
+}
+
+variable "tenant_id_secret_name" {
+  type        = string
+  description = "The name of Azure Key Vault secret that contains tenant ID secret of Service Principal to access in Azure Key Vault"
+}
+
+variable "mountpoints" {
+  type = map(object({
+    storage_account_name = string
+    container_name       = string
+  }))
+  description = "Mountpoints for databricks"
+  default     = {}
+}