Description
Update-DatabricksSQLWarehouseConfig will not set the SQLConfigurationParameters.
I have the following code. I tried several cases (case 01 and case 02) and nothing works.
The function sets the "data access configs" but the the "Sql configuration parameters"
See the verbose output. It is not in the body.
$conf_pairs = @([ordered]@{"key" = "spark.hadoop.fs.azure.account.auth.type.storlakeprod.dfs.core.windows.net"
"value" = "OAuth"
},
[ordered]@{"key" = "spark.hadoop.fs.azure.account.oauth.provider.type.storlakeprod.dfs.core.windows.net"
"value" = "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"
},
[ordered]@{"key" = "spark.hadoop.fs.azure.account.oauth2.client.id.storlakeprod.dfs.core.windows.net"
"value" = "23456789"
},
[ordered]@{"key" = "spark.hadoop.fs.azure.account.oauth2.client.endpoint.storlakeprod.dfs.core.windows.net"
"value" = "https://login.microsoftonline.com/b4e7845a-xxxx-234-yyy-3b2333688f69a/oauth2/token"
},
[ordered]@{"key" = "spark.hadoop.fs.azure.account.oauth2.client.secret.storlakeprod.dfs.core.windows.net"
"value" = "{{secrets/myvault/myname}}"
},
[ordered]@{"key" = "spark.sql.session.timeZone"
"value" = "America/Chicago"
}
)
#case01
$sql_config_params01 = @{
configuration_pairs = @(
@{
"key" = "STATEMENT_TIMEOUT"
"value" = "600"
}
)
}
#case02
$sql_config_params02 = @(
@{
"key" = "STATEMENT_TIMEOUT"
"value" = "600"
}
)
#case 01
Update-DatabricksSQLWarehouseConfig -SecurityPolicy "DATA_ACCESS_CONTROL" -DataAccessConfig $conf_pairs -SQLConfigurationParameters $sql_config_params01 -EnableServerlessCompute $false -Force -Verbose
VERBOSE: Building Body/Parameters for final API call ...
VERBOSE: Checking if Databricks environment has been initialized yet ...
VERBOSE: Databricks environment already initialized!
VERBOSE: Setting final ApiURL ...
VERBOSE: Getting Headers for Databricks API call ...
VERBOSE: API Call: PUT https://adb-1993256495013067.7.azuredatabricks.net/api/2.0/sql/config/warehouses
VERBOSE: Headers:
Name Value
Content-Type application/json
Authorization Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Ii1LSTNROW5OUjdiUm9meG1lWm9YcWJIWkdldyIsImtpZCI6Ii1LSTNROW5OUjdiUm9meG1lWm9YcWJIWkdldyJ9.eyJhdWQiOiIyZmY4MTRhNi0zMzA0LTRhYjgtODVjYi1jZDBlNmY4NzljM…
VERBOSE: Body:
{
"security_policy": "DATA_ACCESS_CONTROL",
"data_access_config": [
{
"key": "spark.hadoop.fs.azure.account.auth.type.storlakeprod.dfs.core.windows.net",
"value": "OAuth"
},
{
"key": "spark.hadoop.fs.azure.account.oauth.provider.type.storlakeprod.dfs.core.windows.net",
"value": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"
},
{
"key": "spark.hadoop.fs.azure.account.oauth2.client.id.storlakeprod.dfs.core.windows.net",
"value": "23456789"
},
{
"key": "spark.hadoop.fs.azure.account.oauth2.client.endpoint.storlakeprod.dfs.core.windows.net",
"value": "https://login.microsoftonline.com/b4e7845a-xxxx-234-yyy-3b2333688f69a/oauth2/token"
},
{
"key": "spark.hadoop.fs.azure.account.oauth2.client.secret.storlakeprod.dfs.core.windows.net",
"value": "{{secrets/myvault/myname}}"
},
{
"key": "spark.sql.session.timeZone",
"value": "America/Chicago"
}
],
"enable_serverless_compute": "false"
}
VERBOSE: HTTP/1.1 PUT with 1033-byte payload
VERBOSE: received -byte response of content type application/json
VERBOSE: Content encoding: utf-8
VERBOSE: Response: {}
#case 02
Update-DatabricksSQLWarehouseConfig -SecurityPolicy "DATA_ACCESS_CONTROL" -DataAccessConfig $conf_pairs -SQLConfigurationParameters $sql_config_params02 -EnableServerlessCompute $false -Force -Verbose
VERBOSE: Building Body/Parameters for final API call ...
VERBOSE: Checking if Databricks environment has been initialized yet ...
VERBOSE: Databricks environment already initialized!
VERBOSE: Setting final ApiURL ...
VERBOSE: Getting Headers for Databricks API call ...
VERBOSE: API Call: PUT https://adb-1993256495013067.7.azuredatabricks.net/api/2.0/sql/config/warehouses
VERBOSE: Headers:
Name Value
Content-Type application/json
Authorization Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Ii1LSTNROW5OUjdiUm9meG1lWm9YcWJIWkdldyIsImtpZCI6Ii1LSTNROW5OUjdiUm9meG1lWm9YcWJIWkdldyJ9.eyJhdWQiOiIyZmY4MTRhNi0zMzA0LTRhYjgtODVjYi1jZDBlNmY4NzljM…
VERBOSE: Body:
{
"security_policy": "DATA_ACCESS_CONTROL",
"data_access_config": [
{
"key": "spark.hadoop.fs.azure.account.auth.type.storlakeprod.dfs.core.windows.net",
"value": "OAuth"
},
{
"key": "spark.hadoop.fs.azure.account.oauth.provider.type.storlakeprod.dfs.core.windows.net",
"value": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"
},
{
"key": "spark.hadoop.fs.azure.account.oauth2.client.id.storlakeprod.dfs.core.windows.net",
"value": "23456789"
},
{
"key": "spark.hadoop.fs.azure.account.oauth2.client.endpoint.storlakeprod.dfs.core.windows.net",
"value": "https://login.microsoftonline.com/b4e7845a-xxxx-234-yyy-3b2333688f69a/oauth2/token"
},
{
"key": "spark.hadoop.fs.azure.account.oauth2.client.secret.storlakeprod.dfs.core.windows.net",
"value": "{{secrets/myvault/myname}}"
},
{
"key": "spark.sql.session.timeZone",
"value": "America/Chicago"
}
],
"enable_serverless_compute": "false"
}
VERBOSE: HTTP/1.1 PUT with 1033-byte payload
VERBOSE: received -byte response of content type application/json
VERBOSE: Content encoding: utf-8
VERBOSE: Response: {}