Add runner to run tests on DBR

Makefile

@@ -97,6 +97,12 @@ test-update-aws:
 
 test-update-all: test-update test-update-aws
 
+# DBR acceptance tests - run on Databricks Runtime using serverless compute
+# These require deco env run for authentication
+# Set DBR_TEST_VERBOSE=1 for detailed output (e.g., DBR_TEST_VERBOSE=1 make dbr-test)
+dbr-test:
+	deco env run -i -n aws-prod-ucws -- go test -v -timeout 4h -run TestDbrAcceptance$$ ./acceptance
+
 slowest:
 	${GO_TOOL} gotestsum tool slowest --jsonfile test-output.json --threshold 1s --num 50
 

acceptance/acceptance_test.go

@@ -46,7 +46,6 @@ var (
 	SkipLocal       bool
 	UseVersion      string
 	WorkspaceTmpDir bool
-	TerraformDir    string
 	OnlyOutTestToml bool
 )
 
@@ -79,11 +78,6 @@ func init() {
 	// DABs in the workspace runs on the workspace file system. This flags does the same for acceptance tests
 	// to simulate an identical environment.
 	flag.BoolVar(&WorkspaceTmpDir, "workspace-tmp-dir", false, "Run tests on the workspace file system (For DBR testing).")
-
-	// Symlinks from workspace file system to local file mount are not supported on DBR. Terraform implicitly
-	// creates these symlinks when a file_mirror is used for a provider (in .terraformrc). This flag
-	// allows us to download the provider to the workspace file system on DBR enabling DBR integration testing.
-	flag.StringVar(&TerraformDir, "terraform-dir", "", "Directory to download the terraform provider to")
 	flag.BoolVar(&OnlyOutTestToml, "only-out-test-toml", false, "Only regenerate out.test.toml files without running tests")
 }
 
@@ -173,14 +167,11 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 
 	buildDir := getBuildDir(t, cwd, runtime.GOOS, runtime.GOARCH)
 
-	terraformDir := TerraformDir
-	if terraformDir == "" {
-		terraformDir = buildDir
+	// Set up terraform for tests. Skip on DBR - tests with RunsOnDbr only use direct deployment.
+	if !WorkspaceTmpDir {
+		setupTerraform(t, cwd, buildDir, &repls)
 	}
 
-	// Download terraform and provider and create config.
-	RunCommand(t, []string{"python3", filepath.Join(cwd, "install_terraform.py"), "--targetdir", terraformDir}, ".", []string{})
-
 	wheelPath := buildDatabricksBundlesWheel(t, buildDir)
 	if wheelPath != "" {
 		t.Setenv("DATABRICKS_BUNDLES_WHEEL", wheelPath)
@@ -210,7 +201,12 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 		}
 	}
 
-	BuildYamlfmt(t)
+	// Skip building yamlfmt when running on workspace filesystem (DBR).
+	// This fails today on DBR. Can be looked into and fixed as a follow-up
+	// as and when needed.
+	if !WorkspaceTmpDir {
+		BuildYamlfmt(t)
+	}
 
 	t.Setenv("CLI", execPath)
 	repls.SetPath(execPath, "[CLI]")
@@ -255,16 +251,6 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 		t.Setenv("CLI_RELEASES_DIR", releasesDir)
 	}
 
-	terraformrcPath := filepath.Join(terraformDir, ".terraformrc")
-	t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath)
-	t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath)
-	repls.SetPath(terraformrcPath, "[DATABRICKS_TF_CLI_CONFIG_FILE]")
-
-	terraformExecPath := filepath.Join(terraformDir, "terraform") + exeSuffix
-	t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath)
-	t.Setenv("TERRAFORM", terraformExecPath)
-	repls.SetPath(terraformExecPath, "[TERRAFORM]")
-
 	// do it last so that full paths match first:
 	repls.SetPath(buildDir, "[BUILD_DIR]")
 
@@ -429,8 +415,8 @@ func getSkipReason(config *internal.TestConfig, configPath string) string {
 		return ""
 	}
 
-	if isTruePtr(config.SkipOnDbr) && WorkspaceTmpDir {
-		return "Disabled via SkipOnDbr setting in " + configPath
+	if WorkspaceTmpDir && !isTruePtr(config.RunsOnDbr) {
+		return "Disabled because RunsOnDbr is not set in " + configPath
 	}
 
 	if isTruePtr(config.Slow) && testing.Short() {
@@ -530,7 +516,7 @@ func runTest(t *testing.T,
 		// If the test is being run on DBR, auth is already configured
 		// by the dbr_runner notebook by reading a token from the notebook context and
 		// setting DATABRICKS_TOKEN and DATABRICKS_HOST environment variables.
-		_, _, tmpDir = workspaceTmpDir(t.Context(), t)
+		tmpDir = workspaceTmpDir(t.Context(), t)
 
 		// Run DBR tests on the workspace file system to mimic usage from
 		// DABs in the workspace.
@@ -569,6 +555,10 @@ func runTest(t *testing.T,
 		timeout = max(timeout, config.TimeoutCloud)
 	}
 
+	if WorkspaceTmpDir {
+		timeout = max(timeout, config.TimeoutDbr)
+	}
+
 	if ApplyCITimeoutMultipler {
 		timeout = time.Duration(float64(timeout) * config.TimeoutCIMultiplier)
 	}
@@ -1383,6 +1373,22 @@ func BuildYamlfmt(t *testing.T) {
 	RunCommand(t, args, "..", []string{})
 }
 
+// setupTerraform installs terraform and configures environment variables for tests.
+func setupTerraform(t *testing.T, cwd, buildDir string, repls *testdiff.ReplacementsContext) {
+	RunCommand(t, []string{"python3", filepath.Join(cwd, "install_terraform.py"), "--targetdir", buildDir}, ".", []string{})
+
+	terraformrcPath := filepath.Join(buildDir, ".terraformrc")
+	terraformExecPath := filepath.Join(buildDir, "terraform") + exeSuffix
+
+	t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath)
+	t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath)
+	t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath)
+	t.Setenv("TERRAFORM", terraformExecPath)
+
+	repls.SetPath(terraformrcPath, "[DATABRICKS_TF_CLI_CONFIG_FILE]")
+	repls.SetPath(terraformExecPath, "[TERRAFORM]")
+}
+
 func loadUserReplacements(t *testing.T, repls *testdiff.ReplacementsContext, tmpDir string) {
 	b, err := os.ReadFile(filepath.Join(tmpDir, userReplacementsFilename))
 	if os.IsNotExist(err) {

acceptance/bundle/artifacts/test.toml

@@ -11,7 +11,7 @@ RecordRequests = true
 #   Failed to inspect Python interpreter from active virtual environment at `.venv/bin/python3`
 #   Caused by: Failed to query Python interpreter
 #   Caused by: failed to canonicalize path `/Workspace/abcd/.venv/bin/python3`: Invalid cross-device link (os error 18)
-SkipOnDbr = true
+# Thus we cannot run this test on DBR.
 
 Ignore = [
     '.venv',

acceptance/bundle/integration_whl/test.toml

@@ -12,7 +12,7 @@ CloudSlow = true
 #   Failed to inspect Python interpreter from active virtual environment at `.venv/bin/python3`
 #   Caused by: Failed to query Python interpreter
 #   Caused by: failed to canonicalize path `/Workspace/abcd/.venv/bin/python3`: Invalid cross-device link (os error 18)
-SkipOnDbr = true
+# Thus we cannot run this test on DBR.
 
 Ignore = [
     ".databricks",

acceptance/bundle/resources/alerts/basic/test.toml

@@ -3,6 +3,10 @@ Cloud = false
 RecordRequests = false
 Ignore = [".databricks"]
 
+# known_failures.txt in ciconfig is not respected yet. This test is broken due to
+# workspace limits so we do not run it on DBR.
+RunsOnDbr = false
+
 # Alert tests timeout during bundle deploy (hang at file upload for 50+ minutes).
 # Use aggressive 5-minute timeout until the issue is resolved.
 # See: https://github.com/databricks/cli/issues/4221

acceptance/bundle/resources/clusters/deploy/data_security_mode/test.toml

@@ -1,6 +1,7 @@
 Local = false
 Cloud = true
 RecordRequests = false
+RunsOnDbr = true
 
 Ignore = [
     "databricks.yml",

acceptance/bundle/resources/clusters/deploy/simple/test.toml

@@ -1,6 +1,7 @@
 Local = false
 Cloud = true
 RecordRequests = false
+RunsOnDbr = true
 
 Ignore = [
     "databricks.yml",

acceptance/bundle/resources/clusters/run/spark_python_task/test.toml

@@ -1,6 +1,7 @@
 Local = false
 CloudSlow = true
 RecordRequests = false
+RunsOnDbr = true
 
 Ignore = [
     "databricks.yml",

acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/test.toml

@@ -2,6 +2,7 @@ Local = true
 Cloud = true
 RequiresWarehouse = true
 RecordRequests = false
+RunsOnDbr = true
 
 Ignore = [
     "databricks.yml",

acceptance/bundle/resources/dashboards/destroy/test.toml

@@ -2,6 +2,7 @@ Local = true
 Cloud = true
 RequiresWarehouse = true
 RecordRequests = false
+RunsOnDbr = true
 
 Ignore = [
     "databricks.yml",

acceptance/bundle/resources/dashboards/generate_inplace/test.toml

@@ -1,6 +1,7 @@
 Cloud = true
 Local = false
 RecordRequests = false
+RunsOnDbr = true
 
 [EnvMatrix]
   DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]

acceptance/bundle/resources/dashboards/nested-folders/test.toml

@@ -3,6 +3,7 @@ Local = true
 Cloud = true
 RequiresWarehouse = true
 RecordRequests = false
+RunsOnDbr = true
 
 Ignore = [
     "databricks.yml",

acceptance/bundle/resources/dashboards/simple/test.toml

@@ -2,6 +2,7 @@ Local = true
 Cloud = true
 RequiresWarehouse = true
 RecordRequests = false
+RunsOnDbr = true
 
 Ignore = [
     "databricks.yml",

acceptance/bundle/resources/dashboards/simple_outside_bundle_root/test.toml

@@ -2,6 +2,7 @@ Local = true
 Cloud = true
 RequiresWarehouse = true
 RecordRequests = false
+RunsOnDbr = true
 
 Ignore = [
     "databricks.yml",

acceptance/bundle/resources/dashboards/simple_syncroot/test.toml

@@ -2,6 +2,7 @@ Local = true
 Cloud = true
 RequiresWarehouse = true
 RecordRequests = false
+RunsOnDbr = true
 
 Ignore = [
     "databricks.yml",

acceptance/bundle/resources/dashboards/unpublish-out-of-band/test.toml

@@ -2,6 +2,7 @@ Local = true
 Cloud = true
 RequiresWarehouse = true
 RecordRequests = false
+RunsOnDbr = true
 
 Ignore = [
     "databricks.yml",

acceptance/bundle/resources/database_catalogs/basic/test.toml

@@ -5,3 +5,4 @@ Cloud = true
 CloudSlow = true
 
 RecordRequests = false
+RunsOnDbr = true

acceptance/bundle/resources/experiments/basic/test.toml

@@ -1,6 +1,7 @@
 Cloud = true
 Local = true
 RecordRequests = false
+RunsOnDbr = true
 
 [[Repls]]
 Old = '\d{3,}'

acceptance/bundle/resources/jobs/double-underscore-keys/test.toml

@@ -1,6 +1,7 @@
 Local = true
 Cloud = true
 RecordRequests = false
+RunsOnDbr = true
 
 Ignore = [
     "databricks.yml",

acceptance/bundle/resources/jobs/fail-on-active-runs/test.toml

@@ -1,6 +1,7 @@
 Local = true
 Cloud = true
 RecordRequests = false
+RunsOnDbr = true
 
 Ignore = [
     "databricks.yml",