PLAT-28 : Set secret_directory and extensions_directory setting for finer controls in duckdb

runtime/drivers/duckdb/duckdb.go

@@ -544,8 +544,19 @@ func (c *connection) reopenDB(ctx context.Context) error {
 	// We want to set preserve_insertion_order=false in hosted environments only (where source data is never viewed directly). Setting it reduces batch data ingestion time by ~40%.
 	// Hack: Using AllowHostAccess as a proxy indicator for a hosted environment.
 	if !c.config.AllowHostAccess {
+		extensionDir, err := extensions.ExtensionsDir()
+		if err != nil {
+			return err
+		}
+
+		secretDir, err := c.storage.DataDir("secrets")
+		if err != nil {
+			return err
+		}
 		dbInitQueries = append(dbInitQueries,
 			"SET GLOBAL preserve_insertion_order TO false",
+			fmt.Sprintf("SET extension_directory=%s", safeSQLString(extensionDir)),
+			fmt.Sprintf("SET secret_directory=%s", safeSQLString(secretDir)),
 		)
 	}
 

runtime/drivers/duckdb/extensions/extensions.go

@@ -24,6 +24,15 @@ func InstallExtensionsOnce() error {
 	return err
 }
 
+func ExtensionsDir() (string, error) {
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return "", err
+	}
+	duckdbExtensionsPath := filepath.Join(homeDir, ".duckdb", "extensions")
+	return duckdbExtensionsPath, nil
+}
+
 // installExtensions installs the embedded DuckDB extensions
 func installExtensions() error {
 	// Connect to DuckDB and get the version
@@ -58,11 +67,11 @@ func installExtensions() error {
 	}
 
 	// Define destination path
-	homeDir, err := os.UserHomeDir()
+	extDir, err := ExtensionsDir()
 	if err != nil {
 		return err
 	}
-	duckdbExtensionsPath := filepath.Join(homeDir, ".duckdb", "extensions", duckdbVersion, platformName)
+	duckdbExtensionsPath := filepath.Join(extDir, duckdbVersion, platformName)
 
 	// Create the destination directory if it doesn't exist
 	err = os.MkdirAll(duckdbExtensionsPath, os.ModePerm)

runtime/testruntime/connectors.go

@@ -16,6 +16,7 @@ import (
 	"github.com/joho/godotenv"
 	"github.com/rilldata/rill/admin/pkg/pgtestcontainer"
 	"github.com/rilldata/rill/runtime/drivers/clickhouse/testclickhouse"
+	"github.com/rilldata/rill/runtime/testruntime/testmode"
 	"github.com/stretchr/testify/require"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/modules/azurite"
@@ -80,6 +81,7 @@ var Connectors = map[string]ConnectorAcquireFunc{
 		return map[string]string{"dsn": dsn}
 	},
 	"motherduck": func(t TestingT) map[string]string {
+		testmode.Expensive(t)
 		loadDotEnv(t)
 		path := os.Getenv("RILL_RUNTIME_MOTHERDUCK_TEST_PATH")
 		require.NotEmpty(t, path)