Skip to content

perf(duckdb): enable parquet metadata cache and prefetch #171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xe-nvdk merged 1 commit into from
Jan 29, 2026
Merged

perf(duckdb): enable parquet metadata cache and prefetch #171

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions internal/database/duckdb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,10 +127,9 @@ func configureDatabase(db *sql.DB, cfg *Config, logger zerolog.Logger) error {
}
}

// Enable object cache for Parquet metadata - significantly improves repeated queries
// This caches Parquet file metadata (schema, row group info) reducing I/O on repeated access
if _, err := db.Exec("SET enable_object_cache=true"); err != nil {
logger.Warn().Err(err).Msg("Failed to enable object cache (continuing without it)")
// Cache Parquet file metadata (schema, row group info) to reduce I/O on repeated access
if _, err := db.Exec("SET GLOBAL parquet_metadata_cache=true"); err != nil {
logger.Warn().Err(err).Msg("Failed to enable parquet metadata cache (continuing without it)")
}

// Preserve insertion order for deterministic results (important for LIMIT queries)
Expand Down Expand Up @@ -206,6 +205,10 @@ func configureS3Access(db *sql.DB, cfg *Config, logger zerolog.Logger) error {
return fmt.Errorf("failed to set s3_use_ssl: %w", err)
}

if _, err := db.Exec("SET GLOBAL prefetch_all_parquet_files=true"); err != nil {
logger.Warn().Err(err).Msg("Failed to set prefetch_all_parquet_files")
}

// Configure cache_httpfs extension for S3 file caching if enabled
if cfg.EnableS3Cache {
logger.Info().Msg("Enabling S3 file caching via cache_httpfs extension")
Expand Down