fix: Remove datafusion.execution.parquet.cache_metadata config (#17062)

jonathanc-n · web-flow · commit 60ac1cc5f909 · 2025-08-07T16:10:29.000-04:00
* fix: Remove `datafusion.execution.parquet.cache_metadata` config

* prettier

* fix prettier?

* fix

* fix config

* fix test behaviour

* fix
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs
@@ -559,12 +559,6 @@ config_namespace! {
         /// (reading) Use any available bloom filters when reading parquet files
         pub bloom_filter_on_read: bool, default = true
 
-        /// (reading) Whether or not to enable the caching of embedded metadata of Parquet files
-        /// (footer and page metadata). Enabling it can offer substantial performance improvements
-        /// for repeated queries over large files. By default, the cache is automatically
-        /// invalidated when the underlying file is modified.
-        pub cache_metadata: bool, default = false
-
         // The following options affect writing to parquet files
         // and map to parquet::file::properties::WriterProperties
 
diff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rs
@@ -234,7 +234,6 @@ impl ParquetOptions {
             binary_as_string: _, // not used for writer props
             coerce_int96: _,     // not used for writer props
             skip_arrow_metadata: _,
-            cache_metadata: _,
         } = self;
 
         let mut builder = WriterProperties::builder()
@@ -502,7 +501,6 @@ mod tests {
             binary_as_string: defaults.binary_as_string,
             skip_arrow_metadata: defaults.skip_arrow_metadata,
             coerce_int96: None,
-            cache_metadata: defaults.cache_metadata,
         }
     }
 
@@ -613,7 +611,6 @@ mod tests {
                 binary_as_string: global_options_defaults.binary_as_string,
                 skip_arrow_metadata: global_options_defaults.skip_arrow_metadata,
                 coerce_int96: None,
-                cache_metadata: global_options_defaults.cache_metadata,
             },
             column_specific_options,
             key_value_metadata,
diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs
@@ -254,11 +254,6 @@ pub struct ParquetReadOptions<'a> {
     pub file_sort_order: Vec<Vec<SortExpr>>,
     /// Properties for decryption of Parquet files that use modular encryption
     pub file_decryption_properties: Option<ConfigFileDecryptionProperties>,
-    /// Whether or not to enable the caching of embedded metadata of this Parquet file (footer and
-    /// page metadata). Enabling it can offer substantial performance improvements for repeated
-    /// queries over large files. By default, the cache is automatically invalidated when the
-    /// underlying file is modified.
-    pub cache_metadata: Option<bool>,
 }
 
 impl Default for ParquetReadOptions<'_> {
@@ -271,7 +266,6 @@ impl Default for ParquetReadOptions<'_> {
             schema: None,
             file_sort_order: vec![],
             file_decryption_properties: None,
-            cache_metadata: None,
         }
     }
 }
@@ -331,12 +325,6 @@ impl<'a> ParquetReadOptions<'a> {
         self.file_decryption_properties = Some(file_decryption_properties);
         self
     }
-
-    /// Specify whether to enable or not metadata caching
-    pub fn cache_metadata(mut self, cache_metadata: bool) -> Self {
-        self.cache_metadata = Some(cache_metadata);
-        self
-    }
 }
 
 /// Options that control the reading of ARROW files.
@@ -602,9 +590,6 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> {
         if let Some(file_decryption_properties) = &self.file_decryption_properties {
             options.crypto.file_decryption = Some(file_decryption_properties.clone());
         }
-        if let Some(cache_metadata) = self.cache_metadata {
-            options.global.cache_metadata = cache_metadata;
-        }
         let mut file_format = ParquetFormat::new().with_options(options);
 
         if let Some(parquet_pruning) = self.parquet_pruning {
diff --git a/datafusion/core/tests/parquet/page_pruning.rs b/datafusion/core/tests/parquet/page_pruning.rs
@@ -903,8 +903,8 @@ async fn without_pushdown_filter() {
     )
     .unwrap();
 
-    // Without filter will not read pageIndex.
-    assert!(bytes_scanned_with_filter > bytes_scanned_without_filter);
+    // Same amount of bytes are scanned when defaulting to cache parquet metadata
+    assert!(bytes_scanned_with_filter == bytes_scanned_without_filter);
 }
 
 #[tokio::test]
diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rs
@@ -447,17 +447,14 @@ impl FileFormat for ParquetFormat {
 
         let mut source = ParquetSource::new(self.options.clone());
 
-        // Use the CachedParquetFileReaderFactory when metadata caching is enabled
-        if self.options.global.cache_metadata {
-            let metadata_cache =
-                state.runtime_env().cache_manager.get_file_metadata_cache();
-            let store = state
-                .runtime_env()
-                .object_store(conf.object_store_url.clone())?;
-            let cached_parquet_read_factory =
-                Arc::new(CachedParquetFileReaderFactory::new(store, metadata_cache));
-            source = source.with_parquet_file_reader_factory(cached_parquet_read_factory);
-        }
+        // Use the CachedParquetFileReaderFactory
+        let metadata_cache = state.runtime_env().cache_manager.get_file_metadata_cache();
+        let store = state
+            .runtime_env()
+            .object_store(conf.object_store_url.clone())?;
+        let cached_parquet_read_factory =
+            Arc::new(CachedParquetFileReaderFactory::new(store, metadata_cache));
+        source = source.with_parquet_file_reader_factory(cached_parquet_read_factory);
 
         if let Some(metadata_size_hint) = metadata_size_hint {
             source = source.with_metadata_size_hint(metadata_size_hint)
diff --git a/datafusion/proto-common/proto/datafusion_common.proto b/datafusion/proto-common/proto/datafusion_common.proto
@@ -530,7 +530,6 @@ message ParquetOptions {
   bool schema_force_view_types = 28; // default = false
   bool binary_as_string = 29; // default = false
   bool skip_arrow_metadata = 30; // default = false
-  bool cache_metadata = 33; // default = false
 
   oneof metadata_size_hint_opt {
     uint64 metadata_size_hint = 4;
diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rs
@@ -999,7 +999,6 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
                 protobuf::parquet_options::CoerceInt96Opt::CoerceInt96(v) => Some(v),
             }).unwrap_or(None),
             skip_arrow_metadata: value.skip_arrow_metadata,
-            cache_metadata: value.cache_metadata,
         })
     }
 }
diff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rs
@@ -5568,9 +5568,6 @@ impl serde::Serialize for ParquetOptions {
         if self.skip_arrow_metadata {
             len += 1;
         }
-        if self.cache_metadata {
-            len += 1;
-        }
         if self.dictionary_page_size_limit != 0 {
             len += 1;
         }
@@ -5670,9 +5667,6 @@ impl serde::Serialize for ParquetOptions {
         if self.skip_arrow_metadata {
             struct_ser.serialize_field("skipArrowMetadata", &self.skip_arrow_metadata)?;
         }
-        if self.cache_metadata {
-            struct_ser.serialize_field("cacheMetadata", &self.cache_metadata)?;
-        }
         if self.dictionary_page_size_limit != 0 {
             #[allow(clippy::needless_borrow)]
             #[allow(clippy::needless_borrows_for_generic_args)]
@@ -5810,8 +5804,6 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {
             "binaryAsString",
             "skip_arrow_metadata",
             "skipArrowMetadata",
-            "cache_metadata",
-            "cacheMetadata",
             "dictionary_page_size_limit",
             "dictionaryPageSizeLimit",
             "data_page_row_count_limit",
@@ -5858,7 +5850,6 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {
             SchemaForceViewTypes,
             BinaryAsString,
             SkipArrowMetadata,
-            CacheMetadata,
             DictionaryPageSizeLimit,
             DataPageRowCountLimit,
             MaxRowGroupSize,
@@ -5910,7 +5901,6 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {
                             "schemaForceViewTypes" | "schema_force_view_types" => Ok(GeneratedField::SchemaForceViewTypes),
                             "binaryAsString" | "binary_as_string" => Ok(GeneratedField::BinaryAsString),
                             "skipArrowMetadata" | "skip_arrow_metadata" => Ok(GeneratedField::SkipArrowMetadata),
-                            "cacheMetadata" | "cache_metadata" => Ok(GeneratedField::CacheMetadata),
                             "dictionaryPageSizeLimit" | "dictionary_page_size_limit" => Ok(GeneratedField::DictionaryPageSizeLimit),
                             "dataPageRowCountLimit" | "data_page_row_count_limit" => Ok(GeneratedField::DataPageRowCountLimit),
                             "maxRowGroupSize" | "max_row_group_size" => Ok(GeneratedField::MaxRowGroupSize),
@@ -5960,7 +5950,6 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {
                 let mut schema_force_view_types__ = None;
                 let mut binary_as_string__ = None;
                 let mut skip_arrow_metadata__ = None;
-                let mut cache_metadata__ = None;
                 let mut dictionary_page_size_limit__ = None;
                 let mut data_page_row_count_limit__ = None;
                 let mut max_row_group_size__ = None;
@@ -6081,12 +6070,6 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {
                             }
                             skip_arrow_metadata__ = Some(map_.next_value()?);
                         }
-                        GeneratedField::CacheMetadata => {
-                            if cache_metadata__.is_some() {
-                                return Err(serde::de::Error::duplicate_field("cacheMetadata"));
-                            }
-                            cache_metadata__ = Some(map_.next_value()?);
-                        }
                         GeneratedField::DictionaryPageSizeLimit => {
                             if dictionary_page_size_limit__.is_some() {
                                 return Err(serde::de::Error::duplicate_field("dictionaryPageSizeLimit"));
@@ -6196,7 +6179,6 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {
                     schema_force_view_types: schema_force_view_types__.unwrap_or_default(),
                     binary_as_string: binary_as_string__.unwrap_or_default(),
                     skip_arrow_metadata: skip_arrow_metadata__.unwrap_or_default(),
-                    cache_metadata: cache_metadata__.unwrap_or_default(),
                     dictionary_page_size_limit: dictionary_page_size_limit__.unwrap_or_default(),
                     data_page_row_count_limit: data_page_row_count_limit__.unwrap_or_default(),
                     max_row_group_size: max_row_group_size__.unwrap_or_default(),
diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rs
@@ -795,9 +795,6 @@ pub struct ParquetOptions {
     /// default = false
     #[prost(bool, tag = "30")]
     pub skip_arrow_metadata: bool,
-    /// default = false
-    #[prost(bool, tag = "33")]
-    pub cache_metadata: bool,
     #[prost(uint64, tag = "12")]
     pub dictionary_page_size_limit: u64,
     #[prost(uint64, tag = "18")]
diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rs
@@ -842,7 +842,6 @@ impl TryFrom<&ParquetOptions> for protobuf::ParquetOptions {
             binary_as_string: value.binary_as_string,
             skip_arrow_metadata: value.skip_arrow_metadata,
             coerce_int96_opt: value.coerce_int96.clone().map(protobuf::parquet_options::CoerceInt96Opt::CoerceInt96),
-            cache_metadata: value.cache_metadata,
         })
     }
 }
diff --git a/datafusion/proto/src/generated/datafusion_proto_common.rs b/datafusion/proto/src/generated/datafusion_proto_common.rs
@@ -795,9 +795,6 @@ pub struct ParquetOptions {
     /// default = false
     #[prost(bool, tag = "30")]
     pub skip_arrow_metadata: bool,
-    /// default = false
-    #[prost(bool, tag = "33")]
-    pub cache_metadata: bool,
     #[prost(uint64, tag = "12")]
     pub dictionary_page_size_limit: u64,
     #[prost(uint64, tag = "18")]
diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rs
@@ -411,7 +411,6 @@ impl TableParquetOptionsProto {
                 coerce_int96_opt: global_options.global.coerce_int96.map(|compression| {
                     parquet_options::CoerceInt96Opt::CoerceInt96(compression)
                 }),
-                cache_metadata: global_options.global.cache_metadata,
             }),
             column_specific_options: column_specific_options.into_iter().map(|(column_name, options)| {
                 ParquetColumnSpecificOptions {
@@ -505,7 +504,6 @@ impl From<&ParquetOptionsProto> for ParquetOptions {
             coerce_int96: proto.coerce_int96_opt.as_ref().map(|opt| match opt {
                 parquet_options::CoerceInt96Opt::CoerceInt96(coerce_int96) => coerce_int96.clone(),
             }),
-            cache_metadata: proto.cache_metadata,
         }
     }
 }
diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt
@@ -231,7 +231,6 @@ datafusion.execution.parquet.bloom_filter_fpp NULL
 datafusion.execution.parquet.bloom_filter_ndv NULL
 datafusion.execution.parquet.bloom_filter_on_read true
 datafusion.execution.parquet.bloom_filter_on_write false
-datafusion.execution.parquet.cache_metadata false
 datafusion.execution.parquet.coerce_int96 NULL
 datafusion.execution.parquet.column_index_truncate_length 64
 datafusion.execution.parquet.compression zstd(3)
@@ -345,7 +344,6 @@ datafusion.execution.parquet.bloom_filter_fpp NULL (writing) Sets bloom filter f
 datafusion.execution.parquet.bloom_filter_ndv NULL (writing) Sets bloom filter number of distinct values. If NULL, uses default parquet writer setting
 datafusion.execution.parquet.bloom_filter_on_read true (reading) Use any available bloom filters when reading parquet files
 datafusion.execution.parquet.bloom_filter_on_write false (writing) Write bloom filters for all columns when creating parquet files
-datafusion.execution.parquet.cache_metadata false (reading) Whether or not to enable the caching of embedded metadata of Parquet files (footer and page metadata). Enabling it can offer substantial performance improvements for repeated queries over large files. By default, the cache is automatically invalidated when the underlying file is modified.
 datafusion.execution.parquet.coerce_int96 NULL (reading) If true, parquet reader will read columns of physical type int96 as originating from a different resolution than nanosecond. This is useful for reading data from systems like Spark which stores microsecond resolution timestamps in an int96 allowing it to write values with a larger date range than 64-bit timestamps with nanosecond resolution.
 datafusion.execution.parquet.column_index_truncate_length 64 (writing) Sets column index truncate length
 datafusion.execution.parquet.compression zstd(3) (writing) Sets default parquet compression codec. Valid values are: uncompressed, snappy, gzip(level), lzo, brotli(level), lz4, zstd(level), and lz4_raw. These values are not case sensitive. If NULL, uses default parquet writer setting Note that this default setting is not the same as the default parquet writer setting.
diff --git a/datafusion/sqllogictest/test_files/parquet.slt b/datafusion/sqllogictest/test_files/parquet.slt
@@ -766,10 +766,6 @@ OPTIONS (MAX_ROW_GROUP_SIZE 4096, DATA_PAGE_ROW_COUNT_LIMIT 2048);
 ----
 20000
 
-# Enable the cache
-statement ok
-set datafusion.execution.parquet.cache_metadata = true;
-
 statement ok
 CREATE EXTERNAL TABLE t
 STORED AS PARQUET
@@ -867,5 +863,3 @@ select part, k, v from t order by k
 statement ok
 DROP TABLE t;
 
-statement ok
-set datafusion.execution.parquet.cache_metadata = false;
diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md

Original file line number	Diff line number	Diff line change
`@@ -254,11 +254,6 @@ pub struct ParquetReadOptions<'a> {`
`254`	`254`	`pub file_sort_order: Vec<Vec<SortExpr>>,`
`255`	`255`	`/// Properties for decryption of Parquet files that use modular encryption`
`256`	`256`	`pub file_decryption_properties: Option<ConfigFileDecryptionProperties>,`
`257`		`- /// Whether or not to enable the caching of embedded metadata of this Parquet file (footer and`
`258`		`- /// page metadata). Enabling it can offer substantial performance improvements for repeated`
`259`		`- /// queries over large files. By default, the cache is automatically invalidated when the`
`260`		`- /// underlying file is modified.`
`261`		`- pub cache_metadata: Option<bool>,`
`262`	`257`	`}`
`263`	`258`
`264`	`259`	`impl Default for ParquetReadOptions<'_> {`
`@@ -271,7 +266,6 @@ impl Default for ParquetReadOptions<'_> {`
`271`	`266`	`schema: None,`
`272`	`267`	`file_sort_order: vec![],`
`273`	`268`	`file_decryption_properties: None,`
`274`		`- cache_metadata: None,`
`275`	`269`	`}`
`276`	`270`	`}`
`277`	`271`	`}`
`@@ -331,12 +325,6 @@ impl<'a> ParquetReadOptions<'a> {`
`331`	`325`	`self.file_decryption_properties = Some(file_decryption_properties);`
`332`	`326`	`self`
`333`	`327`	`}`
`334`		`-`
`335`		`- /// Specify whether to enable or not metadata caching`
`336`		`- pub fn cache_metadata(mut self, cache_metadata: bool) -> Self {`
`337`		`- self.cache_metadata = Some(cache_metadata);`
`338`		`- self`
`339`		`- }`
`340`	`328`	`}`
`341`	`329`
`342`	`330`	`/// Options that control the reading of ARROW files.`
`@@ -602,9 +590,6 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> {`
`602`	`590`	`if let Some(file_decryption_properties) = &self.file_decryption_properties {`
`603`	`591`	`options.crypto.file_decryption = Some(file_decryption_properties.clone());`
`604`	`592`	`}`
`605`		`- if let Some(cache_metadata) = self.cache_metadata {`
`606`		`- options.global.cache_metadata = cache_metadata;`
`607`		`- }`
`608`	`593`	`let mut file_format = ParquetFormat::new().with_options(options);`
`609`	`594`
`610`	`595`	`if let Some(parquet_pruning) = self.parquet_pruning {`
Original file line number	Diff line number	Diff line change
`@@ -903,8 +903,8 @@ async fn without_pushdown_filter() {`
`903`	`903`	`)`
`904`	`904`	`.unwrap();`
`905`	`905`
`906`		`- // Without filter will not read pageIndex.`
`907`		`- assert!(bytes_scanned_with_filter > bytes_scanned_without_filter);`
	`906`	`+ // Same amount of bytes are scanned when defaulting to cache parquet metadata`
	`907`	`+ assert!(bytes_scanned_with_filter == bytes_scanned_without_filter);`
`908`	`908`	`}`
`909`	`909`
`910`	`910`	`#[tokio::test]`
Original file line number	Diff line number	Diff line change
`@@ -999,7 +999,6 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {`
`999`	`999`	`protobuf::parquet_options::CoerceInt96Opt::CoerceInt96(v) => Some(v),`
`1000`	`1000`	`}).unwrap_or(None),`
`1001`	`1001`	`skip_arrow_metadata: value.skip_arrow_metadata,`
`1002`		`- cache_metadata: value.cache_metadata,`
`1003`	`1002`	`})`
`1004`	`1003`	`}`
`1005`	`1004`	`}`
Original file line number	Diff line number	Diff line change
`@@ -842,7 +842,6 @@ impl TryFrom<&ParquetOptions> for protobuf::ParquetOptions {`
`842`	`842`	`binary_as_string: value.binary_as_string,`
`843`	`843`	`skip_arrow_metadata: value.skip_arrow_metadata,`
`844`	`844`	`coerce_int96_opt: value.coerce_int96.clone().map(protobuf::parquet_options::CoerceInt96Opt::CoerceInt96),`
`845`		`- cache_metadata: value.cache_metadata,`
`846`	`845`	`})`
`847`	`846`	`}`
`848`	`847`	`}`