From 42f0223e2443f479e7ee72d43ae944a1926b0e3b Mon Sep 17 00:00:00 2001 From: jp0317 Date: Tue, 19 Nov 2024 02:55:32 +0000 Subject: [PATCH] move integer logical type from format.rs to schema type.rs --- parquet/src/format.rs | 6 ------ parquet/src/schema/types.rs | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/parquet/src/format.rs b/parquet/src/format.rs index 3cfd79642a5d..287d08b7a95c 100644 --- a/parquet/src/format.rs +++ b/parquet/src/format.rs @@ -1738,12 +1738,6 @@ impl crate::thrift::TSerializable for IntType { bit_width: f_1.expect("auto-generated code should have checked for presence of required fields"), is_signed: f_2.expect("auto-generated code should have checked for presence of required fields"), }; - if ret.bit_width != 8 && ret.bit_width != 16 && ret.bit_width != 32 && ret.bit_width != 64 { - return Err(thrift::Error::Protocol(ProtocolError::new( - ProtocolErrorKind::InvalidData, - "Bit width must be 8, 16, 32, or 64 for Integer logical type", - ))); - } Ok(ret) } fn write_to_out_protocol(&self, o_prot: &mut T) -> thrift::Result<()> { diff --git a/parquet/src/schema/types.rs b/parquet/src/schema/types.rs index 568a92f1d722..a0577d31d438 100644 --- a/parquet/src/schema/types.rs +++ b/parquet/src/schema/types.rs @@ -533,7 +533,8 @@ impl<'a> PrimitiveTypeBuilder<'a> { } } PhysicalType::FIXED_LEN_BYTE_ARRAY => { - let max_precision = (2f64.powi(8 * self.length - 1) - 1f64).log10().floor() as i32; + let length = self.length.checked_mul(8).unwrap_or(i32::MAX); + let max_precision = (2f64.powi(length - 1) - 1f64).log10().floor() as i32; if self.precision > max_precision { return Err(general_err!( @@ -1129,6 +1130,18 @@ pub fn from_thrift(elements: &[SchemaElement]) -> Result { Ok(schema_nodes.remove(0)) } +/// Checks if the logical type is valid. +fn check_logical_type(logical_type: &Option) -> Result<()> { + if let Some(LogicalType::Integer { bit_width, ..}) = *logical_type { + if bit_width != 8 && bit_width != 16 && bit_width != 32 && bit_width != 64 { + return Err(general_err!( + "Bit width must be 8, 16, 32, or 64 for Integer logical type" + )); + } + } + Ok(()) +} + /// Constructs a new Type from the `elements`, starting at index `index`. /// The first result is the starting index for the next Type after this one. If it is /// equal to `elements.len()`, then this Type is the last one. @@ -1153,6 +1166,9 @@ fn from_thrift_helper(elements: &[SchemaElement], index: usize) -> Result<(usize .logical_type .as_ref() .map(|value| LogicalType::from(value.clone())); + + check_logical_type(&logical_type)?; + let field_id = elements[index].field_id; match elements[index].num_children { // From parquet-format: