Merge remote-tracking branch 'apache/main' into alamb/upgrade_arrow_57

Author: alamb

.github/workflows/audit.yml

@@ -42,7 +42,7 @@ jobs:
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
       - name: Install cargo-audit
-        uses: taiki-e/install-action@2cdf2d81f4edfc3b41d5ddf56083c70b48ac2475  # v2.62.35
+        uses: taiki-e/install-action@ebb229c6baa68383264f2822689b07b4916d9177  # v2.62.36
         with:
           tool: cargo-audit
       - name: Run audit check

.github/workflows/rust.yml

@@ -425,7 +425,7 @@ jobs:
           sudo apt-get update -qq
           sudo apt-get install -y -qq clang
       - name: Setup wasm-pack
-        uses: taiki-e/install-action@2cdf2d81f4edfc3b41d5ddf56083c70b48ac2475  # v2.62.35
+        uses: taiki-e/install-action@ebb229c6baa68383264f2822689b07b4916d9177  # v2.62.36
         with:
           tool: wasm-pack
       - name: Run tests with headless mode
@@ -752,7 +752,7 @@ jobs:
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
       - name: Install cargo-msrv
-        uses: taiki-e/install-action@2cdf2d81f4edfc3b41d5ddf56083c70b48ac2475  # v2.62.35
+        uses: taiki-e/install-action@ebb229c6baa68383264f2822689b07b4916d9177  # v2.62.36
         with:
           tool: cargo-msrv
 

datafusion-cli/src/object_storage/instrumented.rs

@@ -58,7 +58,7 @@ pub enum InstrumentedObjectStoreMode {
 
 impl fmt::Display for InstrumentedObjectStoreMode {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{:?}", self)
+        write!(f, "{self:?}")
     }
 }
 
@@ -426,7 +426,7 @@ pub enum Operation {
 
 impl fmt::Display for Operation {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{:?}", self)
+        write!(f, "{self:?}")
     }
 }
 
@@ -556,11 +556,11 @@ impl RequestSummaries {
                 let size_stats = s.size_stats.as_ref();
                 let dur_avg = duration_stats.map(|d| {
                     let avg = d.sum.as_secs_f32() / count;
-                    format!("{:.6}s", avg)
+                    format!("{avg:.6}s")
                 });
                 let size_avg = size_stats.map(|s| {
                     let avg = s.sum as f32 / count;
-                    format!("{} B", avg)
+                    format!("{avg} B")
                 });
                 [dur_avg, size_avg]
             })

datafusion-cli/src/print_options.rs

@@ -206,7 +206,7 @@ impl PrintOptions {
 
                         writeln!(writer, "Summaries:")?;
                         let summaries = RequestSummaries::new(&requests);
-                        writeln!(writer, "{}", summaries)?;
+                        writeln!(writer, "{summaries}")?;
                     }
                 }
             }

datafusion/common/src/datatype.rs

@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! [DataTypeExt] extension trait for converting DataTypes to Fields
+
+use crate::arrow::datatypes::{DataType, Field, FieldRef};
+use std::sync::Arc;
+
+/// DataFusion extension methods for Arrow [`DataType`]
+pub trait DataTypeExt {
+    /// Convert the type to field with nullable type and "" name
+    ///
+    /// This is used to track the places where we convert a [`DataType`]
+    /// into a nameless field to interact with an API that is
+    /// capable of representing an extension type and/or nullability.
+    fn into_nullable_field(self) -> Field;
+
+    /// Convert the type to field ref with nullable type and "" name
+    ///
+    /// Concise wrapper around [`DataTypeExt::into_nullable_field`] that
+    /// constructs a [`FieldRef`].
+    fn into_nullable_field_ref(self) -> FieldRef;
+}
+
+impl DataTypeExt for DataType {
+    fn into_nullable_field(self) -> Field {
+        Field::new("", self, true)
+    }
+
+    fn into_nullable_field_ref(self) -> FieldRef {
+        Arc::new(Field::new("", self, true))
+    }
+}
+
+/// DataFusion extension methods for Arrow [`Field`]
+pub trait FieldExt {
+    /// Returns a new Field representing a List of this Field's DataType.
+    fn into_list(self) -> Self;
+
+    /// Return a new Field representing this Field as the item type of a FixedSizeList
+    fn into_fixed_size_list(self, list_size: i32) -> Self;
+
+    /// Create a field with the default list field name ("item")
+    ///
+    /// Note that lists are allowed to have an arbitrarily named field;
+    /// however, a name other than 'item' will cause it to fail an
+    /// == check against a more idiomatically created list in
+    /// arrow-rs which causes issues.
+    fn into_list_item(self) -> Self;
+}
+
+impl FieldExt for Field {
+    fn into_list(self) -> Self {
+        DataType::List(Arc::new(self.into_list_item())).into_nullable_field()
+    }
+
+    fn into_fixed_size_list(self, list_size: i32) -> Self {
+        DataType::FixedSizeList(self.into_list_item().into(), list_size)
+            .into_nullable_field()
+    }
+
+    fn into_list_item(self) -> Self {
+        if self.name() != Field::LIST_FIELD_DEFAULT_NAME {
+            self.with_name(Field::LIST_FIELD_DEFAULT_NAME)
+        } else {
+            self
+        }
+    }
+}
+
+impl FieldExt for Arc<Field> {
+    fn into_list(self) -> Self {
+        DataType::List(self.into_list_item())
+            .into_nullable_field()
+            .into()
+    }
+
+    fn into_fixed_size_list(self, list_size: i32) -> Self {
+        DataType::FixedSizeList(self.into_list_item(), list_size)
+            .into_nullable_field()
+            .into()
+    }
+
+    fn into_list_item(self) -> Self {
+        if self.name() != Field::LIST_FIELD_DEFAULT_NAME {
+            Arc::unwrap_or_clone(self)
+                .with_name(Field::LIST_FIELD_DEFAULT_NAME)
+                .into()
+        } else {
+            self
+        }
+    }
+}

datafusion/common/src/lib.rs

@@ -39,6 +39,7 @@ pub mod alias;
 pub mod cast;
 pub mod config;
 pub mod cse;
+pub mod datatype;
 pub mod diagnostic;
 pub mod display;
 pub mod encryption;
@@ -47,6 +48,7 @@ pub mod file_options;
 pub mod format;
 pub mod hash_utils;
 pub mod instant;
+pub mod metadata;
 pub mod nested_struct;
 mod null_equality;
 pub mod parsers;