Migrate arrow-avro to Rust 2024 (#8545)

# Which issue does this PR close?

- Contribute to #6827

# Rationale for this change

Splitting up #8227.

# What changes are included in this PR?

Migrate `arrow-avro` to Rust 2024

# Are these changes tested?

CI

# Are there any user-facing changes?

Yes

Author: mbrobbel

arrow-avro/Cargo.toml

@@ -25,7 +25,7 @@ authors = { workspace = true }
 license = { workspace = true }
 keywords = { workspace = true }
 include = { workspace = true }
-edition = { workspace = true }
+edition = "2024"
 rust-version = { workspace = true }
 
 [lib]
@@ -93,4 +93,4 @@ harness = false
 
 [[bench]]
 name = "avro_writer"
-harness = false
+harness = false

arrow-avro/benches/avro_writer.rs

@@ -22,21 +22,21 @@ extern crate criterion;
 extern crate once_cell;
 
 use arrow_array::{
+    ArrayRef, BinaryArray, BooleanArray, Decimal32Array, Decimal64Array, Decimal128Array,
+    Decimal256Array, FixedSizeBinaryArray, Float32Array, Float64Array, ListArray, PrimitiveArray,
+    RecordBatch, StringArray, StructArray,
     builder::{ListBuilder, StringBuilder},
     types::{Int32Type, Int64Type, IntervalMonthDayNanoType, TimestampMicrosecondType},
-    ArrayRef, BinaryArray, BooleanArray, Decimal128Array, Decimal256Array, Decimal32Array,
-    Decimal64Array, FixedSizeBinaryArray, Float32Array, Float64Array, ListArray, PrimitiveArray,
-    RecordBatch, StringArray, StructArray,
 };
 use arrow_avro::writer::AvroWriter;
-use arrow_buffer::{i256, Buffer};
+use arrow_buffer::{Buffer, i256};
 use arrow_schema::{DataType, Field, IntervalUnit, Schema, TimeUnit, UnionFields, UnionMode};
-use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput};
+use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
 use once_cell::sync::Lazy;
 use rand::{
+    Rng, SeedableRng,
     distr::uniform::{SampleRange, SampleUniform},
     rngs::StdRng,
-    Rng, SeedableRng,
 };
 use std::collections::HashMap;
 use std::io::Cursor;

arrow-avro/benches/decoder.rs

@@ -26,10 +26,10 @@ extern crate once_cell;
 extern crate uuid;
 
 use apache_avro::types::Value;
-use apache_avro::{to_avro_datum, Decimal, Schema as ApacheSchema};
-use arrow_avro::schema::{Fingerprint, FingerprintAlgorithm, CONFLUENT_MAGIC, SINGLE_OBJECT_MAGIC};
+use apache_avro::{Decimal, Schema as ApacheSchema, to_avro_datum};
+use arrow_avro::schema::{CONFLUENT_MAGIC, Fingerprint, FingerprintAlgorithm, SINGLE_OBJECT_MAGIC};
 use arrow_avro::{reader::ReaderBuilder, schema::AvroSchema};
-use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput};
+use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
 use once_cell::sync::Lazy;
 use std::{hint::black_box, time::Duration};
 use uuid::Uuid;

arrow-avro/examples/decode_kafka_stream.rs

@@ -35,7 +35,7 @@
 use arrow_array::{Int64Array, RecordBatch, StringArray};
 use arrow_avro::reader::ReaderBuilder;
 use arrow_avro::schema::{
-    AvroSchema, Fingerprint, FingerprintAlgorithm, SchemaStore, CONFLUENT_MAGIC,
+    AvroSchema, CONFLUENT_MAGIC, Fingerprint, FingerprintAlgorithm, SchemaStore,
 };
 use arrow_schema::ArrowError;
 

arrow-avro/src/codec.rs

@@ -15,13 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 use crate::schema::{
-    make_full_name, Array, Attributes, ComplexType, Enum, Fixed, Map, Nullability, PrimitiveType,
-    Record, Schema, Type, TypeName, AVRO_ENUM_SYMBOLS_METADATA_KEY,
-    AVRO_FIELD_DEFAULT_METADATA_KEY, AVRO_NAMESPACE_METADATA_KEY, AVRO_NAME_METADATA_KEY,
+    AVRO_ENUM_SYMBOLS_METADATA_KEY, AVRO_FIELD_DEFAULT_METADATA_KEY, AVRO_NAME_METADATA_KEY,
+    AVRO_NAMESPACE_METADATA_KEY, Array, Attributes, ComplexType, Enum, Fixed, Map, Nullability,
+    PrimitiveType, Record, Schema, Type, TypeName, make_full_name,
 };
 use arrow_schema::{
-    ArrowError, DataType, Field, Fields, IntervalUnit, TimeUnit, UnionFields, UnionMode,
-    DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION,
+    ArrowError, DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION, DataType, Field, Fields,
+    IntervalUnit, TimeUnit, UnionFields, UnionMode,
 };
 #[cfg(feature = "small_decimals")]
 use arrow_schema::{DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION};
@@ -324,14 +324,14 @@ impl AvroDataType {
             Codec::Null => {
                 return Err(ArrowError::SchemaError(
                     "Default for `null` type must be JSON null".to_string(),
-                ))
+                ));
             }
             Codec::Boolean => match default_json {
                 Value::Bool(b) => AvroLiteral::Boolean(*b),
                 _ => {
                     return Err(ArrowError::SchemaError(
                         "Boolean default must be a JSON boolean".to_string(),
-                    ))
+                    ));
                 }
             },
             Codec::Int32 | Codec::Date32 | Codec::TimeMillis => {
@@ -393,7 +393,7 @@ impl AvroDataType {
                 _ => {
                     return Err(ArrowError::SchemaError(
                         "Default value must be a JSON array for Avro array type".to_string(),
-                    ))
+                    ));
                 }
             },
             Codec::Map(val_dt) => match default_json {
@@ -407,7 +407,7 @@ impl AvroDataType {
                 _ => {
                     return Err(ArrowError::SchemaError(
                         "Default value must be a JSON object for Avro map type".to_string(),
-                    ))
+                    ));
                 }
             },
             Codec::Struct(fields) => match default_json {
@@ -449,7 +449,7 @@ impl AvroDataType {
                 _ => {
                     return Err(ArrowError::SchemaError(
                         "Default value for record/struct must be a JSON object".to_string(),
-                    ))
+                    ));
                 }
             },
             Codec::Union(encodings, _, _) => {
@@ -1622,7 +1622,7 @@ impl<'a> Maker<'a> {
             _ => {
                 return Err(ArrowError::ParseError(format!(
                     "Illegal promotion {write_primitive:?} to {read_primitive:?}"
-                )))
+                )));
             }
         };
         let mut datatype = self.parse_type(reader_schema, None)?;
@@ -1894,8 +1894,8 @@ impl<'a> Maker<'a> {
 mod tests {
     use super::*;
     use crate::schema::{
-        Array, Attributes, ComplexType, Field as AvroFieldSchema, Fixed, PrimitiveType, Record,
-        Schema, Type, TypeName, AVRO_ROOT_RECORD_DEFAULT_NAME,
+        AVRO_ROOT_RECORD_DEFAULT_NAME, Array, Attributes, ComplexType, Field as AvroFieldSchema,
+        Fixed, PrimitiveType, Record, Schema, Type, TypeName,
     };
     use indexmap::IndexMap;
     use serde_json::{self, Value};

arrow-avro/src/reader/header.rs

@@ -17,9 +17,9 @@
 
 //! Decoder for [`Header`]
 
-use crate::compression::{CompressionCodec, CODEC_METADATA_KEY};
+use crate::compression::{CODEC_METADATA_KEY, CompressionCodec};
 use crate::reader::vlq::VLQDecoder;
-use crate::schema::{Schema, SCHEMA_METADATA_KEY};
+use crate::schema::{SCHEMA_METADATA_KEY, Schema};
 use arrow_schema::ArrowError;
 use std::io::BufRead;
 

arrow-avro/src/reader/mod.rs

@@ -477,8 +477,8 @@
 use crate::codec::AvroFieldBuilder;
 use crate::reader::header::read_header;
 use crate::schema::{
-    AvroSchema, Fingerprint, FingerprintAlgorithm, Schema, SchemaStore, CONFLUENT_MAGIC,
-    SINGLE_OBJECT_MAGIC,
+    AvroSchema, CONFLUENT_MAGIC, Fingerprint, FingerprintAlgorithm, SINGLE_OBJECT_MAGIC, Schema,
+    SchemaStore,
 };
 use arrow_array::{RecordBatch, RecordBatchReader};
 use arrow_schema::{ArrowError, SchemaRef};
@@ -695,7 +695,7 @@ impl Decoder {
                 None => {
                     return Err(ArrowError::ParseError(
                         "Missing magic bytes and fingerprint".to_string(),
-                    ))
+                    ));
                 }
             }
         }
@@ -1267,9 +1267,9 @@ mod test {
     use crate::reader::record::RecordDecoder;
     use crate::reader::{Decoder, Reader, ReaderBuilder};
     use crate::schema::{
-        AvroSchema, Fingerprint, FingerprintAlgorithm, PrimitiveType, SchemaStore,
-        AVRO_ENUM_SYMBOLS_METADATA_KEY, AVRO_NAMESPACE_METADATA_KEY, AVRO_NAME_METADATA_KEY,
-        CONFLUENT_MAGIC, SINGLE_OBJECT_MAGIC,
+        AVRO_ENUM_SYMBOLS_METADATA_KEY, AVRO_NAME_METADATA_KEY, AVRO_NAMESPACE_METADATA_KEY,
+        AvroSchema, CONFLUENT_MAGIC, Fingerprint, FingerprintAlgorithm, PrimitiveType,
+        SINGLE_OBJECT_MAGIC, SchemaStore,
     };
     use crate::test_util::arrow_test_data;
     use crate::writer::AvroWriter;
@@ -1289,7 +1289,7 @@ mod test {
     use arrow_array::types::{Int32Type, IntervalMonthDayNanoType};
     use arrow_array::*;
     use arrow_buffer::{
-        i256, Buffer, IntervalMonthDayNano, NullBuffer, OffsetBuffer, ScalarBuffer,
+        Buffer, IntervalMonthDayNano, NullBuffer, OffsetBuffer, ScalarBuffer, i256,
     };
     #[cfg(feature = "avro_custom_types")]
     use arrow_schema::{
@@ -1302,8 +1302,8 @@ mod test {
     };
     use bytes::Bytes;
     use futures::executor::block_on;
-    use futures::{stream, Stream, StreamExt, TryStreamExt};
-    use serde_json::{json, Value};
+    use futures::{Stream, StreamExt, TryStreamExt, stream};
+    use serde_json::{Value, json};
     use std::collections::HashMap;
     use std::fs::File;
     use std::io::{BufReader, Cursor};
@@ -2810,7 +2810,7 @@ mod test {
             let top_meta = proj_field.metadata().clone();
             let (expected_field_ref, expected_col): (Arc<Field>, ArrayRef) =
                 match (full_field.data_type(), proj_field.data_type()) {
-                    (&DataType::List(_), DataType::List(ref proj_elem)) => {
+                    (&DataType::List(_), DataType::List(proj_elem)) => {
                         let new_col =
                             rebuild_list_array_with_element(&col_full, proj_elem.clone(), false);
                         let nf = Field::new(
@@ -2821,7 +2821,7 @@ mod test {
                         .with_metadata(top_meta);
                         (Arc::new(nf), new_col)
                     }
-                    (&DataType::LargeList(_), DataType::LargeList(ref proj_elem)) => {
+                    (&DataType::LargeList(_), DataType::LargeList(proj_elem)) => {
                         let new_col =
                             rebuild_list_array_with_element(&col_full, proj_elem.clone(), true);
                         let nf = Field::new(

arrow-avro/src/reader/record.rs

@@ -21,15 +21,15 @@ use crate::codec::{
 };
 use crate::reader::cursor::AvroCursor;
 use crate::schema::Nullability;
-use arrow_array::builder::{Decimal128Builder, Decimal256Builder, IntervalMonthDayNanoBuilder};
 #[cfg(feature = "small_decimals")]
 use arrow_array::builder::{Decimal32Builder, Decimal64Builder};
+use arrow_array::builder::{Decimal128Builder, Decimal256Builder, IntervalMonthDayNanoBuilder};
 use arrow_array::types::*;
 use arrow_array::*;
 use arrow_buffer::*;
 use arrow_schema::{
-    ArrowError, DataType, Field as ArrowField, FieldRef, Fields, Schema as ArrowSchema, SchemaRef,
-    UnionFields, UnionMode, DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION,
+    ArrowError, DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION, DataType, Field as ArrowField,
+    FieldRef, Fields, Schema as ArrowSchema, SchemaRef, UnionFields, UnionMode,
 };
 #[cfg(feature = "small_decimals")]
 use arrow_schema::{DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION};
@@ -4047,8 +4047,8 @@ mod tests {
     }
 
     #[test]
-    fn test_record_append_default_missing_fields_without_projector_defaults_yields_type_nulls_or_empties(
-    ) {
+    fn test_record_append_default_missing_fields_without_projector_defaults_yields_type_nulls_or_empties()
+     {
         let fields = vec![("a", DataType::Int32, true), ("b", DataType::Utf8, true)];
         let mut field_refs: Vec<FieldRef> = Vec::new();
         let mut encoders: Vec<Decoder> = Vec::new();

arrow-avro/src/schema.rs

@@ -22,7 +22,7 @@ use arrow_schema::{
     UnionMode,
 };
 use serde::{Deserialize, Serialize};
-use serde_json::{json, Map as JsonMap, Value};
+use serde_json::{Map as JsonMap, Value, json};
 #[cfg(feature = "sha256")]
 use sha2::{Digest, Sha256};
 use std::borrow::Cow;
@@ -1432,7 +1432,7 @@ fn datatype_to_avro(
                 _ => {
                     return Err(ArrowError::SchemaError(
                         "Map 'entries' field must be Struct(key,value)".into(),
-                    ))
+                    ));
                 }
             };
             let values_schema = process_datatype(
@@ -1556,7 +1556,7 @@ fn datatype_to_avro(
         other => {
             return Err(ArrowError::NotYetImplemented(format!(
                 "Arrow type {other:?} has no Avro representation"
-            )))
+            )));
         }
     };
     Ok((val, extras))
@@ -2148,9 +2148,11 @@ mod tests {
                     store.lookup(&Fingerprint::Rabin(fp_val)).cloned(),
                     Some(schema.clone())
                 );
-                assert!(store
-                    .lookup(&Fingerprint::Rabin(fp_val.wrapping_add(1)))
-                    .is_none());
+                assert!(
+                    store
+                        .lookup(&Fingerprint::Rabin(fp_val.wrapping_add(1)))
+                        .is_none()
+                );
             }
             Fingerprint::Id(_id) => {
                 unreachable!("This test should only generate Rabin fingerprints")