Skip to content

Avoid panics and warnings when building avro without default features #8576

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 9, 2025
Merged

Avoid panics and warnings when building avro without default features #8576

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions arrow-avro/benches/avro_writer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,13 @@ extern crate criterion;
extern crate once_cell;

use arrow_array::{
ArrayRef, BinaryArray, BooleanArray, Decimal32Array, Decimal64Array, Decimal128Array,
Decimal256Array, FixedSizeBinaryArray, Float32Array, Float64Array, ListArray, PrimitiveArray,
RecordBatch, StringArray, StructArray,
ArrayRef, BinaryArray, BooleanArray, Decimal128Array, Decimal256Array, FixedSizeBinaryArray,
Float32Array, Float64Array, ListArray, PrimitiveArray, RecordBatch, StringArray, StructArray,
builder::{ListBuilder, StringBuilder},
types::{Int32Type, Int64Type, IntervalMonthDayNanoType, TimestampMicrosecondType},
};
#[cfg(feature = "small_decimals")]
use arrow_array::{Decimal32Array, Decimal64Array};
use arrow_avro::writer::AvroWriter;
use arrow_buffer::{Buffer, i256};
use arrow_schema::{DataType, Field, IntervalUnit, Schema, TimeUnit, UnionFields, UnionMode};
Expand Down Expand Up @@ -177,11 +178,13 @@ fn make_ts_micros_array_with_tag(n: usize, tag: u64) -> PrimitiveArray<Timestamp
// === Decimal helpers & generators ===

#[inline]
#[cfg(feature = "small_decimals")]
fn pow10_i32(p: u8) -> i32 {
(0..p).fold(1i32, |acc, _| acc.saturating_mul(10))
}

#[inline]
#[cfg(feature = "small_decimals")]
fn pow10_i64(p: u8) -> i64 {
(0..p).fold(1i64, |acc, _| acc.saturating_mul(10))
}
Expand All @@ -192,6 +195,7 @@ fn pow10_i128(p: u8) -> i128 {
}

#[inline]
#[cfg(feature = "small_decimals")]
fn make_decimal32_array_with_tag(n: usize, tag: u64, precision: u8, scale: i8) -> Decimal32Array {
let mut rng = rng_for(tag, n);
let max = pow10_i32(precision).saturating_sub(1);
Expand All @@ -202,6 +206,7 @@ fn make_decimal32_array_with_tag(n: usize, tag: u64, precision: u8, scale: i8) -
}

#[inline]
#[cfg(feature = "small_decimals")]
fn make_decimal64_array_with_tag(n: usize, tag: u64, precision: u8, scale: i8) -> Decimal64Array {
let mut rng = rng_for(tag, n);
let max = pow10_i64(precision).saturating_sub(1);
Expand Down Expand Up @@ -539,6 +544,7 @@ static STRUCT_DATA: Lazy<Vec<RecordBatch>> = Lazy::new(|| {
.collect()
});

#[cfg(feature = "small_decimals")]
static DECIMAL32_DATA: Lazy<Vec<RecordBatch>> = Lazy::new(|| {
// Choose a representative precision/scale within Decimal32 limits
let precision: u8 = 7;
Expand All @@ -554,6 +560,7 @@ static DECIMAL32_DATA: Lazy<Vec<RecordBatch>> = Lazy::new(|| {
.collect()
});

#[cfg(feature = "small_decimals")]
static DECIMAL64_DATA: Lazy<Vec<RecordBatch>> = Lazy::new(|| {
let precision: u8 = 13;
let scale: i8 = 3;
Expand Down Expand Up @@ -821,7 +828,9 @@ fn criterion_benches(c: &mut Criterion) {
bench_writer_scenario(c, "write-FixedSizeBinary16", &FIXED16_DATA);
bench_writer_scenario(c, "write-UUID(logicalType)", &UUID16_DATA);
bench_writer_scenario(c, "write-IntervalMonthDayNanoDuration", &INTERVAL_MDN_DATA);
#[cfg(feature = "small_decimals")]
bench_writer_scenario(c, "write-Decimal32(bytes)", &DECIMAL32_DATA);
#[cfg(feature = "small_decimals")]
bench_writer_scenario(c, "write-Decimal64(bytes)", &DECIMAL64_DATA);
bench_writer_scenario(c, "write-Decimal128(bytes)", &DECIMAL128_BYTES_DATA);
bench_writer_scenario(c, "write-Decimal128(fixed16)", &DECIMAL128_FIXED16_DATA);
Expand Down
8 changes: 8 additions & 0 deletions arrow-avro/src/compression.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@
// under the License.

use arrow_schema::ArrowError;
#[cfg(any(
feature = "deflate",
feature = "zstd",
feature = "bzip2",
feature = "xz"
))]
use std::io::{Read, Write};

/// The metadata key used for storing the JSON encoded [`CompressionCodec`]
Expand All @@ -40,6 +46,7 @@ pub enum CompressionCodec {
}

impl CompressionCodec {
#[allow(unused_variables)]
pub(crate) fn decompress(&self, block: &[u8]) -> Result<Vec<u8>, ArrowError> {
match self {
#[cfg(feature = "deflate")]
Expand Down Expand Up @@ -112,6 +119,7 @@ impl CompressionCodec {
}
}

#[allow(unused_variables)]
pub(crate) fn compress(&self, data: &[u8]) -> Result<Vec<u8>, ArrowError> {
match self {
#[cfg(feature = "deflate")]
Expand Down
93 changes: 54 additions & 39 deletions arrow-avro/src/reader/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1273,11 +1273,12 @@ mod test {
};
use crate::test_util::arrow_test_data;
use crate::writer::AvroWriter;
use arrow::array::ArrayDataBuilder;
use arrow_array::builder::{
ArrayBuilder, BooleanBuilder, Float32Builder, Float64Builder, Int32Builder, Int64Builder,
ListBuilder, MapBuilder, MapFieldNames, StringBuilder, StructBuilder,
ArrayBuilder, BooleanBuilder, Float32Builder, Int32Builder, Int64Builder, ListBuilder,
MapBuilder, StringBuilder, StructBuilder,
};
#[cfg(feature = "snappy")]
use arrow_array::builder::{Float64Builder, MapFieldNames};
use arrow_array::cast::AsArray;
#[cfg(not(feature = "avro_custom_types"))]
use arrow_array::types::Int64Type;
Expand All @@ -1288,9 +1289,9 @@ mod test {
};
use arrow_array::types::{Int32Type, IntervalMonthDayNanoType};
use arrow_array::*;
use arrow_buffer::{
Buffer, IntervalMonthDayNano, NullBuffer, OffsetBuffer, ScalarBuffer, i256,
};
#[cfg(feature = "snappy")]
use arrow_buffer::{Buffer, NullBuffer};
use arrow_buffer::{IntervalMonthDayNano, OffsetBuffer, ScalarBuffer, i256};
#[cfg(feature = "avro_custom_types")]
use arrow_schema::{
ArrowError, DataType, Field, FieldRef, Fields, IntervalUnit, Schema, TimeUnit, UnionFields,
Expand All @@ -1309,6 +1310,23 @@ mod test {
use std::io::{BufReader, Cursor};
use std::sync::Arc;

fn files() -> impl Iterator<Item = &'static str> {
[
// TODO: avoid requiring snappy for this file
#[cfg(feature = "snappy")]
"avro/alltypes_plain.avro",
#[cfg(feature = "snappy")]
"avro/alltypes_plain.snappy.avro",
#[cfg(feature = "zstd")]
"avro/alltypes_plain.zstandard.avro",
#[cfg(feature = "bzip2")]
"avro/alltypes_plain.bzip2.avro",
#[cfg(feature = "xz")]
"avro/alltypes_plain.xz.avro",
]
.into_iter()
}

fn read_file(path: &str, batch_size: usize, utf8_view: bool) -> RecordBatch {
let file = File::open(path).unwrap();
let reader = ReaderBuilder::new()
Expand Down Expand Up @@ -1714,14 +1732,7 @@ mod test {

#[test]
fn test_alltypes_schema_promotion_mixed() {
let files = [
"avro/alltypes_plain.avro",
"avro/alltypes_plain.snappy.avro",
"avro/alltypes_plain.zstandard.avro",
"avro/alltypes_plain.bzip2.avro",
"avro/alltypes_plain.xz.avro",
];
for file in files {
for file in files() {
let file = arrow_test_data(file);
let mut promotions: HashMap<&str, &str> = HashMap::new();
promotions.insert("id", "long");
Expand Down Expand Up @@ -1829,14 +1840,7 @@ mod test {

#[test]
fn test_alltypes_schema_promotion_long_to_float_only() {
let files = [
"avro/alltypes_plain.avro",
"avro/alltypes_plain.snappy.avro",
"avro/alltypes_plain.zstandard.avro",
"avro/alltypes_plain.bzip2.avro",
"avro/alltypes_plain.xz.avro",
];
for file in files {
for file in files() {
let file = arrow_test_data(file);
let mut promotions: HashMap<&str, &str> = HashMap::new();
promotions.insert("bigint_col", "float");
Expand Down Expand Up @@ -1933,14 +1937,7 @@ mod test {

#[test]
fn test_alltypes_schema_promotion_bytes_to_string_only() {
let files = [
"avro/alltypes_plain.avro",
"avro/alltypes_plain.snappy.avro",
"avro/alltypes_plain.zstandard.avro",
"avro/alltypes_plain.bzip2.avro",
"avro/alltypes_plain.xz.avro",
];
for file in files {
for file in files() {
let file = arrow_test_data(file);
let mut promotions: HashMap<&str, &str> = HashMap::new();
promotions.insert("date_string_col", "string");
Expand Down Expand Up @@ -2033,6 +2030,8 @@ mod test {
}

#[test]
// TODO: avoid requiring snappy for this file
Copy link
Member Author

@mbrobbel mbrobbel Oct 9, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are some files without .snappy that still seem to require snappy, not sure if that's expected or if we should fix that?

Copy link
Contributor

@jecsand838 jecsand838 Oct 9, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I noticed that as well. Not 100% why that's the case. My guess would be that it was an attempt at making the test files smaller.

We probably do want some of these files truly uncompressed testing purposes though.

#[cfg(feature = "snappy")]
fn test_alltypes_illegal_promotion_bool_to_double_errors() {
let file = arrow_test_data("avro/alltypes_plain.avro");
let mut promotions: HashMap<&str, &str> = HashMap::new();
Expand Down Expand Up @@ -2691,6 +2690,8 @@ mod test {
}

#[test]
// TODO: avoid requiring snappy for this file
#[cfg(feature = "snappy")]
fn test_alltypes_skip_writer_fields_keep_double_only() {
let file = arrow_test_data("avro/alltypes_plain.avro");
let reader_schema =
Expand All @@ -2708,6 +2709,8 @@ mod test {
}

#[test]
// TODO: avoid requiring snappy for this file
#[cfg(feature = "snappy")]
fn test_alltypes_skip_writer_fields_reorder_and_skip_many() {
let file = arrow_test_data("avro/alltypes_plain.avro");
let reader_schema =
Expand Down Expand Up @@ -4470,14 +4473,6 @@ mod test {

#[test]
fn test_alltypes() {
let files = [
"avro/alltypes_plain.avro",
"avro/alltypes_plain.snappy.avro",
"avro/alltypes_plain.zstandard.avro",
"avro/alltypes_plain.bzip2.avro",
"avro/alltypes_plain.xz.avro",
];

let expected = RecordBatch::try_from_iter_with_nullable([
(
"id",
Expand Down Expand Up @@ -4562,7 +4557,7 @@ mod test {
])
.unwrap();

for file in files {
for file in files() {
let file = arrow_test_data(file);

assert_eq!(read_file(&file, 8, false), expected);
Expand All @@ -4571,6 +4566,8 @@ mod test {
}

#[test]
// TODO: avoid requiring snappy for this file
#[cfg(feature = "snappy")]
fn test_alltypes_dictionary() {
let file = "avro/alltypes_dictionary.avro";
let expected = RecordBatch::try_from_iter_with_nullable([
Expand Down Expand Up @@ -4693,6 +4690,8 @@ mod test {
}

#[test]
// TODO: avoid requiring snappy for this file
#[cfg(feature = "snappy")]
fn test_binary() {
let file = arrow_test_data("avro/binary.avro");
let batch = read_file(&file, 8, false);
Expand All @@ -4719,6 +4718,8 @@ mod test {
}

#[test]
// TODO: avoid requiring snappy for these files
#[cfg(feature = "snappy")]
fn test_decimal() {
// Choose expected Arrow types depending on the `small_decimals` feature flag.
// With `small_decimals` enabled, Decimal32/Decimal64 are used where their
Expand Down Expand Up @@ -5040,6 +5041,8 @@ mod test {
}

#[test]
// TODO: avoid requiring snappy for this file
#[cfg(feature = "snappy")]
fn test_dict_pages_offset_zero() {
let file = arrow_test_data("avro/dict-page-offset-zero.avro");
let batch = read_file(&file, 32, false);
Expand All @@ -5055,6 +5058,8 @@ mod test {
}

#[test]
// TODO: avoid requiring snappy for this file
#[cfg(feature = "snappy")]
fn test_list_columns() {
let file = arrow_test_data("avro/list_columns.avro");
let mut int64_list_builder = ListBuilder::new(Int64Builder::new());
Expand Down Expand Up @@ -5117,6 +5122,7 @@ mod test {
}

#[test]
#[cfg(feature = "snappy")]
fn test_nested_lists() {
use arrow_data::ArrayDataBuilder;
let file = arrow_test_data("avro/nested_lists.snappy.avro");
Expand Down Expand Up @@ -5314,6 +5320,7 @@ mod test {
}

#[test]
#[cfg(feature = "snappy")]
fn test_single_nan() {
let file = arrow_test_data("avro/single_nan.avro");
let actual = read_file(&file, 1, false);
Expand Down Expand Up @@ -5392,6 +5399,7 @@ mod test {
}

#[test]
#[cfg(feature = "snappy")]
fn test_datapage_v2() {
let file = arrow_test_data("avro/datapage_v2.snappy.avro");
let batch = read_file(&file, 8, false);
Expand Down Expand Up @@ -5653,7 +5661,10 @@ mod test {
}

#[test]
// TODO: avoid requiring snappy for this file
#[cfg(feature = "snappy")]
fn test_repeated_no_annotation() {
use arrow_data::ArrayDataBuilder;
let file = arrow_test_data("avro/repeated_no_annotation.avro");
let batch_large = read_file(&file, 8, false);
// id column
Expand Down Expand Up @@ -5742,6 +5753,8 @@ mod test {
}

#[test]
// TODO: avoid requiring snappy for this file
#[cfg(feature = "snappy")]
fn test_nonnullable_impala() {
let file = arrow_test_data("avro/nonnullable.impala.avro");
let id = Int64Array::from(vec![Some(8)]);
Expand Down Expand Up @@ -6057,6 +6070,8 @@ mod test {
}

#[test]
// TODO: avoid requiring snappy for this file
#[cfg(feature = "snappy")]
fn test_nullable_impala() {
let file = arrow_test_data("avro/nullable.impala.avro");
let batch1 = read_file(&file, 3, false);
Expand Down
Loading
Loading