Skip to content

Commit 2c9fbb3

Browse files
authored
Revert "GH-3574: Statistics.toParquetStatistics always set null_count(#3575)" (#3688)
This reverts commit 63aebcc.
1 parent ed6aa05 commit 2c9fbb3

2 files changed

Lines changed: 9 additions & 12 deletions

File tree

parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -815,17 +815,15 @@ public static Statistics toParquetStatistics(org.apache.parquet.column.statistic
815815
public static Statistics toParquetStatistics(
816816
org.apache.parquet.column.statistics.Statistics stats, int truncateLength) {
817817
Statistics formatStats = new Statistics();
818-
if (!stats.isEmpty()) {
819-
formatStats.setNull_count(stats.getNumNulls());
820-
if (stats.isNanCountSet()) {
821-
formatStats.setNan_count(stats.getNanCount());
822-
}
823-
}
824818
// Don't write stats larger than the max size rather than truncating. The
825819
// rationale is that some engines may use the minimum value in the page as
826820
// the true minimum for aggregations and there is no way to mark that a
827821
// value has been truncated and is a lower bound and not in the page.
828822
if (!stats.isEmpty() && withinLimit(stats, truncateLength)) {
823+
formatStats.setNull_count(stats.getNumNulls());
824+
if (stats.isNanCountSet()) {
825+
formatStats.setNan_count(stats.getNanCount());
826+
}
829827
if (stats.hasNonNullValue()) {
830828
byte[] min;
831829
byte[] max;

parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -878,7 +878,7 @@ private void testBinaryStats(StatsHelper helper) {
878878
}
879879
assertThat(formatStats.getNull_count()).as("Num nulls should match").isEqualTo(3004);
880880

881-
// min/max are not written because the values are too large, but null count is always written
881+
// convert to empty stats because the values are too large
882882
stats.setMinMaxFromBytes(max, max);
883883

884884
formatStats = helper.toParquetStatistics(stats);
@@ -891,7 +891,9 @@ private void testBinaryStats(StatsHelper helper) {
891891
assertThat(formatStats.isSetMax_value())
892892
.as("Max_value should not be set")
893893
.isFalse();
894-
assertThat(formatStats.getNull_count()).as("Num nulls should match").isEqualTo(3004);
894+
assertThat(formatStats.isSetNull_count())
895+
.as("Num nulls should not be set")
896+
.isFalse();
895897

896898
Statistics roundTripStats = ParquetMetadataConverter.fromParquetStatisticsInternal(
897899
Version.FULL_VERSION,
@@ -901,10 +903,7 @@ private void testBinaryStats(StatsHelper helper) {
901903

902904
assertThat(roundTripStats.isEmpty())
903905
.as("Round-trip stats should not be empty (null count is set)")
904-
.isFalse();
905-
assertThat(roundTripStats.getNumNulls())
906-
.as("Round-trip null count should match")
907-
.isEqualTo(3004);
906+
.isTrue();
908907
}
909908

910909
@Test

0 commit comments

Comments
 (0)