File tree Expand file tree Collapse file tree
main/java/org/apache/parquet/format/converter
test/java/org/apache/parquet/format/converter Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -815,17 +815,15 @@ public static Statistics toParquetStatistics(org.apache.parquet.column.statistic
815815 public static Statistics toParquetStatistics (
816816 org .apache .parquet .column .statistics .Statistics stats , int truncateLength ) {
817817 Statistics formatStats = new Statistics ();
818- if (!stats .isEmpty ()) {
819- formatStats .setNull_count (stats .getNumNulls ());
820- if (stats .isNanCountSet ()) {
821- formatStats .setNan_count (stats .getNanCount ());
822- }
823- }
824818 // Don't write stats larger than the max size rather than truncating. The
825819 // rationale is that some engines may use the minimum value in the page as
826820 // the true minimum for aggregations and there is no way to mark that a
827821 // value has been truncated and is a lower bound and not in the page.
828822 if (!stats .isEmpty () && withinLimit (stats , truncateLength )) {
823+ formatStats .setNull_count (stats .getNumNulls ());
824+ if (stats .isNanCountSet ()) {
825+ formatStats .setNan_count (stats .getNanCount ());
826+ }
829827 if (stats .hasNonNullValue ()) {
830828 byte [] min ;
831829 byte [] max ;
Original file line number Diff line number Diff line change @@ -878,7 +878,7 @@ private void testBinaryStats(StatsHelper helper) {
878878 }
879879 assertThat (formatStats .getNull_count ()).as ("Num nulls should match" ).isEqualTo (3004 );
880880
881- // min/max are not written because the values are too large, but null count is always written
881+ // convert to empty stats because the values are too large
882882 stats .setMinMaxFromBytes (max , max );
883883
884884 formatStats = helper .toParquetStatistics (stats );
@@ -891,7 +891,9 @@ private void testBinaryStats(StatsHelper helper) {
891891 assertThat (formatStats .isSetMax_value ())
892892 .as ("Max_value should not be set" )
893893 .isFalse ();
894- assertThat (formatStats .getNull_count ()).as ("Num nulls should match" ).isEqualTo (3004 );
894+ assertThat (formatStats .isSetNull_count ())
895+ .as ("Num nulls should not be set" )
896+ .isFalse ();
895897
896898 Statistics roundTripStats = ParquetMetadataConverter .fromParquetStatisticsInternal (
897899 Version .FULL_VERSION ,
@@ -901,10 +903,7 @@ private void testBinaryStats(StatsHelper helper) {
901903
902904 assertThat (roundTripStats .isEmpty ())
903905 .as ("Round-trip stats should not be empty (null count is set)" )
904- .isFalse ();
905- assertThat (roundTripStats .getNumNulls ())
906- .as ("Round-trip null count should match" )
907- .isEqualTo (3004 );
906+ .isTrue ();
908907 }
909908
910909 @ Test
You can’t perform that action at this time.
0 commit comments