Skip to content

Commit

Permalink
make metadata metric test more robust
Browse files Browse the repository at this point in the history
  • Loading branch information
Prashant Singh committed Apr 23, 2023
1 parent 2c9a60e commit d2faa94
Show file tree
Hide file tree
Showing 4 changed files with 117 additions and 32 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/delta-conversion-ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ jobs:
runs-on: ubuntu-22.04
strategy:
matrix:
jvm: [ 8, 11, 17 ]
jvm: [8, 11, 17]
env:
SPARK_LOCAL_IP: localhost
steps:
Expand Down
73 changes: 58 additions & 15 deletions ...spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTableReadableMetrics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.nio.ByteBuffer;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.Files;
import org.apache.iceberg.PartitionSpec;
Expand All @@ -45,7 +46,6 @@
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.junit.After;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
Expand Down Expand Up @@ -77,6 +77,8 @@ public class TestMetadataTableReadableMetrics extends SparkTestBaseWithCatalog {
optional(8, "fixedCol", Types.FixedType.ofLength(3)),
optional(9, "binaryCol", Types.BinaryType.get()));

private DataFile dataFile;

public TestMetadataTableReadableMetrics() {
// only SparkCatalog supports metadata table sql queries
super(SparkCatalogConfig.HIVE);
Expand Down Expand Up @@ -123,8 +125,7 @@ private Table createPrimitiveTable() throws IOException {
createPrimitiveRecord(
false, 2, 2L, Float.NaN, 2.0D, new BigDecimal("2.00"), "2", null, null));

DataFile dataFile =
FileHelpers.writeDataFile(table, Files.localOutput(temp.newFile()), records);
dataFile = FileHelpers.writeDataFile(table, Files.localOutput(temp.newFile()), records);
table.newAppend().appendFile(dataFile).commit();
return table;
}
Expand All @@ -142,8 +143,7 @@ private void createNestedTable() throws IOException {
createNestedRecord(0L, 0.0),
createNestedRecord(1L, Double.NaN),
createNestedRecord(null, null));
DataFile dataFile =
FileHelpers.writeDataFile(table, Files.localOutput(temp.newFile()), records);
dataFile = FileHelpers.writeDataFile(table, Files.localOutput(temp.newFile()), records);
table.newAppend().appendFile(dataFile).commit();
}

Expand Down Expand Up @@ -191,33 +191,76 @@ private GenericRecord createNestedRecord(Long longCol, Double doubleCol) {
}

@Test
@Ignore
public void testPrimitiveColumns() throws Exception {
createPrimitiveTable();
Map<Integer, Long> columSizeStats = dataFile.columnSizes();

Object[] binaryCol =
row(
59L,
columSizeStats.get(PRIMITIVE_SCHEMA.findField("binaryCol").fieldId()),
4L,
2L,
null,
Base64.getDecoder().decode("1111"),
Base64.getDecoder().decode("2222"));
Object[] booleanCol = row(44L, 4L, 0L, null, false, true);
Object[] decimalCol = row(97L, 4L, 1L, null, new BigDecimal("1.00"), new BigDecimal("2.00"));
Object[] doubleCol = row(99L, 4L, 0L, 1L, 1.0D, 2.0D);
Object[] booleanCol =
row(
columSizeStats.get(PRIMITIVE_SCHEMA.findField("booleanCol").fieldId()),
4L,
0L,
null,
false,
true);
Object[] decimalCol =
row(
columSizeStats.get(PRIMITIVE_SCHEMA.findField("decimalCol").fieldId()),
4L,
1L,
null,
new BigDecimal("1.00"),
new BigDecimal("2.00"));
Object[] doubleCol =
row(
columSizeStats.get(PRIMITIVE_SCHEMA.findField("doubleCol").fieldId()),
4L,
0L,
1L,
1.0D,
2.0D);
Object[] fixedCol =
row(
55L,
columSizeStats.get(PRIMITIVE_SCHEMA.findField("fixedCol").fieldId()),
4L,
2L,
null,
Base64.getDecoder().decode("1111"),
Base64.getDecoder().decode("2222"));
Object[] floatCol = row(90L, 4L, 0L, 2L, 0f, 0f);
Object[] intCol = row(91L, 4L, 0L, null, 1, 2);
Object[] longCol = row(91L, 4L, 0L, null, 1L, 2L);
Object[] stringCol = row(99L, 4L, 0L, null, "1", "2");
Object[] floatCol =
row(
columSizeStats.get(PRIMITIVE_SCHEMA.findField("floatCol").fieldId()),
4L,
0L,
2L,
0f,
0f);
Object[] intCol =
row(columSizeStats.get(PRIMITIVE_SCHEMA.findField("intCol").fieldId()), 4L, 0L, null, 1, 2);
Object[] longCol =
row(
columSizeStats.get(PRIMITIVE_SCHEMA.findField("longCol").fieldId()),
4L,
0L,
null,
1L,
2L);
Object[] stringCol =
row(
columSizeStats.get(PRIMITIVE_SCHEMA.findField("stringCol").fieldId()),
4L,
0L,
null,
"1",
"2");

Object[] metrics =
row(
Expand Down
1 change: 0 additions & 1 deletion spark/v3.4/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,6 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
}

test {
useJUnitPlatform()
jvmArgs += project.property('extraJvmArgs')
}
}
Expand Down
73 changes: 58 additions & 15 deletions ...spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTableReadableMetrics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.nio.ByteBuffer;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.Files;
import org.apache.iceberg.PartitionSpec;
Expand All @@ -45,7 +46,6 @@
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.junit.After;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
Expand Down Expand Up @@ -77,6 +77,8 @@ public class TestMetadataTableReadableMetrics extends SparkTestBaseWithCatalog {
optional(8, "fixedCol", Types.FixedType.ofLength(3)),
optional(9, "binaryCol", Types.BinaryType.get()));

private DataFile dataFile;

public TestMetadataTableReadableMetrics() {
// only SparkCatalog supports metadata table sql queries
super(SparkCatalogConfig.HIVE);
Expand Down Expand Up @@ -123,8 +125,7 @@ private Table createPrimitiveTable() throws IOException {
createPrimitiveRecord(
false, 2, 2L, Float.NaN, 2.0D, new BigDecimal("2.00"), "2", null, null));

DataFile dataFile =
FileHelpers.writeDataFile(table, Files.localOutput(temp.newFile()), records);
dataFile = FileHelpers.writeDataFile(table, Files.localOutput(temp.newFile()), records);
table.newAppend().appendFile(dataFile).commit();
return table;
}
Expand All @@ -142,8 +143,7 @@ private void createNestedTable() throws IOException {
createNestedRecord(0L, 0.0),
createNestedRecord(1L, Double.NaN),
createNestedRecord(null, null));
DataFile dataFile =
FileHelpers.writeDataFile(table, Files.localOutput(temp.newFile()), records);
dataFile = FileHelpers.writeDataFile(table, Files.localOutput(temp.newFile()), records);
table.newAppend().appendFile(dataFile).commit();
}

Expand Down Expand Up @@ -191,33 +191,76 @@ private GenericRecord createNestedRecord(Long longCol, Double doubleCol) {
}

@Test
@Ignore
public void testPrimitiveColumns() throws Exception {
createPrimitiveTable();
Map<Integer, Long> columSizeStats = dataFile.columnSizes();

Object[] binaryCol =
row(
59L,
columSizeStats.get(PRIMITIVE_SCHEMA.findField("binaryCol").fieldId()),
4L,
2L,
null,
Base64.getDecoder().decode("1111"),
Base64.getDecoder().decode("2222"));
Object[] booleanCol = row(44L, 4L, 0L, null, false, true);
Object[] decimalCol = row(97L, 4L, 1L, null, new BigDecimal("1.00"), new BigDecimal("2.00"));
Object[] doubleCol = row(99L, 4L, 0L, 1L, 1.0D, 2.0D);
Object[] booleanCol =
row(
columSizeStats.get(PRIMITIVE_SCHEMA.findField("booleanCol").fieldId()),
4L,
0L,
null,
false,
true);
Object[] decimalCol =
row(
columSizeStats.get(PRIMITIVE_SCHEMA.findField("decimalCol").fieldId()),
4L,
1L,
null,
new BigDecimal("1.00"),
new BigDecimal("2.00"));
Object[] doubleCol =
row(
columSizeStats.get(PRIMITIVE_SCHEMA.findField("doubleCol").fieldId()),
4L,
0L,
1L,
1.0D,
2.0D);
Object[] fixedCol =
row(
55L,
columSizeStats.get(PRIMITIVE_SCHEMA.findField("fixedCol").fieldId()),
4L,
2L,
null,
Base64.getDecoder().decode("1111"),
Base64.getDecoder().decode("2222"));
Object[] floatCol = row(90L, 4L, 0L, 2L, 0f, 0f);
Object[] intCol = row(91L, 4L, 0L, null, 1, 2);
Object[] longCol = row(91L, 4L, 0L, null, 1L, 2L);
Object[] stringCol = row(99L, 4L, 0L, null, "1", "2");
Object[] floatCol =
row(
columSizeStats.get(PRIMITIVE_SCHEMA.findField("floatCol").fieldId()),
4L,
0L,
2L,
0f,
0f);
Object[] intCol =
row(columSizeStats.get(PRIMITIVE_SCHEMA.findField("intCol").fieldId()), 4L, 0L, null, 1, 2);
Object[] longCol =
row(
columSizeStats.get(PRIMITIVE_SCHEMA.findField("longCol").fieldId()),
4L,
0L,
null,
1L,
2L);
Object[] stringCol =
row(
columSizeStats.get(PRIMITIVE_SCHEMA.findField("stringCol").fieldId()),
4L,
0L,
null,
"1",
"2");

Object[] metrics =
row(
Expand Down

0 comments on commit d2faa94

Please sign in to comment.