Skip to content

Commit

Permalink
ORC: Switch tests to JUnit5 (#7954)
Browse files Browse the repository at this point in the history
  • Loading branch information
@skytin1004
skytin1004 authored Jul 3, 2023
1 parent d5ec743 commit ffdfb97
Show file tree
Hide file tree
Showing 12 changed files with 281 additions and 242 deletions.
3 changes: 3 additions & 0 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -697,6 +697,9 @@ project(':iceberg-hive-metastore') {
}

project(':iceberg-orc') {
test {
useJUnitPlatform()
}
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
api project(':iceberg-api')
Expand Down
20 changes: 8 additions & 12 deletions orc/src/test/java/org/apache/iceberg/orc/TestBloomFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,8 @@
import org.apache.orc.impl.RecordReaderImpl;
import org.apache.orc.impl.WriterImpl;
import org.assertj.core.api.Assertions;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

public class TestBloomFilter {
private static final Schema DATA_SCHEMA =
Expand All @@ -54,12 +52,11 @@ public class TestBloomFilter {
required(101, "name", Types.StringType.get()),
required(102, "price", Types.DoubleType.get()));

@Rule public TemporaryFolder temp = new TemporaryFolder();
@TempDir private File testFile;

@Test
public void testWriteOption() throws Exception {
File testFile = temp.newFile();
Assert.assertTrue("Delete should succeed", testFile.delete());
Assertions.assertThat(testFile.delete()).as("Delete should succeed").isTrue();

OutputFile outFile = Files.localOutput(testFile);
try (FileAppender<Record> writer =
Expand All @@ -84,9 +81,9 @@ public void testWriteOption() throws Exception {
double bloomFilterFpp = (double) bloomFilterFppField.get(orcWriter);

// Validate whether the bloom filters are set in ORC SDK or not
Assert.assertTrue(bloomFilterColumns[1]);
Assert.assertTrue(bloomFilterColumns[2]);
Assert.assertEquals(0.04, bloomFilterFpp, 1e-15);
Assertions.assertThat(bloomFilterColumns[1]).isTrue();
Assertions.assertThat(bloomFilterColumns[2]).isTrue();
Assertions.assertThat(bloomFilterFpp).isCloseTo(0.04, Assertions.offset(1e-15));

Record recordTemplate = GenericRecord.create(DATA_SCHEMA);
Record record1 = recordTemplate.copy("id", 1L, "name", "foo", "price", 1.0);
Expand Down Expand Up @@ -132,8 +129,7 @@ public void testWriteOption() throws Exception {

@Test
public void testInvalidFppOption() throws Exception {
File testFile = temp.newFile();
Assert.assertTrue("Delete should succeed", testFile.delete());
Assertions.assertThat(testFile.delete()).as("Delete should succeed").isTrue();

Assertions.assertThatThrownBy(
() ->
Expand Down
72 changes: 42 additions & 30 deletions orc/src/test/java/org/apache/iceberg/orc/TestBuildOrcProjection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,12 @@

import static org.apache.iceberg.types.Types.NestedField.optional;
import static org.apache.iceberg.types.Types.NestedField.required;
import static org.junit.Assert.assertEquals;

import org.apache.iceberg.Schema;
import org.apache.iceberg.types.Types;
import org.apache.orc.TypeDescription;
import org.assertj.core.api.Assertions;
import org.junit.Test;
import org.junit.jupiter.api.Test;

/** Test projections on ORC types. */
public class TestBuildOrcProjection {
Expand All @@ -39,11 +38,13 @@ public void testProjectionPrimitiveNoOp() {

// Original mapping (stored in ORC)
TypeDescription orcSchema = ORCSchemaUtil.convert(originalSchema);
assertEquals(2, orcSchema.getChildren().size());
assertEquals(1, orcSchema.findSubtype("a").getId());
assertEquals(TypeDescription.Category.INT, orcSchema.findSubtype("a").getCategory());
assertEquals(2, orcSchema.findSubtype("b").getId());
assertEquals(TypeDescription.Category.STRING, orcSchema.findSubtype("b").getCategory());
Assertions.assertThat(orcSchema.getChildren()).hasSize(2);
Assertions.assertThat(orcSchema.findSubtype("a").getId()).isEqualTo(1);
Assertions.assertThat(orcSchema.findSubtype("a").getCategory())
.isEqualTo(TypeDescription.Category.INT);
Assertions.assertThat(orcSchema.findSubtype("b").getId()).isEqualTo(2);
Assertions.assertThat(orcSchema.findSubtype("b").getCategory())
.isEqualTo(TypeDescription.Category.STRING);
}

@Test
Expand All @@ -63,11 +64,13 @@ public void testProjectionPrimitive() {
);

TypeDescription newOrcSchema = ORCSchemaUtil.buildOrcProjection(evolveSchema, orcSchema);
assertEquals(2, newOrcSchema.getChildren().size());
assertEquals(1, newOrcSchema.findSubtype("b").getId());
assertEquals(TypeDescription.Category.STRING, newOrcSchema.findSubtype("b").getCategory());
assertEquals(2, newOrcSchema.findSubtype("c_r3").getId());
assertEquals(TypeDescription.Category.DATE, newOrcSchema.findSubtype("c_r3").getCategory());
Assertions.assertThat(newOrcSchema.getChildren()).hasSize(2);
Assertions.assertThat(newOrcSchema.findSubtype("b").getId()).isEqualTo(1);
Assertions.assertThat(newOrcSchema.findSubtype("b").getCategory())
.isEqualTo(TypeDescription.Category.STRING);
Assertions.assertThat(newOrcSchema.findSubtype("c_r3").getId()).isEqualTo(2);
Assertions.assertThat(newOrcSchema.findSubtype("c_r3").getCategory())
.isEqualTo(TypeDescription.Category.DATE);
}

@Test
Expand All @@ -81,13 +84,16 @@ public void testProjectionNestedNoOp() {
TypeDescription orcSchema = ORCSchemaUtil.convert(originalSchema);

TypeDescription newOrcSchema = ORCSchemaUtil.buildOrcProjection(originalSchema, orcSchema);
assertEquals(1, newOrcSchema.getChildren().size());
assertEquals(TypeDescription.Category.STRUCT, newOrcSchema.findSubtype("a").getCategory());
Assertions.assertThat(newOrcSchema.getChildren()).hasSize(1);
Assertions.assertThat(newOrcSchema.findSubtype("a").getCategory())
.isEqualTo(TypeDescription.Category.STRUCT);
TypeDescription nestedCol = newOrcSchema.findSubtype("a");
assertEquals(2, nestedCol.findSubtype("b").getId());
assertEquals(TypeDescription.Category.STRING, nestedCol.findSubtype("b").getCategory());
assertEquals(3, nestedCol.findSubtype("c").getId());
assertEquals(TypeDescription.Category.DATE, nestedCol.findSubtype("c").getCategory());
Assertions.assertThat(nestedCol.findSubtype("b").getId()).isEqualTo(2);
Assertions.assertThat(nestedCol.findSubtype("b").getCategory())
.isEqualTo(TypeDescription.Category.STRING);
Assertions.assertThat(nestedCol.findSubtype("c").getId()).isEqualTo(3);
Assertions.assertThat(nestedCol.findSubtype("c").getCategory())
.isEqualTo(TypeDescription.Category.DATE);
}

@Test
Expand All @@ -107,13 +113,16 @@ public void testProjectionNested() {
Schema evolveSchema = new Schema(optional(1, "aa", newNestedStructType));

TypeDescription newOrcSchema = ORCSchemaUtil.buildOrcProjection(evolveSchema, orcSchema);
assertEquals(1, newOrcSchema.getChildren().size());
assertEquals(TypeDescription.Category.STRUCT, newOrcSchema.findSubtype("a").getCategory());
Assertions.assertThat(newOrcSchema.getChildren()).hasSize(1);
Assertions.assertThat(newOrcSchema.findSubtype("a").getCategory())
.isEqualTo(TypeDescription.Category.STRUCT);
TypeDescription nestedCol = newOrcSchema.findSubtype("a");
assertEquals(2, nestedCol.findSubtype("c").getId());
assertEquals(TypeDescription.Category.DATE, nestedCol.findSubtype("c").getCategory());
assertEquals(3, nestedCol.findSubtype("b").getId());
assertEquals(TypeDescription.Category.STRING, nestedCol.findSubtype("b").getCategory());
Assertions.assertThat(nestedCol.findSubtype("c").getId()).isEqualTo(2);
Assertions.assertThat(nestedCol.findSubtype("c").getCategory())
.isEqualTo(TypeDescription.Category.DATE);
Assertions.assertThat(nestedCol.findSubtype("b").getId()).isEqualTo(3);
Assertions.assertThat(nestedCol.findSubtype("b").getCategory())
.isEqualTo(TypeDescription.Category.STRING);
}

@Test
Expand All @@ -127,13 +136,16 @@ public void testEvolutionAddContainerField() {
optional(2, "b", Types.StructType.of(required(3, "c", Types.LongType.get()))));

TypeDescription newOrcSchema = ORCSchemaUtil.buildOrcProjection(evolvedSchema, baseOrcSchema);
assertEquals(2, newOrcSchema.getChildren().size());
assertEquals(TypeDescription.Category.INT, newOrcSchema.findSubtype("a").getCategory());
assertEquals(2, newOrcSchema.findSubtype("b_r2").getId());
assertEquals(TypeDescription.Category.STRUCT, newOrcSchema.findSubtype("b_r2").getCategory());
Assertions.assertThat(newOrcSchema.getChildren()).hasSize(2);
Assertions.assertThat(newOrcSchema.findSubtype("a").getCategory())
.isEqualTo(TypeDescription.Category.INT);
Assertions.assertThat(newOrcSchema.findSubtype("b_r2").getId()).isEqualTo(2);
Assertions.assertThat(newOrcSchema.findSubtype("b_r2").getCategory())
.isEqualTo(TypeDescription.Category.STRUCT);
TypeDescription nestedCol = newOrcSchema.findSubtype("b_r2");
assertEquals(3, nestedCol.findSubtype("c_r3").getId());
assertEquals(TypeDescription.Category.LONG, nestedCol.findSubtype("c_r3").getCategory());
Assertions.assertThat(nestedCol.findSubtype("c_r3").getId()).isEqualTo(3);
Assertions.assertThat(nestedCol.findSubtype("c_r3").getCategory())
.isEqualTo(TypeDescription.Category.LONG);
}

@Test
Expand Down
73 changes: 53 additions & 20 deletions orc/src/test/java/org/apache/iceberg/orc/TestEstimateOrcAvgWidthVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
import org.apache.iceberg.Schema;
import org.apache.iceberg.types.Types;
import org.apache.orc.TypeDescription;
import org.junit.Assert;
import org.junit.Test;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.Test;

public class TestEstimateOrcAvgWidthVisitor {

Expand Down Expand Up @@ -78,125 +78,156 @@ public void testEstimateIntegerWidth() {
Schema integerSchema = new Schema(ID_FIELD);
TypeDescription integerOrcSchema = ORCSchemaUtil.convert(integerSchema);
long estimateLength = getEstimateLength(integerOrcSchema);
Assert.assertEquals("Estimated average length of integer must be 8.", 8, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of integer must be 8.")
.isEqualTo(8);
}

@Test
public void testEstimateStringWidth() {
Schema stringSchema = new Schema(DATA_FIELD);
TypeDescription stringOrcSchema = ORCSchemaUtil.convert(stringSchema);
long estimateLength = getEstimateLength(stringOrcSchema);
Assert.assertEquals("Estimated average length of string must be 128.", 128, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of string must be 128.")
.isEqualTo(128);
}

@Test
public void testEstimateFloatWidth() {
Schema floatSchema = new Schema(FLOAT_FIELD);
TypeDescription floatOrcSchema = ORCSchemaUtil.convert(floatSchema);
long estimateLength = getEstimateLength(floatOrcSchema);
Assert.assertEquals("Estimated average length of float must be 8.", 8, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of float must be 8.")
.isEqualTo(8);
}

@Test
public void testEstimateDoubleWidth() {
Schema doubleSchema = new Schema(DOUBLE_FIELD);
TypeDescription doubleOrcSchema = ORCSchemaUtil.convert(doubleSchema);
long estimateLength = getEstimateLength(doubleOrcSchema);
Assert.assertEquals("Estimated average length of double must be 8.", 8, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of double must be 8.")
.isEqualTo(8);
}

@Test
public void testEstimateDecimalWidth() {
Schema decimalSchema = new Schema(DECIMAL_FIELD);
TypeDescription decimalOrcSchema = ORCSchemaUtil.convert(decimalSchema);
long estimateLength = getEstimateLength(decimalOrcSchema);
Assert.assertEquals("Estimated average length of decimal must be 7.", 7, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of decimal must be 7.")
.isEqualTo(7);
}

@Test
public void testEstimateFixedWidth() {
Schema fixedSchema = new Schema(FIXED_FIELD);
TypeDescription fixedOrcSchema = ORCSchemaUtil.convert(fixedSchema);
long estimateLength = getEstimateLength(fixedOrcSchema);
Assert.assertEquals("Estimated average length of fixed must be 128.", 128, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of fixed must be 128.")
.isEqualTo(128);
}

@Test
public void testEstimateBinaryWidth() {
Schema binarySchema = new Schema(BINARY_FIELD);
TypeDescription binaryOrcSchema = ORCSchemaUtil.convert(binarySchema);
long estimateLength = getEstimateLength(binaryOrcSchema);
Assert.assertEquals("Estimated average length of binary must be 128.", 128, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of binary must be 128.")
.isEqualTo(128);
}

@Test
public void testEstimateListWidth() {
Schema listSchema = new Schema(FLOAT_LIST_FIELD);
TypeDescription listOrcSchema = ORCSchemaUtil.convert(listSchema);
long estimateLength = getEstimateLength(listOrcSchema);
Assert.assertEquals("Estimated average length of list must be 8.", 8, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of list must be 8.")
.isEqualTo(8);
}

@Test
public void testEstimateLongWidth() {
Schema longSchema = new Schema(LONG_FIELD);
TypeDescription longOrcSchema = ORCSchemaUtil.convert(longSchema);
long estimateLength = getEstimateLength(longOrcSchema);
Assert.assertEquals("Estimated average length of long must be 8.", 8, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of long must be 8.")
.isEqualTo(8);
}

@Test
public void testEstimateBooleanWidth() {
Schema booleanSchema = new Schema(BOOLEAN_FIELD);
TypeDescription booleanOrcSchema = ORCSchemaUtil.convert(booleanSchema);
long estimateLength = getEstimateLength(booleanOrcSchema);
Assert.assertEquals("Estimated average length of boolean must be 8.", 8, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of boolean must be 8.")
.isEqualTo(8);
}

@Test
public void testEstimateTimestampWidth() {
Schema timestampZoneSchema = new Schema(TIMESTAMP_ZONE_FIELD);
TypeDescription timestampZoneOrcSchema = ORCSchemaUtil.convert(timestampZoneSchema);
long estimateLength = getEstimateLength(timestampZoneOrcSchema);
Assert.assertEquals(
"Estimated average length of timestamps with zone must be 12.", 12, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of timestamps with zone must be 12.")
.isEqualTo(12);

Schema timestampSchema = new Schema(TIMESTAMP_FIELD);
TypeDescription timestampOrcSchema = ORCSchemaUtil.convert(timestampSchema);
estimateLength = getEstimateLength(timestampOrcSchema);
Assert.assertEquals("Estimated average length of timestamp must be 12.", 12, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of timestamp must be 12.")
.isEqualTo(12);
}

@Test
public void testEstimateDateWidth() {
Schema dateSchema = new Schema(DATE_FIELD);
TypeDescription dateOrcSchema = ORCSchemaUtil.convert(dateSchema);
long estimateLength = getEstimateLength(dateOrcSchema);
Assert.assertEquals("Estimated average length of date must be 8.", 8, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of date must be 8.")
.isEqualTo(8);
}

@Test
public void testEstimateUUIDWidth() {
Schema uuidSchema = new Schema(UUID_FIELD);
TypeDescription uuidOrcSchema = ORCSchemaUtil.convert(uuidSchema);
long estimateLength = getEstimateLength(uuidOrcSchema);
Assert.assertEquals("Estimated average length of uuid must be 128.", 128, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of uuid must be 128.")
.isEqualTo(128);
}

@Test
public void testEstimateMapWidth() {
Schema mapSchema = new Schema(MAP_FIELD_1);
TypeDescription mapOrcSchema = ORCSchemaUtil.convert(mapSchema);
long estimateLength = getEstimateLength(mapOrcSchema);
Assert.assertEquals("Estimated average length of map must be 136.", 136, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of map must be 136.")
.isEqualTo(136);
}

@Test
public void testEstimateStructWidth() {
Schema structSchema = new Schema(STRUCT_FIELD);
TypeDescription structOrcSchema = ORCSchemaUtil.convert(structSchema);
long estimateLength = getEstimateLength(structOrcSchema);
Assert.assertEquals("Estimated average length of struct must be 28.", 28, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of struct must be 28.")
.isEqualTo(28);
}

@Test
Expand All @@ -217,7 +248,9 @@ public void testEstimateFullWidth() {
STRUCT_FIELD);
TypeDescription fullOrcSchema = ORCSchemaUtil.convert(fullSchema);
long estimateLength = getEstimateLength(fullOrcSchema);
Assert.assertEquals("Estimated average length of the row must be 611.", 611, estimateLength);
Assertions.assertThat(estimateLength)
.as("Estimated average length of the row must be 611.")
.isEqualTo(611);
}

private Integer getEstimateLength(TypeDescription orcSchemaWithDate) {
Expand Down
Loading

0 comments on commit ffdfb97

Please sign in to comment.