[MINOR]move hoodie hfile/orc reader/writer test cases from hudi-clien…

…t-common to hudi-common
apache · Jun 30, 2023 · 6c97beb · 6c97beb
1 parent 0a3a58e
commit 6c97beb
Show file tree

Hide file tree

Showing 20 changed files with 12 additions and 20 deletions.
diff --git a/.../storage/TestHoodieHFileReaderWriter.java → .../storage/TestHoodieHFileReaderWriter.java b/.../storage/TestHoodieHFileReaderWriter.java → .../storage/TestHoodieHFileReaderWriter.java
@@ -29,17 +29,16 @@
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hudi.common.bootstrap.index.HFileBootstrapIndex;
+import org.apache.hudi.common.config.HoodieStorageConfig;
 import org.apache.hudi.common.engine.TaskContextSupplier;
 import org.apache.hudi.common.fs.FSUtils;
 import org.apache.hudi.common.model.EmptyHoodieRecordPayload;
 import org.apache.hudi.common.model.HoodieAvroRecord;
 import org.apache.hudi.common.model.HoodieKey;
 import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.table.HoodieTableConfig;
 import org.apache.hudi.common.util.FileIOUtils;
 import org.apache.hudi.common.util.Option;
-import org.apache.hudi.config.HoodieIndexConfig;
-import org.apache.hudi.config.HoodieWriteConfig;
-
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -57,6 +56,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 import java.util.Spliterator;
 import java.util.Spliterators;
@@ -70,8 +70,8 @@
 import static org.apache.hudi.common.testutils.FileSystemTestUtils.RANDOM;
 import static org.apache.hudi.common.testutils.SchemaTestUtil.getSchemaFromResource;
 import static org.apache.hudi.common.util.CollectionUtils.toStream;
-import static org.apache.hudi.io.storage.HoodieHFileConfig.HFILE_COMPARATOR;
 import static org.apache.hudi.io.storage.HoodieAvroHFileReader.SCHEMA_KEY;
+import static org.apache.hudi.io.storage.HoodieHFileConfig.HFILE_COMPARATOR;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
@@ -94,20 +94,16 @@ protected Path getFilePath() {
   protected HoodieAvroHFileWriter createWriter(
       Schema avroSchema, boolean populateMetaFields) throws Exception {
     String instantTime = "000";
-    HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder()
-        .withPath(DUMMY_BASE_PATH)
-        .withIndexConfig(HoodieIndexConfig.newBuilder()
-            .bloomFilterNumEntries(1000).bloomFilterFPP(0.00001).build())
-        .withPopulateMetaFields(populateMetaFields)
-        .build();
     Configuration conf = new Configuration();
+    Properties props = new Properties();
+    props.setProperty(HoodieTableConfig.POPULATE_META_FIELDS.key(), Boolean.toString(populateMetaFields));
     TaskContextSupplier mockTaskContextSupplier = Mockito.mock(TaskContextSupplier.class);
     Supplier<Integer> partitionSupplier = Mockito.mock(Supplier.class);
     when(mockTaskContextSupplier.getPartitionIdSupplier()).thenReturn(partitionSupplier);
     when(partitionSupplier.get()).thenReturn(10);
 
     return (HoodieAvroHFileWriter)HoodieFileWriterFactory.getFileWriter(
-        instantTime, getFilePath(), conf, writeConfig.getStorageConfig(), avroSchema, mockTaskContextSupplier, writeConfig.getRecordMerger().getRecordType());
+        instantTime, getFilePath(), conf, HoodieStorageConfig.newBuilder().fromProperties(props).build(), avroSchema, mockTaskContextSupplier, HoodieRecord.HoodieRecordType.AVRO);
   }
 
   @Override

diff --git a/...io/storage/TestHoodieOrcReaderWriter.java → ...io/storage/TestHoodieOrcReaderWriter.java b/...io/storage/TestHoodieOrcReaderWriter.java → ...io/storage/TestHoodieOrcReaderWriter.java
@@ -18,17 +18,16 @@
 
 package org.apache.hudi.io.storage;
 
+import org.apache.avro.Schema;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
 import org.apache.hudi.avro.HoodieBloomFilterWriteSupport;
 import org.apache.hudi.common.bloom.BloomFilter;
 import org.apache.hudi.common.bloom.BloomFilterFactory;
 import org.apache.hudi.common.bloom.BloomFilterTypeCode;
-import org.apache.hudi.common.engine.TaskContextSupplier;
 import org.apache.hudi.common.config.HoodieStorageConfig;
+import org.apache.hudi.common.engine.TaskContextSupplier;
 import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType;
-
-import org.apache.avro.Schema;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
 import org.apache.orc.CompressionKind;
 import org.apache.orc.OrcFile;
 import org.apache.orc.Reader;

diff --git a/...o/storage/TestHoodieReaderWriterBase.java → ...o/storage/TestHoodieReaderWriterBase.java b/...o/storage/TestHoodieReaderWriterBase.java → ...o/storage/TestHoodieReaderWriterBase.java
@@ -49,10 +49,7 @@
 import java.util.stream.IntStream;
 
 import static org.apache.hudi.common.testutils.SchemaTestUtil.getSchemaFromResource;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.*;
 
 /**
  * Abstract class for unit tests of {@link HoodieFileReader} and {@link HoodieFileWriter}

diff --git a/.../test/resources/exampleEvolvedSchema.avsc → .../test/resources/exampleEvolvedSchema.avsc b/.../test/resources/exampleEvolvedSchema.avsc → .../test/resources/exampleEvolvedSchema.avsc
diff --git a/...rces/exampleEvolvedSchemaChangeOrder.avsc → ...rces/exampleEvolvedSchemaChangeOrder.avsc b/...rces/exampleEvolvedSchemaChangeOrder.avsc → ...rces/exampleEvolvedSchemaChangeOrder.avsc
diff --git a/...es/exampleEvolvedSchemaColumnRequire.avsc → ...es/exampleEvolvedSchemaColumnRequire.avsc b/...es/exampleEvolvedSchemaColumnRequire.avsc → ...es/exampleEvolvedSchemaColumnRequire.avsc
diff --git a/...urces/exampleEvolvedSchemaColumnType.avsc → ...urces/exampleEvolvedSchemaColumnType.avsc b/...urces/exampleEvolvedSchemaColumnType.avsc → ...urces/exampleEvolvedSchemaColumnType.avsc
diff --git a/...ces/exampleEvolvedSchemaDeleteColumn.avsc → ...ces/exampleEvolvedSchemaDeleteColumn.avsc b/...ces/exampleEvolvedSchemaDeleteColumn.avsc → ...ces/exampleEvolvedSchemaDeleteColumn.avsc
diff --git a/...mon/src/test/resources/exampleSchema.avsc → ...mon/src/test/resources/exampleSchema.avsc b/...mon/src/test/resources/exampleSchema.avsc → ...mon/src/test/resources/exampleSchema.avsc
diff --git a/...esources/exampleSchemaWithMetaFields.avsc → ...esources/exampleSchemaWithMetaFields.avsc b/...esources/exampleSchemaWithMetaFields.avsc → ...esources/exampleSchemaWithMetaFields.avsc
diff --git a/.../test/resources/exampleSchemaWithUDT.avsc → .../test/resources/exampleSchemaWithUDT.avsc b/.../test/resources/exampleSchemaWithUDT.avsc → .../test/resources/exampleSchemaWithUDT.avsc
diff --git a/...se_1_2_3_bootstrap_index_partitions.hfile → ...se_1_2_3_bootstrap_index_partitions.hfile b/...se_1_2_3_bootstrap_index_partitions.hfile → ...se_1_2_3_bootstrap_index_partitions.hfile
diff --git a/...urces/hudi_0_10_hbase_1_2_3_complex.hfile → ...urces/hudi_0_10_hbase_1_2_3_complex.hfile b/...urces/hudi_0_10_hbase_1_2_3_complex.hfile → ...urces/hudi_0_10_hbase_1_2_3_complex.hfile
diff --git a/...ources/hudi_0_10_hbase_1_2_3_simple.hfile → ...ources/hudi_0_10_hbase_1_2_3_simple.hfile b/...ources/hudi_0_10_hbase_1_2_3_simple.hfile → ...ources/hudi_0_10_hbase_1_2_3_simple.hfile
diff --git a/...se_2_4_9_bootstrap_index_partitions.hfile → ...se_2_4_9_bootstrap_index_partitions.hfile b/...se_2_4_9_bootstrap_index_partitions.hfile → ...se_2_4_9_bootstrap_index_partitions.hfile
diff --git a/...urces/hudi_0_11_hbase_2_4_9_complex.hfile → ...urces/hudi_0_11_hbase_2_4_9_complex.hfile b/...urces/hudi_0_11_hbase_2_4_9_complex.hfile → ...urces/hudi_0_11_hbase_2_4_9_complex.hfile
diff --git a/...ources/hudi_0_11_hbase_2_4_9_simple.hfile → ...ources/hudi_0_11_hbase_2_4_9_simple.hfile b/...ources/hudi_0_11_hbase_2_4_9_simple.hfile → ...ources/hudi_0_11_hbase_2_4_9_simple.hfile
diff --git a/...se_1_2_3_bootstrap_index_partitions.hfile → ...se_1_2_3_bootstrap_index_partitions.hfile b/...se_1_2_3_bootstrap_index_partitions.hfile → ...se_1_2_3_bootstrap_index_partitions.hfile
diff --git a/...ources/hudi_0_9_hbase_1_2_3_complex.hfile → ...ources/hudi_0_9_hbase_1_2_3_complex.hfile b/...ources/hudi_0_9_hbase_1_2_3_complex.hfile → ...ources/hudi_0_9_hbase_1_2_3_complex.hfile
diff --git a/...sources/hudi_0_9_hbase_1_2_3_simple.hfile → ...sources/hudi_0_9_hbase_1_2_3_simple.hfile b/...sources/hudi_0_9_hbase_1_2_3_simple.hfile → ...sources/hudi_0_9_hbase_1_2_3_simple.hfile