From f26d06b7eb099e698fe7058f3ffba327d4ae5c7f Mon Sep 17 00:00:00 2001 From: Mulavar <978007503@qq.com> Date: Fri, 30 Jun 2023 11:38:52 +0800 Subject: [PATCH] [MINOR]move hoodie hfile/orc reader/writer test cases from hudi-client-common to hudi-common --- .../storage/TestHoodieHFileReaderWriter.java | 18 +++++++----------- .../io/storage/TestHoodieOrcReaderWriter.java | 9 ++++----- .../storage/TestHoodieReaderWriterBase.java | 0 .../test/resources/exampleEvolvedSchema.avsc | 0 .../exampleEvolvedSchemaChangeOrder.avsc | 0 .../exampleEvolvedSchemaColumnRequire.avsc | 0 .../exampleEvolvedSchemaColumnType.avsc | 0 .../exampleEvolvedSchemaDeleteColumn.avsc | 0 .../src/test/resources/exampleSchema.avsc | 0 .../exampleSchemaWithMetaFields.avsc | 0 .../test/resources/exampleSchemaWithUDT.avsc | 0 ...ase_1_2_3_bootstrap_index_partitions.hfile | Bin .../hudi_0_10_hbase_1_2_3_complex.hfile | Bin .../hudi_0_10_hbase_1_2_3_simple.hfile | Bin ...ase_2_4_9_bootstrap_index_partitions.hfile | Bin .../hudi_0_11_hbase_2_4_9_complex.hfile | Bin .../hudi_0_11_hbase_2_4_9_simple.hfile | Bin ...ase_1_2_3_bootstrap_index_partitions.hfile | Bin .../hudi_0_9_hbase_1_2_3_complex.hfile | Bin .../hudi_0_9_hbase_1_2_3_simple.hfile | Bin 20 files changed, 11 insertions(+), 16 deletions(-) rename {hudi-client/hudi-client-common => hudi-common}/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java (97%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java (99%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/java/org/apache/hudi/io/storage/TestHoodieReaderWriterBase.java (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/exampleEvolvedSchema.avsc (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/exampleEvolvedSchemaChangeOrder.avsc (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/exampleEvolvedSchemaColumnRequire.avsc (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/exampleEvolvedSchemaColumnType.avsc (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/exampleEvolvedSchemaDeleteColumn.avsc (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/exampleSchema.avsc (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/exampleSchemaWithMetaFields.avsc (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/exampleSchemaWithUDT.avsc (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/hudi_0_10_hbase_1_2_3_bootstrap_index_partitions.hfile (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/hudi_0_10_hbase_1_2_3_complex.hfile (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/hudi_0_10_hbase_1_2_3_simple.hfile (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/hudi_0_11_hbase_2_4_9_bootstrap_index_partitions.hfile (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/hudi_0_11_hbase_2_4_9_complex.hfile (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/hudi_0_11_hbase_2_4_9_simple.hfile (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/hudi_0_9_hbase_1_2_3_bootstrap_index_partitions.hfile (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/hudi_0_9_hbase_1_2_3_complex.hfile (100%) rename {hudi-client/hudi-client-common => hudi-common}/src/test/resources/hudi_0_9_hbase_1_2_3_simple.hfile (100%) diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java b/hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java similarity index 97% rename from hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java rename to hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java index 90ad0fe1a748..fb52090db856 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java +++ b/hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java @@ -29,17 +29,16 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hudi.common.bootstrap.index.HFileBootstrapIndex; +import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.engine.TaskContextSupplier; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.EmptyHoodieRecordPayload; import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.Option; -import org.apache.hudi.config.HoodieIndexConfig; -import org.apache.hudi.config.HoodieWriteConfig; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -57,6 +56,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.Spliterator; import java.util.Spliterators; @@ -70,8 +70,8 @@ import static org.apache.hudi.common.testutils.FileSystemTestUtils.RANDOM; import static org.apache.hudi.common.testutils.SchemaTestUtil.getSchemaFromResource; import static org.apache.hudi.common.util.CollectionUtils.toStream; -import static org.apache.hudi.io.storage.HoodieHFileConfig.HFILE_COMPARATOR; import static org.apache.hudi.io.storage.HoodieAvroHFileReader.SCHEMA_KEY; +import static org.apache.hudi.io.storage.HoodieHFileConfig.HFILE_COMPARATOR; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -94,20 +94,16 @@ protected Path getFilePath() { protected HoodieAvroHFileWriter createWriter( Schema avroSchema, boolean populateMetaFields) throws Exception { String instantTime = "000"; - HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() - .withPath(DUMMY_BASE_PATH) - .withIndexConfig(HoodieIndexConfig.newBuilder() - .bloomFilterNumEntries(1000).bloomFilterFPP(0.00001).build()) - .withPopulateMetaFields(populateMetaFields) - .build(); Configuration conf = new Configuration(); + Properties props = new Properties(); + props.setProperty(HoodieTableConfig.POPULATE_META_FIELDS.key(), Boolean.toString(populateMetaFields)); TaskContextSupplier mockTaskContextSupplier = Mockito.mock(TaskContextSupplier.class); Supplier partitionSupplier = Mockito.mock(Supplier.class); when(mockTaskContextSupplier.getPartitionIdSupplier()).thenReturn(partitionSupplier); when(partitionSupplier.get()).thenReturn(10); return (HoodieAvroHFileWriter)HoodieFileWriterFactory.getFileWriter( - instantTime, getFilePath(), conf, writeConfig.getStorageConfig(), avroSchema, mockTaskContextSupplier, writeConfig.getRecordMerger().getRecordType()); + instantTime, getFilePath(), conf, HoodieStorageConfig.newBuilder().fromProperties(props).build(), avroSchema, mockTaskContextSupplier, HoodieRecord.HoodieRecordType.AVRO); } @Override diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java b/hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java similarity index 99% rename from hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java rename to hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java index 438024d2f268..de3f71b78276 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java +++ b/hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java @@ -18,17 +18,16 @@ package org.apache.hudi.io.storage; +import org.apache.avro.Schema; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.HoodieBloomFilterWriteSupport; import org.apache.hudi.common.bloom.BloomFilter; import org.apache.hudi.common.bloom.BloomFilterFactory; import org.apache.hudi.common.bloom.BloomFilterTypeCode; -import org.apache.hudi.common.engine.TaskContextSupplier; import org.apache.hudi.common.config.HoodieStorageConfig; +import org.apache.hudi.common.engine.TaskContextSupplier; import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType; - -import org.apache.avro.Schema; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.orc.CompressionKind; import org.apache.orc.OrcFile; import org.apache.orc.Reader; diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieReaderWriterBase.java b/hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieReaderWriterBase.java similarity index 100% rename from hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieReaderWriterBase.java rename to hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieReaderWriterBase.java diff --git a/hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchema.avsc b/hudi-common/src/test/resources/exampleEvolvedSchema.avsc similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchema.avsc rename to hudi-common/src/test/resources/exampleEvolvedSchema.avsc diff --git a/hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaChangeOrder.avsc b/hudi-common/src/test/resources/exampleEvolvedSchemaChangeOrder.avsc similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaChangeOrder.avsc rename to hudi-common/src/test/resources/exampleEvolvedSchemaChangeOrder.avsc diff --git a/hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaColumnRequire.avsc b/hudi-common/src/test/resources/exampleEvolvedSchemaColumnRequire.avsc similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaColumnRequire.avsc rename to hudi-common/src/test/resources/exampleEvolvedSchemaColumnRequire.avsc diff --git a/hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaColumnType.avsc b/hudi-common/src/test/resources/exampleEvolvedSchemaColumnType.avsc similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaColumnType.avsc rename to hudi-common/src/test/resources/exampleEvolvedSchemaColumnType.avsc diff --git a/hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaDeleteColumn.avsc b/hudi-common/src/test/resources/exampleEvolvedSchemaDeleteColumn.avsc similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaDeleteColumn.avsc rename to hudi-common/src/test/resources/exampleEvolvedSchemaDeleteColumn.avsc diff --git a/hudi-client/hudi-client-common/src/test/resources/exampleSchema.avsc b/hudi-common/src/test/resources/exampleSchema.avsc similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/exampleSchema.avsc rename to hudi-common/src/test/resources/exampleSchema.avsc diff --git a/hudi-client/hudi-client-common/src/test/resources/exampleSchemaWithMetaFields.avsc b/hudi-common/src/test/resources/exampleSchemaWithMetaFields.avsc similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/exampleSchemaWithMetaFields.avsc rename to hudi-common/src/test/resources/exampleSchemaWithMetaFields.avsc diff --git a/hudi-client/hudi-client-common/src/test/resources/exampleSchemaWithUDT.avsc b/hudi-common/src/test/resources/exampleSchemaWithUDT.avsc similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/exampleSchemaWithUDT.avsc rename to hudi-common/src/test/resources/exampleSchemaWithUDT.avsc diff --git a/hudi-client/hudi-client-common/src/test/resources/hudi_0_10_hbase_1_2_3_bootstrap_index_partitions.hfile b/hudi-common/src/test/resources/hudi_0_10_hbase_1_2_3_bootstrap_index_partitions.hfile similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/hudi_0_10_hbase_1_2_3_bootstrap_index_partitions.hfile rename to hudi-common/src/test/resources/hudi_0_10_hbase_1_2_3_bootstrap_index_partitions.hfile diff --git a/hudi-client/hudi-client-common/src/test/resources/hudi_0_10_hbase_1_2_3_complex.hfile b/hudi-common/src/test/resources/hudi_0_10_hbase_1_2_3_complex.hfile similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/hudi_0_10_hbase_1_2_3_complex.hfile rename to hudi-common/src/test/resources/hudi_0_10_hbase_1_2_3_complex.hfile diff --git a/hudi-client/hudi-client-common/src/test/resources/hudi_0_10_hbase_1_2_3_simple.hfile b/hudi-common/src/test/resources/hudi_0_10_hbase_1_2_3_simple.hfile similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/hudi_0_10_hbase_1_2_3_simple.hfile rename to hudi-common/src/test/resources/hudi_0_10_hbase_1_2_3_simple.hfile diff --git a/hudi-client/hudi-client-common/src/test/resources/hudi_0_11_hbase_2_4_9_bootstrap_index_partitions.hfile b/hudi-common/src/test/resources/hudi_0_11_hbase_2_4_9_bootstrap_index_partitions.hfile similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/hudi_0_11_hbase_2_4_9_bootstrap_index_partitions.hfile rename to hudi-common/src/test/resources/hudi_0_11_hbase_2_4_9_bootstrap_index_partitions.hfile diff --git a/hudi-client/hudi-client-common/src/test/resources/hudi_0_11_hbase_2_4_9_complex.hfile b/hudi-common/src/test/resources/hudi_0_11_hbase_2_4_9_complex.hfile similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/hudi_0_11_hbase_2_4_9_complex.hfile rename to hudi-common/src/test/resources/hudi_0_11_hbase_2_4_9_complex.hfile diff --git a/hudi-client/hudi-client-common/src/test/resources/hudi_0_11_hbase_2_4_9_simple.hfile b/hudi-common/src/test/resources/hudi_0_11_hbase_2_4_9_simple.hfile similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/hudi_0_11_hbase_2_4_9_simple.hfile rename to hudi-common/src/test/resources/hudi_0_11_hbase_2_4_9_simple.hfile diff --git a/hudi-client/hudi-client-common/src/test/resources/hudi_0_9_hbase_1_2_3_bootstrap_index_partitions.hfile b/hudi-common/src/test/resources/hudi_0_9_hbase_1_2_3_bootstrap_index_partitions.hfile similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/hudi_0_9_hbase_1_2_3_bootstrap_index_partitions.hfile rename to hudi-common/src/test/resources/hudi_0_9_hbase_1_2_3_bootstrap_index_partitions.hfile diff --git a/hudi-client/hudi-client-common/src/test/resources/hudi_0_9_hbase_1_2_3_complex.hfile b/hudi-common/src/test/resources/hudi_0_9_hbase_1_2_3_complex.hfile similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/hudi_0_9_hbase_1_2_3_complex.hfile rename to hudi-common/src/test/resources/hudi_0_9_hbase_1_2_3_complex.hfile diff --git a/hudi-client/hudi-client-common/src/test/resources/hudi_0_9_hbase_1_2_3_simple.hfile b/hudi-common/src/test/resources/hudi_0_9_hbase_1_2_3_simple.hfile similarity index 100% rename from hudi-client/hudi-client-common/src/test/resources/hudi_0_9_hbase_1_2_3_simple.hfile rename to hudi-common/src/test/resources/hudi_0_9_hbase_1_2_3_simple.hfile