From 5ced61960d193a801a315af232ba51cf3f41e2e1 Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Wed, 31 Jan 2024 14:29:12 -0800 Subject: [PATCH] [HUDI-7364] Move InLineFs classes to hudi-hadoop-common module --- .../java/org/apache/hudi/common/fs/FSUtils.java | 4 ++-- .../table/log/block/HoodieHFileDataBlock.java | 2 +- .../table/log/block/HoodieParquetDataBlock.java | 2 +- .../org/apache/hudi/common/fs/TestFSUtils.java | 2 +- .../hudi/common/fs/inline/InLineFSUtilsTest.java | 2 ++ .../common/fs/inline/TestInLineFileSystem.java | 2 ++ .../TestInLineFileSystemHFileInLiningBase.java | 2 ++ .../TestInLineFileSystemWithHBaseHFileReader.java | 1 + .../TestInLineFileSystemWithHFileReader.java | 1 + .../common/fs/inline/TestInMemoryFileSystem.java | 2 ++ .../common/fs/inline/TestParquetInLining.java | 2 ++ .../common/testutils/FileSystemTestUtils.java | 6 +++--- .../hudi/hadoop}/fs/inline/InLineFSUtils.java | 15 ++++++++------- .../hudi/hadoop}/fs/inline/InLineFileSystem.java | 15 ++++++++------- .../fs/inline/InLineFsDataInputStream.java | 15 ++++++++------- .../hadoop}/fs/inline/InMemoryFileSystem.java | 15 ++++++++------- 16 files changed, 52 insertions(+), 36 deletions(-) rename {hudi-common/src/main/java/org/apache/hudi/common => hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/inline/InLineFSUtils.java (91%) rename {hudi-common/src/main/java/org/apache/hudi/common => hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/inline/InLineFileSystem.java (91%) rename {hudi-common/src/main/java/org/apache/hudi/common => hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/inline/InLineFsDataInputStream.java (90%) rename {hudi-common/src/main/java/org/apache/hudi/common => hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/inline/InMemoryFileSystem.java (90%) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java index 36b6e6c1b422..5880d7f90689 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java @@ -22,8 +22,6 @@ import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.config.SerializableConfiguration; import org.apache.hudi.common.engine.HoodieEngineContext; -import org.apache.hudi.common.fs.inline.InLineFSUtils; -import org.apache.hudi.common.fs.inline.InLineFileSystem; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.HoodieTableConfig; @@ -41,6 +39,8 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils; import org.apache.hudi.hadoop.fs.HoodieWrapperFileSystem; import org.apache.hudi.hadoop.fs.NoOpConsistencyGuard; +import org.apache.hudi.hadoop.fs.inline.InLineFSUtils; +import org.apache.hudi.hadoop.fs.inline.InLineFileSystem; import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.storage.HoodieLocation; import org.apache.hudi.storage.HoodieStorage; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java index ee8d9578c659..ad488a673cbe 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java @@ -23,7 +23,6 @@ import org.apache.hudi.common.config.HoodieReaderConfig; import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.fs.FSUtils; -import org.apache.hudi.common.fs.inline.InLineFSUtils; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType; @@ -32,6 +31,7 @@ import org.apache.hudi.common.util.collection.CloseableMappingIterator; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.hadoop.fs.HadoopFSUtils; +import org.apache.hudi.hadoop.fs.inline.InLineFSUtils; import org.apache.hudi.io.SeekableDataInputStream; import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase; import org.apache.hudi.io.storage.HoodieFileReader; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java index 485d70890b44..77f784dfc388 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java @@ -21,12 +21,12 @@ import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.fs.FSUtils; -import org.apache.hudi.common.fs.inline.InLineFSUtils; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ClosableIterator; +import org.apache.hudi.hadoop.fs.inline.InLineFSUtils; import org.apache.hudi.io.SeekableDataInputStream; import org.apache.hudi.io.storage.HoodieFileReaderFactory; import org.apache.hudi.io.storage.HoodieFileWriter; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java index 2fb91c91b18c..ab638bb37a95 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java @@ -20,7 +20,6 @@ import org.apache.hudi.common.config.SerializableConfiguration; import org.apache.hudi.common.engine.HoodieLocalEngineContext; -import org.apache.hudi.common.fs.inline.InLineFSUtils; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; @@ -34,6 +33,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils; import org.apache.hudi.hadoop.fs.HoodieWrapperFileSystem; import org.apache.hudi.hadoop.fs.NoOpConsistencyGuard; +import org.apache.hudi.hadoop.fs.inline.InLineFSUtils; import org.apache.hudi.storage.HoodieLocation; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.hadoop.HoodieHadoopStorage; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/InLineFSUtilsTest.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/InLineFSUtilsTest.java index 896ebe2f4497..1d4d02d30418 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/InLineFSUtilsTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/InLineFSUtilsTest.java @@ -19,6 +19,8 @@ package org.apache.hudi.common.fs.inline; import org.apache.hudi.common.testutils.FileSystemTestUtils; +import org.apache.hudi.hadoop.fs.inline.InLineFSUtils; +import org.apache.hudi.hadoop.fs.inline.InLineFileSystem; import org.apache.hadoop.fs.Path; import org.junit.jupiter.params.ParameterizedTest; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystem.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystem.java index 5e80b9ca9667..e143f653f51c 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystem.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystem.java @@ -20,6 +20,8 @@ import org.apache.hudi.common.testutils.FileSystemTestUtils; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.hadoop.fs.inline.InLineFSUtils; +import org.apache.hudi.hadoop.fs.inline.InLineFileSystem; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemHFileInLiningBase.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemHFileInLiningBase.java index 090d47aacc7c..011eb45eac54 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemHFileInLiningBase.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemHFileInLiningBase.java @@ -19,6 +19,8 @@ package org.apache.hudi.common.fs.inline; import org.apache.hudi.common.testutils.FileSystemTestUtils; +import org.apache.hudi.hadoop.fs.inline.InLineFileSystem; +import org.apache.hudi.hadoop.fs.inline.InMemoryFileSystem; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHBaseHFileReader.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHBaseHFileReader.java index 0f3617f27193..752c6b708b50 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHBaseHFileReader.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHBaseHFileReader.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.fs.inline; +import org.apache.hudi.hadoop.fs.inline.InLineFileSystem; import org.apache.hudi.io.storage.HoodieHFileUtils; import org.apache.hudi.io.util.IOUtils; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHFileReader.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHFileReader.java index 36240054037c..2ae8fd2f6516 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHFileReader.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHFileReader.java @@ -20,6 +20,7 @@ package org.apache.hudi.common.fs.inline; import org.apache.hudi.common.util.Option; +import org.apache.hudi.hadoop.fs.inline.InLineFileSystem; import org.apache.hudi.io.hfile.HFileReader; import org.apache.hudi.io.hfile.HFileReaderImpl; import org.apache.hudi.io.hfile.Key; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInMemoryFileSystem.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInMemoryFileSystem.java index 41722256231d..b499dab198e4 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInMemoryFileSystem.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInMemoryFileSystem.java @@ -18,6 +18,8 @@ package org.apache.hudi.common.fs.inline; +import org.apache.hudi.hadoop.fs.inline.InMemoryFileSystem; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestParquetInLining.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestParquetInLining.java index 9ed27c4b2d63..7094fac6da0a 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestParquetInLining.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestParquetInLining.java @@ -23,6 +23,8 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.testutils.FileSystemTestUtils; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import org.apache.hudi.hadoop.fs.inline.InLineFileSystem; +import org.apache.hudi.hadoop.fs.inline.InMemoryFileSystem; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileSystemTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileSystemTestUtils.java index a00e215f10fa..dd64f62a9db2 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileSystemTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileSystemTestUtils.java @@ -18,11 +18,11 @@ package org.apache.hudi.common.testutils; -import org.apache.hudi.common.fs.inline.InLineFSUtils; -import org.apache.hudi.common.fs.inline.InLineFileSystem; -import org.apache.hudi.common.fs.inline.InMemoryFileSystem; import org.apache.hudi.common.table.log.TestLogReaderUtils; import org.apache.hudi.common.util.FileIOUtils; +import org.apache.hudi.hadoop.fs.inline.InLineFSUtils; +import org.apache.hudi.hadoop.fs.inline.InLineFileSystem; +import org.apache.hudi.hadoop.fs.inline.InMemoryFileSystem; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InLineFSUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/InLineFSUtils.java similarity index 91% rename from hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InLineFSUtils.java rename to hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/InLineFSUtils.java index 06a96542585c..b7c043f39cfe 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InLineFSUtils.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/InLineFSUtils.java @@ -7,16 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package org.apache.hudi.common.fs.inline; +package org.apache.hudi.hadoop.fs.inline; import org.apache.hudi.storage.HoodieLocation; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InLineFileSystem.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/InLineFileSystem.java similarity index 91% rename from hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InLineFileSystem.java rename to hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/InLineFileSystem.java index 1b2ea3cbedcf..02c85e9c7805 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InLineFileSystem.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/InLineFileSystem.java @@ -7,16 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package org.apache.hudi.common.fs.inline; +package org.apache.hudi.hadoop.fs.inline; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InLineFsDataInputStream.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/InLineFsDataInputStream.java similarity index 90% rename from hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InLineFsDataInputStream.java rename to hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/InLineFsDataInputStream.java index fbd067c6c18c..2466654c7f49 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InLineFsDataInputStream.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/InLineFsDataInputStream.java @@ -7,16 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package org.apache.hudi.common.fs.inline; +package org.apache.hudi.hadoop.fs.inline; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.ReadOption; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InMemoryFileSystem.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/InMemoryFileSystem.java similarity index 90% rename from hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InMemoryFileSystem.java rename to hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/InMemoryFileSystem.java index e433340f6000..7831e76c88fc 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InMemoryFileSystem.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/InMemoryFileSystem.java @@ -7,16 +7,17 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package org.apache.hudi.common.fs.inline; +package org.apache.hudi.hadoop.fs.inline; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream;