[core] Extract file format of DataFileMeta through suffix of filename (…

…apache#2389)
zhu3pang · Nov 27, 2023 · d23607d · d23607d
1 parent cc6aa1d
commit d23607d
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 2 deletions.
diff --git a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java
@@ -37,6 +37,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 
 import static org.apache.paimon.data.BinaryRow.EMPTY_ROW;
 import static org.apache.paimon.utils.Preconditions.checkArgument;
@@ -223,6 +224,16 @@ public long creationTimeEpochMillis() {
                 .toEpochMilli();
     }
 
+    public Optional<CoreOptions.FileFormatType> fileFormat() {
+        String[] split = fileName.split("\\.");
+        try {
+            return Optional.of(
+                    CoreOptions.FileFormatType.valueOf(split[split.length - 1].toUpperCase()));
+        } catch (IllegalArgumentException e) {
+            return Optional.empty();
+        }
+    }
+
     public DataFileMeta upgrade(int newLevel) {
         checkArgument(newLevel > this.level);
         return new DataFileMeta(

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java
@@ -442,7 +442,13 @@ private RawFile makeRawTableFile(String bucketPath, DataFileMeta meta) {
                 bucketPath + "/" + meta.fileName(),
                 0,
                 meta.fileSize(),
-                new CoreOptions(tableSchema.options()).formatType().toString().toLowerCase(),
+                meta.fileFormat()
+                        .map(t -> t.toString().toLowerCase())
+                        .orElse(
+                                new CoreOptions(tableSchema.options())
+                                        .formatType()
+                                        .toString()
+                                        .toLowerCase()),
                 meta.schemaId());
     }
 }
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
@@ -50,7 +50,9 @@
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -155,7 +157,7 @@ public void testGetPrimaryKeyRawFiles() throws Exception {
                                                     tablePath, partition, meta.fileName()),
                                             0,
                                             meta.fileSize(),
-                                            "avro",
+                                            meta.level() == 5 ? "orc" : "avro",
                                             meta.schemaId())));
         }
 
@@ -266,6 +268,9 @@ private FileStoreTable createFileStoreTable(
         options.set(CoreOptions.BUCKET, 1);
         options.set(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER, 5);
         options.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+        Map<String, String> formatPerLevel = new HashMap<>();
+        formatPerLevel.put("5", "orc");
+        options.set(CoreOptions.FILE_FORMAT_PER_LEVEL, formatPerLevel);
 
         SchemaManager schemaManager = new SchemaManager(fileIO, tablePath);
         TableSchema tableSchema =