From da726010d47bf297e123ffbdb41cd2f6bcbf2423 Mon Sep 17 00:00:00 2001
From: ouyanglinke <ouyanglinke@pjlab.org.cn>
Date: Mon, 22 Jul 2024 10:45:52 +0800
Subject: [PATCH] adding category mapping intro into validation readme

---
 assets/validation/README-zh_CN.md | 95 +++++++++++++++++++++++++++++++
 assets/validation/README.md       | 95 +++++++++++++++++++++++++++++++
 2 files changed, 190 insertions(+)

diff --git a/assets/validation/README-zh_CN.md b/assets/validation/README-zh_CN.md
index 39b0661..65a71cf 100644
--- a/assets/validation/README-zh_CN.md
+++ b/assets/validation/README-zh_CN.md
@@ -21,6 +21,101 @@ python train_net.py --config-file config.yaml --eval-only --num-gpus 8 \
         OUTPUT_DIR /path/to/save/dir
 ```
 
+与其他开源模型的对比部分，我们没有对其模型做特殊参数设置，评测时候用每个模型repo里提供的代码直接进行推理，详情请参考：
+    - [Surya](https://github.com/VikParuchuri/surya?tab=readme-ov-file#layout-analysis)
+    - [360LayoutAnalysis](https://github.com/360AILAB-NLP/360LayoutAnalysis/blob/main/README_EN.md)
+
+评测结果输出依赖于mmeval的[COCODetection](https://mmeval.readthedocs.io/zh-cn/latest/api/generated/mmeval.metrics.COCODetection.html)。
+
+由于每个模型的类别标签不完全一致，我们在验证的时候对标签进行了映射对齐，具体如下：
+
+### Surya
+
+```python
+# 参与验证的类别
+label_classes = ["title", "plain text", "abandon", "figure", "caption", "table", "isolate_formula"] 
+
+# GT的类别映射 (原本的类别与本repo微调的LayoutLmv3-SFT对齐)
+anno_class_change_dict = {
+    'formula_caption': 'caption',
+    'table_caption': 'caption',
+    'table_footnote': 'plain text'
+}
+
+# Surya的类别映射
+class_dict = {
+    'Caption': 'caption',
+    'Section-header' : 'title',
+    'Title': 'title',
+    'Figure': 'figure',
+    'Picture': 'figure',
+    'Footnote': 'abandon',
+    'Page-footer': 'abandon',
+    'Page-header': 'abandon',
+    'Table': 'table',
+    'Text': 'plain text',
+    'List-item': 'plain text',
+    'Formula': 'isolate_formula',
+}
+```
+
+### 360LayoutAnalysis-Paper
+
+```python
+# 参与验证的类别
+label_classes = ["title", "plain text", "abandon", "figure", "figure_caption", "table", "table_caption", "isolate_formula"]
+
+# GT的类别映射表
+anno_class_change_dict = {
+    'formula_caption': 'plain text',
+    'table_footnote': 'plain text'
+}
+
+# 360LayoutAnalysis的类别映射
+class_change_dict = {
+    'Text': 'plain text',  
+    'Title': 'title', 
+    'Figure': 'figure', 
+    'Figure caption': 'figure_caption',    
+    'Table': 'table',      
+    'Table caption': 'table_caption',  
+    'Header': 'abandon', 
+    'Footer': 'abandon',     
+    'Reference': 'plain text',   
+    'Equation': 'isolate_formula',
+    'Toc': 'plain text'   
+}
+```
+
+### 360LayoutAnalysis-Report
+
+```python
+# 参与验证的类别
+label_classes = ["title", "plain text", "abandon", "figure", "figure_caption", "table", "table_caption"]
+
+# GT的类别映射表
+anno_class_change_dict = {
+    'formula_caption': 'plain text',
+    'table_footnote': 'plain text',
+    'isolate_formula': 'plain text',
+}
+
+# 360LayoutAnalysis的类别映射
+class_change_dict = {
+    'Text': 'plain text',  
+    'Title': 'title', 
+    'Figure': 'figure', 
+    'Figure caption': 'figure_caption',    
+    'Table': 'table',      
+    'Table caption': 'table_caption',  
+    'Header': 'abandon', 
+    'Footer': 'abandon',     
+    'Reference': 'plain text',   
+    'Equation': 'isolate_formula',
+    'Toc': 'plain text'   
+}
+```
+
 ## 公式检测
 
 公式检测的部分，我们在[YOLOv8](https://github.com/ultralytics/ultralytics)的基础上新增了验证代码。
diff --git a/assets/validation/README.md b/assets/validation/README.md
index 2fdd3ba..14107c3 100644
--- a/assets/validation/README.md
+++ b/assets/validation/README.md
@@ -21,6 +21,101 @@ python train_net.py --config-file config.yaml --eval-only --num-gpus 8 \
         OUTPUT_DIR /path/to/save/dir
 ```
 
+For validation of other open-source models, we directly used the inference code provided in each model's GitHub repository without any special settings. For details, please refer to:
+    - [Surya](https://github.com/VikParuchuri/surya?tab=readme-ov-file#layout-analysis)
+    - [360LayoutAnalysis](https://github.com/360AILAB-NLP/360LayoutAnalysis/blob/main/README_EN.md)
+
+The evaluation matrix is provided by mmeval using [COCODetection](https://mmeval.readthedocs.io/zh-cn/latest/api/generated/mmeval.metrics.COCODetection.html).
+
+Due to the fact that the category labels of each model are not completely consistent, we implement the category mapping for alignment before the validation process.
+
+### Surya
+
+```python
+# Participated categories in the validation
+label_classes = ["title", "plain text", "abandon", "figure", "caption", "table", "isolate_formula"] 
+
+# Ground Truth Category Mapping (original categories aligned with the LayoutLmv3-SFT fine-tuned in this repository)
+anno_class_change_dict = {
+    'formula_caption': 'caption',
+    'table_caption': 'caption',
+    'table_footnote': 'plain text'
+}
+
+# Surya Category Mapping
+class_dict = {
+    'Caption': 'caption',
+    'Section-header' : 'title',
+    'Title': 'title',
+    'Figure': 'figure',
+    'Picture': 'figure',
+    'Footnote': 'abandon',
+    'Page-footer': 'abandon',
+    'Page-header': 'abandon',
+    'Table': 'table',
+    'Text': 'plain text',
+    'List-item': 'plain text',
+    'Formula': 'isolate_formula',
+}
+```
+
+### 360LayoutAnalysis-Paper
+
+```python
+# Participated categories in the validation
+label_classes = ["title", "plain text", "abandon", "figure", "figure_caption", "table", "table_caption", "isolate_formula"]
+
+# Ground Truth Category Mapping
+anno_class_change_dict = {
+    'formula_caption': 'plain text',
+    'table_footnote': 'plain text'
+}
+
+# 360LayoutAnalysis Category Mapping
+class_change_dict = {
+    'Text': 'plain text',  
+    'Title': 'title', 
+    'Figure': 'figure', 
+    'Figure caption': 'figure_caption',    
+    'Table': 'table',      
+    'Table caption': 'table_caption',  
+    'Header': 'abandon', 
+    'Footer': 'abandon',     
+    'Reference': 'plain text',   
+    'Equation': 'isolate_formula',
+    'Toc': 'plain text'   
+}
+```
+
+### 360LayoutAnalysis-Report
+
+```python
+# Participated categories in the validation
+label_classes = ["title", "plain text", "abandon", "figure", "figure_caption", "table", "table_caption"]
+
+# Ground Truth Category Mapping
+anno_class_change_dict = {
+    'formula_caption': 'plain text',
+    'table_footnote': 'plain text',
+    'isolate_formula': 'plain text',
+}
+
+# 360LayoutAnalysis Category Mapping
+class_change_dict = {
+    'Text': 'plain text',  
+    'Title': 'title', 
+    'Figure': 'figure', 
+    'Figure caption': 'figure_caption',    
+    'Table': 'table',      
+    'Table caption': 'table_caption',  
+    'Header': 'abandon', 
+    'Footer': 'abandon',     
+    'Reference': 'plain text',   
+    'Equation': 'isolate_formula',
+    'Toc': 'plain text'   
+}
+```
+
 ## Formula Detection
 
 For Formula Detection, we have developed validation process based on [YOLOv8](https://github.com/ultralytics/ultralytics).