Merge pull request InnopolisUni#90 from QazyBi/feature/add-sibur-data

Feature/add sibur data

config/datasets/sibur_people_n_head_detection.yaml

@@ -0,0 +1,34 @@
+task:
+  - image-detection
+
+name: detection_people_n_head
+description: Набор данных содержит изображения людей на производстве.
+
+markup_info: Набор данных содержит разметку bounding box, под формат детекции.
+date_time: 17.01.2022
+
+_target_: innofw.core.integrations.ultralytics.datamodule.YOLOV5DataModuleAdapter
+
+
+train:
+  # source: data/sibur/processed8020/train
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/testing/sibur_peoples_n_heads_8020/train.zip
+  target: ./data/sibur_peoples_n_heads_8020/train
+test:
+  # source: data/sibur/processed8020/test
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/testing/sibur_peoples_n_heads_8020/test.zip
+  target: ./data/sibur_peoples_n_heads_8020/test
+infer:
+  # source: data/sibur/processed8020/infer
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/testing/sibur_peoples_n_heads_8020/test.zip
+  target: ./data/sibur_peoples_n_heads_8020/infer
+
+num_workers: 8
+
+val_size: 0.2
+channels_num: 3
+image_size: 600
+num_classes: 2
+names:
+  - people
+  - head

config/experiments/KA_170122_43fdja_yolov5_sibur.yaml

@@ -0,0 +1,12 @@
+# @package _global_
+defaults:
+  - override /models: yolov5
+  - override /datasets: sibur_people_n_head_detection
+
+
+project: "people_n_head_detection"
+task: "image-detection"
+random_seed: 43
+epochs: 300
+batch_size: 2
+weights_freq: 1

examples/infer_people_n_head_detection.sh

@@ -0,0 +1 @@
+python infer.py experiments=KA_170122_43fdja_yolov5_sibur +ckpt_path=logs/train/people_n_head_detection/KA_170122_43fdja_yolov5_sibur/20230117-065904/weights/best.pt

examples/train_people_n_head_detection.sh

@@ -0,0 +1 @@
+python train.py experiments=KA_170122_43fdja_yolov5_sibur

innofw/data_mart/uploader.py

@@ -24,19 +24,19 @@
 
 @validate_arguments
 def upload_dataset(
-        folder_path: DirectoryPath,
-        config_save_path: Path,
-        remote_save_path: AnyUrl,
-        task: Union[str, List[str]],
-        framework: Union[str, List[str]],
-        target: str,
-        name: str,
-        description: str,
-        markup_info: str,
-        date_time: str,
-        access_key: Optional[str] = None,
-        secret_key: Optional[str] = None,
-        **kwargs,
+    folder_path: DirectoryPath,
+    config_save_path: Path,
+    remote_save_path: AnyUrl,
+    task: Union[str, List[str]],
+    framework: Union[str, List[str]],
+    target: str,
+    name: str,
+    description: str,
+    markup_info: str,
+    date_time: str,
+    access_key: Optional[str] = None,
+    secret_key: Optional[str] = None,
+    **kwargs,
 ):
     """Function to upload a dataset into s3(remote storage) and generate config file for the dataset
 

innofw/utils/data_utils/dataset_prep/prep_yolov5.py

@@ -0,0 +1,54 @@
+# create train, test and infer folders
+# with images and labels folders inside
+import shutil
+from pathlib import Path
+import logging
+
+from fire import Fire
+from pydantic import validate_arguments, DirectoryPath, FilePath
+
+
+@validate_arguments
+def prepare_dataset(
+    src_img_path: DirectoryPath,
+    src_label_path: DirectoryPath,
+    train_names_file: FilePath,
+    test_names_file: FilePath,
+    dst_path: Path,
+):
+    with open(train_names_file, "r") as f:
+        train_names = [i for i in f.read().split("\n") if i != ""]
+
+    with open(test_names_file, "r") as f:
+        test_names = [i for i in f.read().split("\n") if i != ""]
+
+    sets = ["train", "test", "infer"]
+    names = [train_names, test_names, test_names]
+
+    for s, n in zip(sets, names):
+        set_dst_path = dst_path / s
+        set_dst_path.mkdir(exist_ok=True, parents=True)
+        set_img_dst_path = set_dst_path / "images"
+        set_label_dst_path = set_dst_path / "labels"
+
+        set_img_dst_path.mkdir(exist_ok=True, parents=True)
+        set_label_dst_path.mkdir(exist_ok=True, parents=True)
+
+        for filename in n:
+            try:
+                img_name = f"{Path(filename).name}.jpeg"
+                shutil.copy(
+                    src_img_path / img_name,
+                    set_img_dst_path / img_name,
+                )
+                label_name = f"{Path(filename).name}.txt"
+                shutil.copy(
+                    src_label_path / label_name,
+                    set_label_dst_path / label_name,
+                )
+            except:
+                logging.warning(f"Could not copy file: {filename}")
+
+
+if __name__ == "__main__":
+    Fire(prepare_dataset)