Migrated examples to OpenVINO 2023.0 and added tests for them (openvi…

…notoolkit#1908)

### Changes
- Updated .gitignore to exude example's artifacts
- Migrated examples to OpenVINO 2023.0.
- Bump up ultralytics to 8.0.43 for yolo8 examples
- Fixed ssd300_vgg16 example
- Fixed yolo8 examples
- Added tests/cross_fw/examples/test_examples.py
- Updated Makefile

### Reason for changes
- Migrated examples to the latest OpenVINO.
- Regular example validation.

### Related tickets
N/A

### Tests
tests/cross_fw/examples/test_examples.py

Run tests via make: `make test-examples`

---------

Co-authored-by: Nikita Savelyev <nikita.savelyev@intel.com>

.gitignore

@@ -120,10 +120,18 @@ examples/post_training_quantization/onnx/mobilenet_v2/mobilenet_v2_*
 examples/post_training_quantization/openvino/mobilenet_v2/mobilenet_v2_*
 examples/post_training_quantization/tensorflow/mobilenet_v2/mobilenet_v2_*
 examples/post_training_quantization/torch/mobilenet_v2/mobilenet_v2_*
+examples/post_training_quantization/torch/ssd300_vgg16/ssd300_vgg16_*
+examples/post_training_quantization/openvino/anomaly_stfpm_quantize_with_accuracy_control/stfpm_*
+examples/post_training_quantization/openvino/yolov8/yolov8n*
+examples/post_training_quantization/openvino/yolov8_quantize_with_accuracy_control/yolov8n*
 examples/**/runs/**
 examples/**/results/**
+compressed_graph.dot
+original_graph.dot
+datasets/**
 
 # Tests
 tests/**/runs/**
 tests/**/tmp*/**
 open_model_zoo/
+nncf-tests.xml

.isort.cfg

@@ -3,3 +3,4 @@ line_length = 120
 force_single_line = true
 profile = black
 single_line_exclusions = typing
+skip_glob=examples/post_training_quantization/torch/ssd300_vgg16/main.py

Makefile

@@ -18,10 +18,11 @@ install-onnx-test:
 	pip install -e .[onnx]
 	pip install -r tests/onnx/requirements.txt
 	pip install -r tests/cross_fw/install/requirements.txt
+	pip install -r tests/cross_fw/examples/requirements.txt
 	pip install -r tests/onnx/benchmarking/requirements.txt
-	pip install -r examples/post_training_quantization/onnx/mobilenet_v2/requirements.txt
 
 install-onnx-dev: install-onnx-test install-pre-commit install-pylint
+	pip install -r examples/post_training_quantization/onnx/mobilenet_v2/requirements.txt
 
 test-onnx:
 	pytest tests/onnx $(DATA_ARG) --junitxml ${JUNITXML_PATH}
@@ -31,7 +32,12 @@ pylint-onnx:
 		$(shell python3 tools/collect_pylint_input_files_for_backend.py onnx)
 
 test-install-onnx:
-	pytest tests/cross_fw/install/ -s       \
+	pytest tests/cross_fw/install -s       \
+		--backend onnx                      \
+		--junitxml ${JUNITXML_PATH}
+
+test-examples-onnx:
+	pytest tests/cross_fw/examples -s       \
 		--backend onnx                      \
 		--junitxml ${JUNITXML_PATH}
 
@@ -42,13 +48,14 @@ install-openvino-test:
 	pip install -e .[openvino]
 	pip install -r tests/openvino/requirements.txt
 	pip install -r tests/cross_fw/install/requirements.txt
-	pip install -r examples/experimental/openvino/bert/requirements.txt
-	pip install -r examples/experimental/openvino/yolo_v5/requirements.txt
+	pip install -r tests/cross_fw/examples/requirements.txt
 	pip install git+https://github.com/openvinotoolkit/open_model_zoo.git@dcbf53280a95dae3c6538689bafe760470f08ec2#subdirectory=tools/model_tools
 
 install-openvino-dev: install-openvino-test install-pre-commit install-pylint
+	pip install -r examples/experimental/openvino/bert/requirements.txt
+	pip install -r examples/experimental/openvino/yolo_v5/requirements.txt
 	pip install -r examples/post_training_quantization/openvino/mobilenet_v2/requirements.txt
-	pip install -r examples/post_training_quantization/openvino/quantize_with_accuracy_control/requirements.txt
+	pip install -r examples/post_training_quantization/openvino/anomaly_stfpm_quantize_with_accuracy_control/requirements.txt
 	pip install -r examples/post_training_quantization/openvino/yolov8/requirements.txt
 	pip install -r examples/post_training_quantization/openvino/yolov8_quantize_with_accuracy_control/requirements.txt
 
@@ -64,16 +71,23 @@ test-install-openvino:
 		--backend openvino                  \
 		--junitxml ${JUNITXML_PATH}
 
+test-examples-openvino:
+	pytest tests/cross_fw/examples -s        \
+		--backend openvino                  \
+		--junitxml ${JUNITXML_PATH}
+
 ###############################################################################
 # TensorFlow backend
 install-tensorflow-test:
 	pip install -U pip
 	pip install -e .[tf]
 	pip install -r tests/tensorflow/requirements.txt
 	pip install -r tests/cross_fw/install/requirements.txt
+	pip install -r tests/cross_fw/examples/requirements.txt
 	pip install -r examples/tensorflow/requirements.txt
 
 install-tensorflow-dev: install-tensorflow-test install-pre-commit install-pylint
+	pip install -r examples/post_training_quantization/tensorflow/mobilenet_v2/requirements.txt
 
 test-tensorflow:
 	pytest tests/common tests/tensorflow    \
@@ -85,7 +99,10 @@ pylint-tensorflow:
 		$(shell python3 tools/collect_pylint_input_files_for_backend.py tensorflow)
 
 test-install-tensorflow:
-	pytest tests/cross_fw/install/ -s --backend tf --junitxml ${JUNITXML_PATH}
+	pytest tests/cross_fw/install -s --backend tf --junitxml ${JUNITXML_PATH}
+
+test-examples-tensorflow:
+	pytest tests/cross_fw/examples -s --backend tf --junitxml ${JUNITXML_PATH}
 
 ###############################################################################
 # PyTorch backend
@@ -94,9 +111,11 @@ install-torch-test:
 	pip install -e .[torch]
 	pip install -r tests/torch/requirements.txt
 	pip install -r tests/cross_fw/install/requirements.txt
+	pip install -r tests/cross_fw/examples/requirements.txt
 	pip install -r examples/torch/requirements.txt
 
 install-torch-dev: install-torch-test install-pre-commit install-pylint
+	pip install -r examples/post_training_quantization/torch/mobilenet_v2/requirements.txt
 	pip install -r examples/post_training_quantization/torch/ssd300_vgg16/requirements.txt
 
 test-torch:
@@ -109,7 +128,7 @@ pylint-torch:
 		$(shell python3 tools/collect_pylint_input_files_for_backend.py torch)
 
 test-install-torch-cpu:
-	pytest tests/cross_fw/install/ -s       \
+	pytest tests/cross_fw/install -s       \
 		--backend torch                     \
 		--host-configuration cpu            \
 		--junitxml ${JUNITXML_PATH}
@@ -119,15 +138,30 @@ test-install-torch-gpu:
 		--backend torch                     \
 		--junitxml ${JUNITXML_PATH}
 
+test-examples-torch:
+	pytest tests/cross_fw/examples -s        \
+		--backend torch                     \
+		--junitxml ${JUNITXML_PATH}
+
 ###############################################################################
 # Common part
+install-common-test:
+	pip install -U pip
+	pip install -e .
+	pip install -r tests/common/requirements.txt
+	pip install -r tests/cross_fw/install/requirements.txt
+	pip install -r tests/cross_fw/examples/requirements.txt
+
 pylint-common:
 	pylint --rcfile .pylintrc   \
 		$(COMMON_PYFILES)
 
 test-common:
 	pytest tests/common $(DATA_ARG) --junitxml ${JUNITXML_PATH}
 
+test-examples:
+	pytest tests/cross_fw/examples -s --junitxml ${JUNITXML_PATH}
+
 ###############################################################################
 # Pre commit check
 pre-commit:

examples/post_training_quantization/onnx/mobilenet_v2/requirements.txt

@@ -4,4 +4,4 @@ scikit-learn
 fastdownload
 onnx~=1.13.1
 onnxruntime~=1.14.1
-openvino-dev
+openvino-dev==2023.0.0

examples/post_training_quantization/openvino/quantize_with_accuracy_control/requirements.txt → examples/post_training_quantization/openvino/anomaly_stfpm_quantize_with_accuracy_control/requirements.txt

@@ -1,2 +1,2 @@
 anomalib@ git+https://github.com/openvinotoolkit/anomalib@d0b8ce8ea100bbdfb058f00a3b246f257a08da93#egg=anomalib
-openvino-dev
+openvino-dev==2023.0.0

examples/post_training_quantization/openvino/mobilenet_v2/requirements.txt

@@ -2,4 +2,4 @@ torchvision
 tqdm
 scikit-learn
 fastdownload
-openvino-dev~=2022.3.0.dev
+openvino-dev==2023.0.0

examples/post_training_quantization/openvino/yolov8/main.py

@@ -18,15 +18,18 @@
 import torch
 from tqdm import tqdm
 from ultralytics import YOLO
-from ultralytics.yolo.configs import get_config
-from ultralytics.yolo.data.utils import check_dataset_yaml
+from ultralytics.yolo.cfg import get_cfg
+from ultralytics.yolo.data.utils import check_det_dataset
 from ultralytics.yolo.engine.validator import BaseValidator as Validator
-from ultralytics.yolo.utils import DEFAULT_CONFIG
+from ultralytics.yolo.utils import DATASETS_DIR
+from ultralytics.yolo.utils import DEFAULT_CFG
 from ultralytics.yolo.utils import ops
 from ultralytics.yolo.utils.metrics import ConfusionMatrix
 
 import nncf
 
+ROOT = Path(__file__).parent.resolve()
+
 
 def validate(
     model: ov.Model, data_loader: torch.utils.data.DataLoader, validator: Validator, num_samples: int = None
@@ -63,12 +66,12 @@ def print_statistics(stats: np.ndarray, total_images: int, total_objects: int) -
 
 
 def prepare_validation(model: YOLO, args: Any) -> Tuple[Validator, torch.utils.data.DataLoader]:
-    data = check_dataset_yaml(args.data)
-    dataset = data["val"]
+    validator = model.ValidatorClass(args)
+    validator.data = check_det_dataset(args.data)
+    dataset = validator.data["val"]
     print(f"{dataset}")
 
-    validator = model.ValidatorClass(args)
-    data_loader = validator.get_dataloader("../datasets/coco128", 1)
+    data_loader = validator.get_dataloader(f"{DATASETS_DIR}/coco128", 1)
 
     validator = model.ValidatorClass(args)
 
@@ -91,7 +94,7 @@ def benchmark_performance(model_path, config) -> float:
 
 
 def prepare_openvino_model(model: YOLO, model_name: str) -> Tuple[ov.Model, Path]:
-    model_path = Path(f"{model_name}_openvino_model/{model_name}.xml")
+    model_path = Path(f"{ROOT}/{model_name}_openvino_model/{model_name}.xml")
     if not model_path.exists():
         model.export(format="openvino", dynamic=True, half=False)
 
@@ -142,8 +145,8 @@ def transform_fn(data_item: Dict):
 def main():
     MODEL_NAME = "yolov8n"
 
-    model = YOLO(f"{MODEL_NAME}.pt")
-    args = get_config(config=DEFAULT_CONFIG)
+    model = YOLO(f"{ROOT}/{MODEL_NAME}.pt")
+    args = get_cfg(cfg=DEFAULT_CFG)
     args.data = "coco128.yaml"
 
     # Prepare validation dataset and helper
@@ -154,7 +157,7 @@ def main():
 
     # Quantize mode in OpenVINO representation
     quantized_model = quantize(ov_model, data_loader, validator)
-    quantized_model_path = Path(f"{MODEL_NAME}_openvino_model/{MODEL_NAME}_quantized.xml")
+    quantized_model_path = Path(f"{ROOT}/{MODEL_NAME}_openvino_model/{MODEL_NAME}_quantized.xml")
     ov.serialize(quantized_model, str(quantized_model_path))
 
     # Validate FP32 model

examples/post_training_quantization/openvino/yolov8/requirements.txt

@@ -1,3 +1,3 @@
-ultralytics==8.0.5
-onnx
-openvino-dev
+ultralytics==8.0.43
+onnx>=1.12.0
+openvino-dev==2023.0.0

examples/post_training_quantization/openvino/yolov8_quantize_with_accuracy_control/main.py

@@ -19,15 +19,18 @@
 import torch
 from tqdm import tqdm
 from ultralytics import YOLO
-from ultralytics.yolo.configs import get_config
-from ultralytics.yolo.data.utils import check_dataset_yaml
+from ultralytics.yolo.cfg import get_cfg
+from ultralytics.yolo.data.utils import check_det_dataset
 from ultralytics.yolo.engine.validator import BaseValidator as Validator
-from ultralytics.yolo.utils import DEFAULT_CONFIG
+from ultralytics.yolo.utils import DATASETS_DIR
+from ultralytics.yolo.utils import DEFAULT_CFG
 from ultralytics.yolo.utils import ops
 from ultralytics.yolo.utils.metrics import ConfusionMatrix
 
 import nncf
 
+ROOT = Path(__file__).parent.resolve()
+
 
 def validate(
     model: ov.Model, data_loader: torch.utils.data.DataLoader, validator: Validator, num_samples: int = None
@@ -50,7 +53,7 @@ def validate(
         else:
             preds = [
                 torch.from_numpy(results[compiled_model.output(0)]),
-                [torch.from_numpy(results[compiled_model.output(1)])],
+                torch.from_numpy(results[compiled_model.output(1)]),
             ]
         preds = validator.postprocess(preds)
         validator.update_metrics(preds, batch)
@@ -88,12 +91,12 @@ def print_statistics(stats: np.ndarray, total_images: int, total_objects: int) -
 
 
 def prepare_validation(model: YOLO, args: Any) -> Tuple[Validator, torch.utils.data.DataLoader]:
-    data = check_dataset_yaml(args.data)
-    dataset = data["val"]
+    validator = model.ValidatorClass(args)
+    validator.data = check_det_dataset(args.data)
+    dataset = validator.data["val"]
     print(f"{dataset}")
 
-    validator = model.ValidatorClass(args)
-    data_loader = validator.get_dataloader("../datasets/coco128-seg", 1)
+    data_loader = validator.get_dataloader(f"{DATASETS_DIR}/coco128-seg", 1)
 
     validator = model.ValidatorClass(args)
 
@@ -119,7 +122,7 @@ def benchmark_performance(model_path, config) -> float:
 
 
 def prepare_openvino_model(model: YOLO, model_name: str) -> Tuple[ov.Model, Path]:
-    model_path = Path(f"{model_name}_openvino_model/{model_name}.xml")
+    model_path = Path(f"{ROOT}/{model_name}_openvino_model/{model_name}.xml")
     if not model_path.exists():
         model.export(format="openvino", dynamic=True, half=False)
 
@@ -156,7 +159,7 @@ def validation_ac(
             else:
                 preds = [
                     torch.from_numpy(results[compiled_model.output(0)]),
-                    [torch.from_numpy(results[compiled_model.output(1)])],
+                    torch.from_numpy(results[compiled_model.output(1)]),
                 ]
             preds = validator.postprocess(preds)
             validator.update_metrics(preds, batch)
@@ -204,8 +207,8 @@ def validation_ac(
 def main():
     MODEL_NAME = "yolov8n-seg"
 
-    model = YOLO(f"{MODEL_NAME}.pt")
-    args = get_config(config=DEFAULT_CONFIG)
+    model = YOLO(f"{ROOT}/{MODEL_NAME}.pt")
+    args = get_cfg(cfg=DEFAULT_CFG)
     args.data = "coco128-seg.yaml"
 
     # Prepare validation dataset and helper
@@ -217,7 +220,7 @@ def main():
     # Quantize mode in OpenVINO representation
     quantized_model = quantize_ac(ov_model, data_loader, validator)
 
-    quantized_model_path = Path(f"{MODEL_NAME}_openvino_model/{MODEL_NAME}_quantized.xml")
+    quantized_model_path = Path(f"{ROOT}/{MODEL_NAME}_openvino_model/{MODEL_NAME}_quantized.xml")
     ov.serialize(quantized_model, str(quantized_model_path))
 
     # Validate FP32 model

examples/post_training_quantization/openvino/yolov8_quantize_with_accuracy_control/requirements.txt

@@ -1,3 +1,3 @@
-ultralytics==8.0.5
-onnx
-openvino-dev
+ultralytics==8.0.43
+onnx>=1.12.0
+openvino-dev==2023.0.0

examples/post_training_quantization/tensorflow/mobilenet_v2/main.py

@@ -35,7 +35,7 @@ def validate(model: ov.Model, val_loader: tf.data.Dataset) -> tf.Tensor:
 
     metric = tf.keras.metrics.CategoricalAccuracy(name="acc@1")
     for images, labels in tqdm(val_loader):
-        pred = compiled_model(images)[output]
+        pred = compiled_model(images.numpy())[output]
         metric.update_state(labels, pred)
 
     return metric.result()

examples/post_training_quantization/tensorflow/mobilenet_v2/requirements.txt

@@ -1,4 +1,4 @@
-tensorflow~=2.11.1
+tensorflow>=2.7.0,<2.12
 tensorflow-datasets
 tqdm
-openvino-dev~=2022.3.0.dev
+openvino-dev==2023.0.0