Add CPUExecutionProvider to the ONNX E2E

Makefile

@@ -15,7 +15,7 @@ install-onnx-test:
 	pip install -r tests/onnx/benchmarking/requirements.txt
 	pip install -r examples/post_training_quantization/onnx/mobilenet_v2/requirements.txt
 
-install-onnx-dev: install-onnx-test 
+install-onnx-dev: install-onnx-test
 	pip install pylint==$(PYLINT_VERSION)
 
 test-onnx:

tests/onnx/README.md

@@ -4,17 +4,17 @@ We provide two types of tests.
 
 1. Pre-commit test (no pytest markers)
 
-    This is a test that the CI server runs for every PR. It includes unit testing of ONNX framework-specific features of NNCF. In order to merge PR into the develop branch, all tests in it must be green. If a new feature or a bug fix is implemented, an appropriate unit test to validate those implementation should also be added to the pre-commit test. To run the pre-commit test, please execute the following command.
+    This is a test that the CI server runs for every PR. It consists of unit tests of ONNX features of NNCF. To run the pre-commit test, please execute the following command.
 
     ```bash
     $ pytest tests/onnx --junitxml nncf-tests.xml
     # (alias)
     $ make test-onnx
     ```
 
-2. E2E test for ONNX Model ZOO (pytest markers: `e2e_ptq` and `e2e_eval_reference_model`)
+2. E2E test (pytest markers: `e2e_ptq` and `e2e_eval_reference_model`)
 
-    This is a test to validate ONNX PTQ API functionality for the models in ONNX Model ZOO. There is no obligation for developers to check this test to merge their PR. It checks the quantized model accuracy and performance and compare them to the references. Our CI server runs it periodically, and if defects are found, the merged changes during the test cycle should be investigated. To run the E2E test, please execute the following command.
+    This is a test to validate ONNX PTQ API functionality for the models in ONNX Model ZOO. It compares the quantized model accuracy with the references. To run the E2E test, please execute the following command.
 
     ```bash
     $ pytest tests/onnx -m e2e_ptq --model-dir (model_dir) --data-dir (data_dir) --output-dir (output_dir) --ckpt-dir (ckpt_dir) --anno-dir (anno_dir) --eval-size (eval_size) --ptq-size (ptq_size)
@@ -25,12 +25,17 @@ We provide two types of tests.
     1. `--model-dir`: The directory path which includes ONNX Model ZOO models (.onnx files). See [#prepare-models](benchmarking/README.md#prepare-models) for details.
     2. `--data-dir`: The directory path which includes datasets (ImageNet2012, COCO, Cityscapes, and VOC) [#prepare-models](benchmarking/README.md#prepare-models).
     3. `--output-dir`: The directory path where the test results will be saved.
-    4. (Optional) `--ckpt-dir`: Directory path to save quantized models.
-    5. (Optional) `--anno-dir`: Directory path for dataset annotations. Please refer to [OpenVINO accuracy checker](https://github.com/openvinotoolkit/open_model_zoo/tree/master/tools/accuracy_checker).
-    6. (Optional) `--eval-size` The number of samples for evaluation.
-    7. (Optional) `--ptq-size` The number of samples for calibrating quantization parameters.
-
-    (Optional) If you want to test the reference (not quantized) model accuracy and performance, try the following command.
+    4. (Optional) `--model-names`: String containing model names to test. Model name is the prefix of the name of AccuracyChecker config before the '.' symbol. Please, provide the model names using ' ' as a separator.
+    5. (Optional) `--ckpt-dir`: Directory path to save quantized models.
+    6. (Optional) `--anno-dir`: Directory path for dataset annotations. Please refer to [OpenVINO accuracy checker](https://github.com/openvinotoolkit/open_model_zoo/tree/master/tools/accuracy_checker).
+    7. (Optional) `--eval-size`: The number of samples for evaluation.
+    8. (Optional) `--ptq-size`: The number of samples for calibrating quantization parameters.
+    9. (Optional) `--enable-ov-ep`: If the parameter is set then the accuracy validation of the quantized models
+             will be enabled for OpenVINOExecutionProvider.
+    10. (Optional) `--disable-cpu-ep`: If the parameter is set then the accuracy validation of the quantized models
+              will be disabled for CPUExecutionProvider. 
+
+    If you want to test the reference (not quantized) model accuracy - try the following command.
 
     ```bash
     $ pytest tests/onnx -m e2e_eval_reference_model --model-dir (model_dir) --data-dir (data_dir) --output-dir (output_dir) --ckpt-dir (ckpt_dir) --anno-dir (anno_dir) --eval-size (eval_size) --ptq-size (ptq_size)

tests/onnx/benchmarking/classification/onnx_models_configs/bvlcalexnet-12.yml

@@ -4,6 +4,11 @@ models:
   - framework: onnx_runtime
     adapter: classification
     execution_providers: ['OpenVINOExecutionProvider']
+    tags: ['OpenVINOExecutionProvider']
+  - framework: onnx_runtime
+    adapter: classification
+    execution_providers: ['CPUExecutionProvider']
+    tags: ['CPUExecutionProvider']
   datasets:
   - name: imagenet_1000_classes
     # read in RGB format

tests/onnx/benchmarking/classification/onnx_models_configs/caffenet-12.yml

@@ -4,6 +4,11 @@ models:
   - framework: onnx_runtime
     adapter: classification
     execution_providers: ['OpenVINOExecutionProvider']
+    tags: ['OpenVINOExecutionProvider']
+  - framework: onnx_runtime
+    adapter: classification
+    execution_providers: ['CPUExecutionProvider']
+    tags: ['CPUExecutionProvider']
   datasets:
   - name: imagenet_1000_classes
     # read in RGB format

tests/onnx/benchmarking/classification/onnx_models_configs/densenet-12.yml

@@ -3,7 +3,12 @@ models:
     launchers:
       - framework: onnx_runtime
         adapter: classification
-        execution_providers: ["OpenVINOExecutionProvider"]
+        execution_providers: ['OpenVINOExecutionProvider']
+        tags: ['OpenVINOExecutionProvider']
+      - framework: onnx_runtime
+        adapter: classification
+        execution_providers: ['CPUExecutionProvider']
+        tags: ['CPUExecutionProvider']
     datasets:
       - name: imagenet_1000_classes
         # read in RGB format

tests/onnx/benchmarking/classification/onnx_models_configs/efficientnet-lite4-11.yml

@@ -4,6 +4,16 @@ models:
   - framework: onnx_runtime
     adapter: classification
     execution_providers: ['OpenVINOExecutionProvider']
+    tags: ['OpenVINOExecutionProvider']
+    inputs:
+      - name: images:0
+        type: INPUT
+        shape: [ 1, 224, 224, 3 ]
+        layout: "NHWC"
+  - framework: onnx_runtime
+    adapter: classification
+    execution_providers: ['CPUExecutionProvider']
+    tags: ['CPUExecutionProvider']
     inputs:
       - name: images:0
         type: INPUT

tests/onnx/benchmarking/classification/onnx_models_configs/googlenet-12.yml

@@ -4,6 +4,11 @@ models:
   - framework: onnx_runtime
     adapter: classification
     execution_providers: ['OpenVINOExecutionProvider']
+    tags: ['OpenVINOExecutionProvider']
+  - framework: onnx_runtime
+    adapter: classification
+    execution_providers: ['CPUExecutionProvider']
+    tags: ['CPUExecutionProvider']
   datasets:
   - name: imagenet_1000_classes
     # read in RGB format

tests/onnx/benchmarking/classification/onnx_models_configs/inception-v1-12.yml

@@ -4,6 +4,11 @@ models:
   - framework: onnx_runtime
     adapter: classification
     execution_providers: ['OpenVINOExecutionProvider']
+    tags: ['OpenVINOExecutionProvider']
+  - framework: onnx_runtime
+    adapter: classification
+    execution_providers: ['CPUExecutionProvider']
+    tags: ['CPUExecutionProvider']
   datasets:
   - name: imagenet_1000_classes
     # read in RGB format

tests/onnx/benchmarking/classification/onnx_models_configs/mobilenetv2-12.yml

@@ -4,6 +4,11 @@ models:
   - framework: onnx_runtime
     adapter: classification
     execution_providers: ['OpenVINOExecutionProvider']
+    tags: ['OpenVINOExecutionProvider']
+  - framework: onnx_runtime
+    adapter: classification
+    execution_providers: ['CPUExecutionProvider']
+    tags: ['CPUExecutionProvider']
   datasets:
   - name: imagenet_1000_classes
     # read in RGB format

tests/onnx/benchmarking/classification/onnx_models_configs/resnet50-v1-12.yml

@@ -4,6 +4,11 @@ models:
   - framework: onnx_runtime
     adapter: classification
     execution_providers: ['OpenVINOExecutionProvider']
+    tags: ['OpenVINOExecutionProvider']
+  - framework: onnx_runtime
+    adapter: classification
+    execution_providers: ['CPUExecutionProvider']
+    tags: ['CPUExecutionProvider']
   datasets:
   - name: imagenet_1000_classes
     # read in RGB format

tests/onnx/benchmarking/classification/onnx_models_configs/resnet50-v2-7.yml

@@ -4,6 +4,11 @@ models:
   - framework: onnx_runtime
     adapter: classification
     execution_providers: ['OpenVINOExecutionProvider']
+    tags: ['OpenVINOExecutionProvider']
+  - framework: onnx_runtime
+    adapter: classification
+    execution_providers: ['CPUExecutionProvider']
+    tags: ['CPUExecutionProvider']
   datasets:
   - name: imagenet_1000_classes
     # read in RGB format

tests/onnx/benchmarking/classification/onnx_models_configs/shufflenet-9.yml

@@ -4,6 +4,11 @@ models:
   - framework: onnx_runtime
     adapter: classification
     execution_providers: ['OpenVINOExecutionProvider']
+    tags: ['OpenVINOExecutionProvider']
+  - framework: onnx_runtime
+    adapter: classification
+    execution_providers: ['CPUExecutionProvider']
+    tags: ['CPUExecutionProvider']
   datasets:
   - name: imagenet_1000_classes
     # read in RGB format

tests/onnx/benchmarking/classification/onnx_models_configs/shufflenet-v2-12.yml

@@ -4,6 +4,11 @@ models:
   - framework: onnx_runtime
     adapter: classification
     execution_providers: ['OpenVINOExecutionProvider']
+    tags: ['OpenVINOExecutionProvider']
+  - framework: onnx_runtime
+    adapter: classification
+    execution_providers: ['CPUExecutionProvider']
+    tags: ['CPUExecutionProvider']
   datasets:
   - name: imagenet_1000_classes
     # read in RGB format

tests/onnx/benchmarking/classification/onnx_models_configs/squeezenet1.0-12.yml

@@ -4,6 +4,11 @@ models:
   - framework: onnx_runtime
     adapter: classification
     execution_providers: ['OpenVINOExecutionProvider']
+    tags: ['OpenVINOExecutionProvider']
+  - framework: onnx_runtime
+    adapter: classification
+    execution_providers: ['CPUExecutionProvider']
+    tags: ['CPUExecutionProvider']
   datasets:
   - name: imagenet_1000_classes
     # read in RGB format

tests/onnx/benchmarking/classification/onnx_models_configs/vgg16-12.yml

@@ -4,6 +4,11 @@ models:
   - framework: onnx_runtime
     adapter: classification
     execution_providers: ['OpenVINOExecutionProvider']
+    tags: ['OpenVINOExecutionProvider']
+  - framework: onnx_runtime
+    adapter: classification
+    execution_providers: ['CPUExecutionProvider']
+    tags: ['CPUExecutionProvider']
   datasets:
   - name: imagenet_1000_classes
     # read in RGB format

tests/onnx/benchmarking/classification/onnx_models_configs/zfnet512-12.yml

@@ -4,6 +4,11 @@ models:
   - framework: onnx_runtime
     adapter: classification
     execution_providers: ['OpenVINOExecutionProvider']
+    tags: ['OpenVINOExecutionProvider']
+  - framework: onnx_runtime
+    adapter: classification
+    execution_providers: ['CPUExecutionProvider']
+    tags: ['CPUExecutionProvider']
   datasets:
   - name: imagenet_1000_classes
     # read in RGB format

tests/onnx/benchmarking/object_detection_segmentation/onnx_models_configs/FasterRCNN-12.yml

@@ -3,6 +3,19 @@ models:
     launchers:
       - framework: onnx_runtime
         execution_providers: [ 'OpenVINOExecutionProvider' ]
+        tags: ['OpenVINOExecutionProvider']
+        adapter:
+          type: faster_rcnn_onnx
+          boxes_out: '6379'
+          scores_out: '6383'
+          labels_out: '6381'
+        inputs:
+          - name: image
+            type: INPUT
+            layout: "CHW"
+      - framework: onnx_runtime
+        execution_providers: [ 'CPUExecutionProvider' ]
+        tags: ['CPUExecutionProvider']
         adapter:
           type: faster_rcnn_onnx
           boxes_out: '6379'
@@ -12,7 +25,6 @@ models:
           - name: image
             type: INPUT
             layout: "CHW"
-
     datasets:
       - name: ms_coco_detection_80_class_with_background
         preprocessing:

tests/onnx/benchmarking/object_detection_segmentation/onnx_models_configs/MaskRCNN-12.yml

@@ -3,6 +3,20 @@ models:
     launchers:
       - framework: onnx_runtime
         execution_providers: [ 'OpenVINOExecutionProvider' ]
+        tags: ['OpenVINOExecutionProvider']
+        adapter:
+          type: mask_rcnn
+          boxes_out: '6568'
+          classes_out: '6570'
+          scores_out: '6572'
+          raw_masks_out: '6887'
+        inputs:
+          - name: image
+            type: INPUT
+            layout: "CHW"
+      - framework: onnx_runtime
+        execution_providers: [ 'CPUExecutionProvider' ]
+        tags: ['CPUExecutionProvider']
         adapter:
           type: mask_rcnn
           boxes_out: '6568'
@@ -38,6 +52,20 @@ models:
     launchers:
       - framework: onnx_runtime
         execution_providers: [ 'OpenVINOExecutionProvider' ]
+        tags: ['OpenVINOExecutionProvider']
+        adapter:
+          type: mask_rcnn
+          boxes_out: '6568'
+          classes_out: '6570'
+          scores_out: '6572'
+          raw_masks_out: '6887'
+        inputs:
+          - name: image
+            type: INPUT
+            layout: "CHW"
+      - framework: onnx_runtime
+        execution_providers: [ 'CPUExecutionProvider' ]
+        tags: ['CPUExecutionProvider']
         adapter:
           type: mask_rcnn
           boxes_out: '6568'

tests/onnx/benchmarking/object_detection_segmentation/onnx_models_configs/ResNet101-DUC-12.yml

@@ -3,6 +3,14 @@ models:
     launchers:
       - framework: onnx_runtime
         execution_providers: ['OpenVINOExecutionProvider']
+        tags: ['OpenVINOExecutionProvider']
+        adapter:
+          type: duc_segmentation
+          ds_rate: 8
+          cell_width: 2
+      - framework: onnx_runtime
+        execution_providers: [ 'CPUExecutionProvider' ]
+        tags: ['CPUExecutionProvider']
         adapter:
           type: duc_segmentation
           ds_rate: 8

tests/onnx/benchmarking/object_detection_segmentation/onnx_models_configs/fcn-resnet50-12.yml

@@ -3,9 +3,14 @@ models:
     launchers:
       - framework: onnx_runtime
         execution_providers: ['OpenVINOExecutionProvider']
+        tags: ['OpenVINOExecutionProvider']
+        adapter:
+          type: segmentation
+      - framework: onnx_runtime
+        execution_providers: ['CPUExecutionProvider']
+        tags: ['CPUExecutionProvider']
         adapter:
           type: segmentation
-
     datasets:
       - name: mscoco_segmentation_to_voc
         annotation_conversion:
@@ -14,6 +19,7 @@ models:
           has_background: True
           sort_annotations: True
           semantic_only: True
+          convert_mask: False
           # Do not change masks_dir path
           masks_dir: mscoco_to_voc_seg_masks
           # Only evaluate on VOC labels

tests/onnx/benchmarking/object_detection_segmentation/onnx_models_configs/retinanet-9.yml

@@ -3,6 +3,18 @@ models:
     launchers:
       - framework: onnx_runtime
         execution_providers: ['OpenVINOExecutionProvider']
+        tags: ['OpenVINOExecutionProvider']
+        adapter:
+          type: retinanet_multihead
+          boxes_outputs: [output6, output7, output8, output9, output10]
+          class_outputs: [output1, output2, output3, output4, output5]
+        inputs:
+          - name: input
+            type: INPUT
+            layout: "NCHW"
+      - framework: onnx_runtime
+        execution_providers: ['CPUExecutionProvider']
+        tags: ['CPUExecutionProvider']
         adapter:
           type: retinanet_multihead
           boxes_outputs: [output6, output7, output8, output9, output10]

tests/onnx/benchmarking/object_detection_segmentation/onnx_models_configs/ssd-12.yml

@@ -3,6 +3,15 @@ models:
     launchers:
       - framework: onnx_runtime
         execution_providers: ['OpenVINOExecutionProvider']
+        tags: ['OpenVINOExecutionProvider']
+        adapter:
+          type: ssd_onnx
+          scores_out: '.*scores*'
+          labels_out: '.*labels*'
+          bboxes_out: '.*bboxes*'
+      - framework: onnx_runtime
+        execution_providers: ['CPUExecutionProvider']
+        tags: ['CPUExecutionProvider']
         adapter:
           type: ssd_onnx
           scores_out: '.*scores*'

tests/onnx/benchmarking/object_detection_segmentation/onnx_models_configs/ssd_mobilenet_v1_12.yml

@@ -3,6 +3,20 @@ models:
     launchers:
       - framework: onnx_runtime
         execution_providers: ['OpenVINOExecutionProvider']
+        tags: ['OpenVINOExecutionProvider']
+        adapter:
+          type: tf_object_detection
+          scores_out: detection_scores
+          classes_out: detection_classes
+          boxes_out: detection_boxes
+          num_detections_out: num_detections
+        inputs:
+          - name: inputs
+            type: INPUT
+            layout: "NHWC"
+      - framework: onnx_runtime
+        execution_providers: ['CPUExecutionProvider']
+        tags: ['CPUExecutionProvider']
         adapter:
           type: tf_object_detection
           scores_out: detection_scores