[CONFORMANCE] Cleaning up (openvinotoolkit#20835)

* 1. Add Op_conformance_utils lib. 2.rename targets. 3. Remove behavior folder from api conformance 4. Remove ov1.0 from opImplCheck 5. move tests and insts to one file 6. small refactoring for op conformance test names * align namespaces, remove test_utils from subgraphs_dumper * update target names * Remove extra * Fix the tests without shapes in meta * fix failed test names * Add attribute reader * Fix buils * Check the registered lib * try * add comparator * Revert "try" This reverts commit 27eaa4e. * try to fix win build * types * tr * remove pragma once
syaifulnizamiphone7 · Nov 16, 2023 · d1e1555 · d1e1555
1 parent 3fbef0d
commit d1e1555
Show file tree

Hide file tree

Showing 96 changed files with 2,655 additions and 2,380 deletions.
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -818,11 +818,11 @@ jobs:
           ${INSTALL_TEST_DIR}/ov_cpu_unit_tests --gtest_print_time=1 \
                 --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CPUUnitTests.xml
 
-      - name: SubgraphsDumper tests
+      - name: ov_subgraphs_dumper_tests tests
         run: |
           source ${INSTALL_DIR}/setupvars.sh
-          ${INSTALL_TEST_DIR}/subgraphsDumperTests --gtest_print_time=1 \
-                --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-SubgraphsDumperTests.xml
+          ${INSTALL_TEST_DIR}/ov_subgraphs_dumper_tests --gtest_print_time=1 \
+                --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_subgraphs_dumper_tests.xml
 
       - name: Template OpImpl tests
         run: |

diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml
@@ -457,11 +457,11 @@ jobs:
           ${{ env.INSTALL_TEST_DIR }}/ov_cpu_unit_tests --gtest_print_time=1 \
                 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-CPUUnitTests.xml
 
-      - name: SubgraphsDumper tests
+      - name: ov_subgraphs_dumper_tests tests
         run: |
           source ${{ env.INSTALL_DIR }}/setupvars.sh
-          ${{ env.INSTALL_TEST_DIR }}/subgraphsDumperTests --gtest_print_time=1 \
-                --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-SubgraphsDumperTests.xml
+          ${{ env.INSTALL_TEST_DIR }}/ov_subgraphs_dumper_tests --gtest_print_time=1 \
+                --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_subgraphs_dumper_tests.xml
 
       - name: Template OpImpl tests
         run: |

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -599,10 +599,10 @@ jobs:
         run: |
           call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_cpu_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-CPUUnitTests.xml
 
-      - name: SubgraphsDumper tests
+      - name: ov_subgraphs_dumper_tests tests
         shell: cmd
         run: |
-          call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/subgraphsDumperTests   --gtest_print_time=1 --device=TEMPLATE --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-SubgraphsDumperTests.xml
+          call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_subgraphs_dumper_tests --gtest_print_time=1 --device=TEMPLATE --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-SubgraphsDumperTests.xml
 
       - name: Template OpImpl tests
         shell: cmd

diff --git a/src/tests/functional/plugin/conformance/CMakeLists.txt b/src/tests/functional/plugin/conformance/CMakeLists.txt
@@ -6,4 +6,5 @@ add_subdirectory(test_runner/conformance_infra)
 add_subdirectory(test_runner/api_conformance_runner)
 add_subdirectory(test_runner/op_conformance_runner)
 add_subdirectory(subgraphs_dumper)
+add_subdirectory(op_conformance_utils)
 add_subdirectory(subgraphs_dumper/tests)
diff --git a/src/tests/functional/plugin/conformance/op_conformance_utils/CMakeLists.txt b/src/tests/functional/plugin/conformance/op_conformance_utils/CMakeLists.txt
@@ -0,0 +1,29 @@
+# Copyright (C) 2018-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set(TARGET_NAME op_conformance_utils)
+
+list(APPEND LIBRARIES
+        openvino::runtime
+        openvino::pugixml
+        openvino::util
+)
+
+# add op_conformance_utils lib to get API
+ov_add_target(
+        NAME "${TARGET_NAME}"
+        TYPE STATIC
+        ROOT "${CMAKE_CURRENT_SOURCE_DIR}/src"
+        INCLUDES
+            PUBLIC
+                "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>"
+        LINK_LIBRARIES
+            PUBLIC
+                ${LIBRARIES}
+        DEPENDENCIES
+            ${LIBRARIES}
+        ADD_CPPLINT
+)
+
+ov_build_target_faster(${TARGET_NAME} UNITY)
diff --git a/..._dumper/include/cache/meta/input_info.hpp → ...onformance_utils/meta_info/input_info.hpp b/..._dumper/include/cache/meta/input_info.hpp → ...onformance_utils/meta_info/input_info.hpp
@@ -5,11 +5,12 @@
 #pragma once
 
 #include <map>
-#include "openvino/openvino.hpp"
+
+#include "openvino/core/shape.hpp"
+#include "openvino/core/partial_shape.hpp"
 
 namespace ov {
-namespace tools {
-namespace subgraph_dumper {
+namespace conformance {
 
 constexpr double DEFAULT_MIN_VALUE = std::numeric_limits<double>::min();
 constexpr double DEFAULT_MAX_VALUE = std::numeric_limits<double>::max();
@@ -73,9 +74,5 @@ struct InputInfo {
     }
 };
 
-// ov_model, input_info, extractor_name
-using ExtractedPattern = std::tuple<std::shared_ptr<ov::Model>, std::map<std::string, InputInfo>, std::string>;
-
-}  // namespace subgraph_dumper
-}  // namespace tools
+}  // namespace conformance
 }  // namespace ov
diff --git a/...s_dumper/include/cache/meta/meta_info.hpp → ...conformance_utils/meta_info/meta_info.hpp b/...s_dumper/include/cache/meta/meta_info.hpp → ...conformance_utils/meta_info/meta_info.hpp
@@ -4,12 +4,15 @@
 
 #pragma once
 
-#include "cache/meta/input_info.hpp"
-#include "cache/meta/model_info.hpp"
+#include <string>
+#include <map>
+#include <unordered_set>
+
+#include "op_conformance_utils/meta_info/input_info.hpp"
+#include "op_conformance_utils/meta_info/model_info.hpp"
 
 namespace ov {
-namespace tools {
-namespace subgraph_dumper {
+namespace conformance {
 
 class MetaInfo {
 public:
@@ -21,10 +24,12 @@ class MetaInfo {
              size_t model_priority = 1);
     MetaInfo(std::map<std::string, InputInfo> _in_info,
              std::map<std::string, ModelInfo> _model_info,
-             std::unordered_set<std::string> _extractors) :
+             std::unordered_set<std::string> _extractors,
+             double _graph_priority = 0) :
              model_info(_model_info),
              input_info(_in_info),
-             extractors(_extractors) {};
+             extractors(_extractors),
+             graph_priority(_graph_priority) {};
     void serialize(const std::string& serialization_path);
     void update(const std::string& model_path,
                 const std::map<std::string, InputInfo>& _input_info,
@@ -39,8 +44,9 @@ class MetaInfo {
     };
     std::map<std::string, ModelInfo> get_model_info() const;
     std::string get_any_extractor() const { return *extractors.begin(); }
+    double get_graph_priority();
 
-    static MetaInfo read_meta_from_file(const std::string& meta_path);
+    static MetaInfo read_meta_from_file(const std::string& meta_path, bool read_priority = false);
 
 protected:
     // { input_node_name: input_info }
@@ -49,18 +55,17 @@ class MetaInfo {
     std::map<std::string, ModelInfo> model_info;
     // { extractors }
     std::unordered_set<std::string> extractors;
+    double graph_priority = 0;
 
     // to store model priority ranges to normilize graph_priority
     static unsigned long MAX_MODEL_PRIORITY;
     static unsigned long MIN_MODEL_PRIORITY;
 
-    double get_graph_priority();
     std::string get_model_name_by_path(const std::string& model_path);
 
     // get abs priority graph before normalization
     unsigned long get_abs_graph_priority();
 };
 
-}  // namespace subgraph_dumper
-}  // namespace tools
+}  // namespace conformance
 }  // namespace ov
diff --git a/..._dumper/include/cache/meta/model_info.hpp → ...onformance_utils/meta_info/model_info.hpp b/..._dumper/include/cache/meta/model_info.hpp → ...onformance_utils/meta_info/model_info.hpp
@@ -8,8 +8,7 @@
 #include <string>
 
 namespace ov {
-namespace tools {
-namespace subgraph_dumper {
+namespace conformance {
 
 struct ModelInfo {
     std::set<std::string> model_paths;
@@ -37,6 +36,5 @@ struct ModelInfo {
     }
 };
 
-}  // namespace subgraph_dumper
-}  // namespace tools
+}  // namespace conformance
 }  // namespace ov
diff --git a/...bgraphs_dumper/include/utils/dynamism.hpp → ...e/op_conformance_utils/utils/dynamism.hpp b/...bgraphs_dumper/include/utils/dynamism.hpp → ...e/op_conformance_utils/utils/dynamism.hpp
@@ -7,8 +7,7 @@
 #include "openvino/core/model.hpp"
 
 namespace ov {
-namespace tools {
-namespace subgraph_dumper {
+namespace util {
 
 inline bool is_dynamic_node(const std::shared_ptr<ov::Node>& node) {
     for (size_t i = 0; i < node->get_input_size(); ++i) {
@@ -38,6 +37,5 @@ inline bool is_dynamic_model(const std::shared_ptr<ov::Model>& model) {
     return false;
 }
 
-}  // namespace subgraph_dumper
-}  // namespace tools
+}  // namespace util
 }  // namespace ov
diff --git a/...ional/plugin/conformance/op_conformance_utils/include/op_conformance_utils/utils/file.hpp b/...ional/plugin/conformance/op_conformance_utils/include/op_conformance_utils/utils/file.hpp
@@ -0,0 +1,34 @@
+// Copyright (C) 2018-2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <vector>
+#include <string>
+#include <regex>
+
+namespace ov {
+namespace util {
+
+std::vector<std::string>
+get_filelist_recursive(const std::vector<std::string>& dir_paths,
+                       const std::vector<std::regex>& patterns);
+
+std::vector<std::string>
+read_lst_file(const std::vector<std::string>& file_paths,
+              const std::vector<std::regex>& patterns = {std::regex(".*")});
+
+std::string replace_extension(std::string file, const std::string& new_extension);
+
+inline void remove_path(const std::string& path) {
+    if (!path.empty()) {
+        std::remove(path.c_str());
+    }
+}
+
+std::vector<std::string>
+split_str(std::string paths, const char delimiter = ',');
+
+}  // namespace util
+}  // namespace ov
diff --git a/...raphs_dumper/src/cache/meta/meta_info.cpp → ...ormance_utils/src/meta_info/meta_info.cpp b/...raphs_dumper/src/cache/meta/meta_info.cpp → ...ormance_utils/src/meta_info/meta_info.cpp
@@ -4,13 +4,11 @@
 
 #include "pugixml.hpp"
 
-#include "common_test_utils/file_utils.hpp"
-
-#include "cache/meta/meta_info.hpp"
+#include "op_conformance_utils/meta_info/meta_info.hpp"
+#include "op_conformance_utils/utils/file.hpp"
 
 namespace ov {
-namespace tools {
-namespace subgraph_dumper {
+namespace conformance {
 
 unsigned long MetaInfo::MIN_MODEL_PRIORITY = std::numeric_limits<unsigned long>::max();
 unsigned long MetaInfo::MAX_MODEL_PRIORITY = std::numeric_limits<unsigned long>::min();
@@ -44,10 +42,13 @@ unsigned long MetaInfo::get_abs_graph_priority() {
 }
 
 double MetaInfo::get_graph_priority() {
-    auto delta = MAX_MODEL_PRIORITY - MIN_MODEL_PRIORITY == 0 ? 1 : MAX_MODEL_PRIORITY - MIN_MODEL_PRIORITY;
-    // return normilized graph priority from [0, 1]
-    double diff = get_abs_graph_priority() - MIN_MODEL_PRIORITY;
-    return diff / delta;
+    if (graph_priority == 0) {
+        auto delta = MAX_MODEL_PRIORITY - MIN_MODEL_PRIORITY == 0 ? 1 : MAX_MODEL_PRIORITY - MIN_MODEL_PRIORITY;
+        // return normilized graph priority from [0, 1]
+        double diff = get_abs_graph_priority() - MIN_MODEL_PRIORITY;
+        return diff / delta;
+    }
+    return graph_priority;
 }
 
 inline ov::PartialShape str_to_ov_shape(std::string str) {
@@ -65,7 +66,7 @@ inline ov::PartialShape str_to_ov_shape(std::string str) {
     return ov::PartialShape{shape_vec};
 }
 
-MetaInfo MetaInfo::read_meta_from_file(const std::string& meta_path) {
+MetaInfo MetaInfo::read_meta_from_file(const std::string& meta_path, bool read_priority) {
     pugi::xml_document doc;
     doc.load_file(meta_path.c_str());
     std::map<std::string, ModelInfo> model_info;
@@ -87,7 +88,7 @@ MetaInfo MetaInfo::read_meta_from_file(const std::string& meta_path) {
         auto input_info_xml = doc.child("meta_info").child("input_info");
         for (const auto &input : input_info_xml.children()) {
             auto in_name = std::string(input.attribute("id").value());
-            ov::tools::subgraph_dumper::InputInfo in_info;
+            InputInfo in_info;
             in_info.is_const = input.attribute("convert_to_const").as_bool();
             if (std::string(input.attribute("min").value()) != "undefined") {
                 in_info.ranges.min = input.attribute("min").as_double();
@@ -100,10 +101,18 @@ MetaInfo MetaInfo::read_meta_from_file(const std::string& meta_path) {
                 in_info.ranges.max = DEFAULT_MAX_VALUE;
             }
             {
-                auto max_shape_str = std::string(input.attribute("max_shape").value());
-                in_info.max_shape = str_to_ov_shape(max_shape_str);
-                auto min_shape_str = std::string(input.attribute("min_shape").value());
-                in_info.min_shape = str_to_ov_shape(min_shape_str);
+                try {
+                    auto max_shape_str = std::string(input.attribute("max_shape").value());
+                    in_info.max_shape = str_to_ov_shape(max_shape_str);
+                } catch (std::exception) {
+                    in_info.max_shape = ov::PartialShape();
+                }
+                try {
+                    auto min_shape_str = std::string(input.attribute("min_shape").value());
+                    in_info.min_shape = str_to_ov_shape(min_shape_str);
+                } catch (std::exception) {
+                    in_info.min_shape = ov::PartialShape();
+                }
             }
             input_info.insert({in_name, in_info});
         }
@@ -115,7 +124,9 @@ MetaInfo MetaInfo::read_meta_from_file(const std::string& meta_path) {
             extractors.insert(std::string(extractor.attribute("name").value()));
         }
     }
-    auto new_meta = MetaInfo(input_info, model_info, extractors);
+    double graph_priority = read_priority ? doc.child("meta_info").child("graph_priority").attribute("value").as_double() : 0;
+
+    auto new_meta = MetaInfo(input_info, model_info, extractors, graph_priority);
     return new_meta;
 }
 
@@ -157,8 +168,8 @@ void MetaInfo::serialize(const std::string& serialization_path) {
                 input_node.append_attribute("max").set_value(input.second.ranges.max);
             }
             input_node.append_attribute("convert_to_const").set_value(input.second.is_const);
-            input_node.append_attribute("max_shape").set_value(ov::test::utils::partialShape2str({ input.second.max_shape }).c_str());
-            input_node.append_attribute("min_shape").set_value(ov::test::utils::partialShape2str({ input.second.min_shape }).c_str());
+            input_node.append_attribute("max_shape").set_value(input.second.max_shape.to_string().c_str());
+            input_node.append_attribute("min_shape").set_value(input.second.min_shape.to_string().c_str());
         }
         doc.save_file(serialization_path.c_str());
 }
@@ -215,11 +226,16 @@ std::map<std::string, ModelInfo> MetaInfo::get_model_info() const {
 }
 
 std::string MetaInfo::get_model_name_by_path(const std::string& model_path) {
-    auto pos = model_path.rfind(ov::test::utils::FileSeparator);
-    auto model_name = ov::test::utils::replaceExt(model_path.substr(pos + 1), "");
-    return model_name;
+    constexpr const auto file_separator =
+    #ifdef _WIN32
+            '\\';
+    #else
+            '/';
+    #endif
+
+    auto model_name = ov::util::split_str(model_path, file_separator).back();
+    return ov::util::replace_extension(model_name, "");
 }
 
-}  // namespace subgraph_dumper
-}  // namespace tools
+}  // namespace conformance
 }  // namespace ov