Merge pull request opencv#18793 from dmatveev:dm/in_graph_metadata

G-API: Introduce runtime in-graph metadata

* G-API: In-graph metadata -- initial implementation

* G-API: Finish the in-graph metadata implementation for Streaming

* G-API: Fix standalone build & warnings for in-graph metadata

* G-API: In-graph meta -- fixed review comments

* G-API: Fix issues with desync causing failing tests

modules/gapi/CMakeLists.txt

@@ -57,6 +57,7 @@ file(GLOB gapi_ext_hdrs
 
 set(gapi_srcs
     # Front-end part
+    src/api/grunarg.cpp
     src/api/gorigin.cpp
     src/api/gmat.cpp
     src/api/garray.cpp
@@ -131,18 +132,19 @@ set(gapi_srcs
     src/backends/ie/giebackend.cpp
     src/backends/ie/giebackend/giewrapper.cpp
 
-    # ONNX Backend.
+    # ONNX backend
     src/backends/onnx/gonnxbackend.cpp
 
-    # Render Backend.
+    # Render backend
     src/backends/render/grenderocv.cpp
     src/backends/render/ft_render.cpp
 
-    #PlaidML Backend
+    # PlaidML Backend
     src/backends/plaidml/gplaidmlcore.cpp
     src/backends/plaidml/gplaidmlbackend.cpp
 
-    # Compound
+    # Common backend code
+    src/backends/common/gmetabackend.cpp
     src/backends/common/gcompoundbackend.cpp
     src/backends/common/gcompoundkernel.cpp
 

modules/gapi/include/opencv2/gapi/garg.hpp

@@ -9,12 +9,14 @@
 #define OPENCV_GAPI_GARG_HPP
 
 #include <vector>
+#include <unordered_map>
 #include <type_traits>
 
 #include <opencv2/gapi/opencv_includes.hpp>
 #include <opencv2/gapi/own/mat.hpp>
 #include <opencv2/gapi/media.hpp>
 
+#include <opencv2/gapi/util/util.hpp>
 #include <opencv2/gapi/util/any.hpp>
 #include <opencv2/gapi/util/variant.hpp>
 
@@ -93,7 +95,7 @@ using GArgs = std::vector<GArg>;
 
 // FIXME: Express as M<GProtoArg...>::type
 // FIXME: Move to a separate file!
-using GRunArg  = util::variant<
+using GRunArgBase  = util::variant<
 #if !defined(GAPI_STANDALONE)
     cv::UMat,
 #endif // !defined(GAPI_STANDALONE)
@@ -105,6 +107,61 @@ using GRunArg  = util::variant<
     cv::detail::OpaqueRef,
     cv::MediaFrame
     >;
+
+namespace detail {
+template<typename,typename>
+struct in_variant;
+
+template<typename T, typename... Types>
+struct in_variant<T, util::variant<Types...> >
+    : std::integral_constant<bool, cv::detail::contains<T, Types...>::value > {
+};
+} // namespace detail
+
+struct GAPI_EXPORTS GRunArg: public GRunArgBase
+{
+    // Metadata information here
+    using Meta = std::unordered_map<std::string, util::any>;
+    Meta meta;
+
+    // Mimic the old GRunArg semantics here, old of the times when
+    // GRunArg was an alias to variant<>
+    GRunArg();
+    GRunArg(const cv::GRunArg &arg);
+    GRunArg(cv::GRunArg &&arg);
+
+    GRunArg& operator= (const GRunArg &arg);
+    GRunArg& operator= (GRunArg &&arg);
+
+    template <typename T>
+    GRunArg(const T &t,
+            const Meta &m = Meta{},
+            typename std::enable_if< detail::in_variant<T, GRunArgBase>::value, int>::type = 0)
+        : GRunArgBase(t)
+        , meta(m)
+    {
+    }
+    template <typename T>
+    GRunArg(T &&t,
+            const Meta &m = Meta{},
+            typename std::enable_if< detail::in_variant<T, GRunArgBase>::value, int>::type = 0)
+        : GRunArgBase(std::move(t))
+        , meta(m)
+    {
+    }
+    template <typename T> auto operator= (const T &t)
+        -> typename std::enable_if< detail::in_variant<T, GRunArgBase>::value, cv::GRunArg>::type&
+    {
+        GRunArgBase::operator=(t);
+        return *this;
+    }
+    template <typename T> auto operator= (T&& t)
+        -> typename std::enable_if< detail::in_variant<T, GRunArgBase>::value, cv::GRunArg>::type&
+    {
+        GRunArgBase::operator=(std::move(t));
+        return *this;
+    }
+};
 using GRunArgs = std::vector<GRunArg>;
 
 // TODO: Think about the addition operator
@@ -129,11 +186,13 @@ namespace gapi
 namespace wip
 {
 /**
- * @brief This aggregate type represents all types which G-API can handle (via variant).
+ * @brief This aggregate type represents all types which G-API can
+ * handle (via variant).
  *
- * It only exists to overcome C++ language limitations (where a `using`-defined class can't be forward-declared).
+ * It only exists to overcome C++ language limitations (where a
+ * `using`-defined class can't be forward-declared).
  */
-struct Data: public GRunArg
+struct GAPI_EXPORTS Data: public GRunArg
 {
     using GRunArg::GRunArg;
     template <typename T>

modules/gapi/include/opencv2/gapi/gopaque.hpp

@@ -15,6 +15,7 @@
 #include <opencv2/gapi/own/exports.hpp>
 #include <opencv2/gapi/opencv_includes.hpp>
 
+#include <opencv2/gapi/util/any.hpp>
 #include <opencv2/gapi/util/variant.hpp>
 #include <opencv2/gapi/util/throw.hpp>
 #include <opencv2/gapi/util/type_traits.hpp>
@@ -119,6 +120,7 @@ namespace detail
 
         virtual void mov(BasicOpaqueRef &ref) = 0;
         virtual const void* ptr() const = 0;
+        virtual void set(const cv::util::any &a) = 0;
     };
 
     template<typename T> class OpaqueRefT final: public BasicOpaqueRef
@@ -212,6 +214,10 @@ namespace detail
         }
 
         virtual const void* ptr() const override { return &rref(); }
+
+        virtual void set(const cv::util::any &a) override {
+            wref() = util::any_cast<T>(a);
+        }
     };
 
     // This class strips type information from OpaqueRefT<> and makes it usable
@@ -285,6 +291,13 @@ namespace detail
 
         // May be used to uniquely identify this object internally
         const void *ptr() const { return m_ref->ptr(); }
+
+        // Introduced for in-graph meta handling
+        OpaqueRef& operator= (const cv::util::any &a)
+        {
+            m_ref->set(a);
+            return *this;
+        }
     };
 } // namespace detail
 

modules/gapi/include/opencv2/gapi/streaming/cap.hpp

@@ -21,9 +21,11 @@
  * Note for developers: please don't put videoio dependency in G-API
  * because of this file.
  */
+#include <chrono>
 
 #include <opencv2/videoio.hpp>
 #include <opencv2/gapi/garg.hpp>
+#include <opencv2/gapi/streaming/meta.hpp>
 
 namespace cv {
 namespace gapi {
@@ -55,6 +57,7 @@ class GCaptureSource: public IStreamSource
     cv::VideoCapture cap;
     cv::Mat first;
     bool first_pulled = false;
+    int64_t counter = 0;
 
     void prep()
     {
@@ -80,19 +83,26 @@ class GCaptureSource: public IStreamSource
             GAPI_Assert(!first.empty());
             first_pulled = true;
             data = first; // no need to clone here since it was cloned already
-            return true;
         }
-
-        if (!cap.isOpened()) return false;
-
-        cv::Mat frame;
-        if (!cap.read(frame))
+        else
         {
-            // end-of-stream happened
-            return false;
+            if (!cap.isOpened()) return false;
+
+            cv::Mat frame;
+            if (!cap.read(frame))
+            {
+                // end-of-stream happened
+                return false;
+            }
+            // Same reason to clone as in prep()
+            data = frame.clone();
         }
-        // Same reason to clone as in prep()
-        data = frame.clone();
+        // Tag data with seq_id/ts
+        const auto now = std::chrono::system_clock::now();
+        const auto dur = std::chrono::duration_cast<std::chrono::microseconds>
+            (now.time_since_epoch());
+        data.meta[cv::gapi::streaming::meta_tag::timestamp] = int64_t{dur.count()};
+        data.meta[cv::gapi::streaming::meta_tag::seq_id]    = int64_t{counter++};
         return true;
     }
 

modules/gapi/include/opencv2/gapi/streaming/meta.hpp

@@ -0,0 +1,79 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2020 Intel Corporation
+
+
+#ifndef OPENCV_GAPI_GSTREAMING_META_HPP
+#define OPENCV_GAPI_GSTREAMING_META_HPP
+
+#include <opencv2/gapi/gopaque.hpp>
+#include <opencv2/gapi/gcall.hpp>
+#include <opencv2/gapi/gkernel.hpp>
+#include <opencv2/gapi/gtype_traits.hpp>
+
+namespace cv {
+namespace gapi {
+namespace streaming {
+
+// FIXME: the name is debatable
+namespace meta_tag {
+static constexpr const char * timestamp = "org.opencv.gapi.meta.timestamp";
+static constexpr const char * seq_id    = "org.opencv.gapi.meta.seq_id";
+} // namespace meta_tag
+
+namespace detail {
+struct GMeta {
+    static const char *id() {
+        return "org.opencv.streaming.meta";
+    }
+    // A universal yield for meta(), same as in GDesync
+    template<typename... R, int... IIs>
+    static std::tuple<R...> yield(cv::GCall &call, cv::detail::Seq<IIs...>) {
+        return std::make_tuple(cv::detail::Yield<R>::yield(call, IIs)...);
+    }
+    // Also a universal outMeta stub here
+    static GMetaArgs getOutMeta(const GMetaArgs &args, const GArgs &) {
+        return args;
+    }
+};
+} // namespace detail
+
+template<typename T, typename G>
+cv::GOpaque<T> meta(G g, const std::string &tag) {
+    using O = cv::GOpaque<T>;
+    cv::GKernel k{
+          detail::GMeta::id()                    // kernel id
+        , tag                                    // kernel tag. Use meta tag here
+        , &detail::GMeta::getOutMeta             // outMeta callback
+        , {cv::detail::GTypeTraits<O>::shape}    // output Shape
+        , {cv::detail::GTypeTraits<G>::op_kind}  // input data kinds
+        , {cv::detail::GObtainCtor<O>::get()}    // output template ctors
+    };
+    cv::GCall call(std::move(k));
+    call.pass(g);
+    return std::get<0>(detail::GMeta::yield<O>(call, cv::detail::MkSeq<1>::type()));
+}
+
+template<typename G>
+cv::GOpaque<int64_t> timestamp(G g) {
+    return meta<int64_t>(g, meta_tag::timestamp);
+}
+
+template<typename G>
+cv::GOpaque<int64_t> seq_id(G g) {
+    return meta<int64_t>(g, meta_tag::seq_id);
+}
+
+template<typename G>
+cv::GOpaque<int64_t> seqNo(G g) {
+    // Old name, compatibility only
+    return seq_id(g);
+}
+
+} // namespace streaming
+} // namespace gapi
+} // namespace cv
+
+#endif // OPENCV_GAPI_GSTREAMING_META_HPP

modules/gapi/src/api/gbackend.cpp

@@ -143,6 +143,14 @@ void bindInArg(Mag& mag, const RcDesc &rc, const GRunArg &arg, HandleRMat handle
         if (handleRMat == HandleRMat::SKIP) return;
         GAPI_Assert(arg.index() == GRunArg::index_of<cv::RMat>());
         bindRMat(mag, rc, util::get<cv::RMat>(arg), RMat::Access::R);
+
+        // FIXME: Here meta may^WWILL be copied multiple times!
+        // Replace it is reference-counted object?
+        mag.meta<cv::RMat>()[rc.id] = arg.meta;
+        mag.meta<cv::Mat>()[rc.id] = arg.meta;
+#if !defined(GAPI_STANDALONE)
+        mag.meta<cv::UMat>()[rc.id] = arg.meta;
+#endif
         break;
     }
 
@@ -154,19 +162,23 @@ void bindInArg(Mag& mag, const RcDesc &rc, const GRunArg &arg, HandleRMat handle
         case GRunArg::index_of<cv::Scalar>() : mag_scalar = util::get<cv::Scalar>(arg);    break;
         default: util::throw_error(std::logic_error("content type of the runtime argument does not match to resource description ?"));
         }
+        mag.meta<cv::Scalar>()[rc.id] = arg.meta;
         break;
     }
 
     case GShape::GARRAY:
-        mag.template slot<cv::detail::VectorRef>()[rc.id] = util::get<cv::detail::VectorRef>(arg);
+        mag.slot<cv::detail::VectorRef>()[rc.id] = util::get<cv::detail::VectorRef>(arg);
+        mag.meta<cv::detail::VectorRef>()[rc.id] = arg.meta;
         break;
 
     case GShape::GOPAQUE:
-        mag.template slot<cv::detail::OpaqueRef>()[rc.id] = util::get<cv::detail::OpaqueRef>(arg);
+        mag.slot<cv::detail::OpaqueRef>()[rc.id] = util::get<cv::detail::OpaqueRef>(arg);
+        mag.meta<cv::detail::OpaqueRef>()[rc.id] = arg.meta;
         break;
 
     case GShape::GFRAME:
-        mag.template slot<cv::MediaFrame>()[rc.id] = util::get<cv::MediaFrame>(arg);
+        mag.slot<cv::MediaFrame>()[rc.id] = util::get<cv::MediaFrame>(arg);
+        mag.meta<cv::MediaFrame>()[rc.id] = arg.meta;
         break;
 
     default:
@@ -250,13 +262,23 @@ cv::GRunArg getArg(const Mag& mag, const RcDesc &ref)
     // Wrap associated CPU object (either host or an internal one)
     switch (ref.shape)
     {
-    case GShape::GMAT:    return GRunArg(mag.template slot<cv::RMat>().at(ref.id));
-    case GShape::GSCALAR: return GRunArg(mag.template slot<cv::Scalar>().at(ref.id));
+    case GShape::GMAT:
+        return GRunArg(mag.slot<cv::RMat>().at(ref.id),
+                       mag.meta<cv::RMat>().at(ref.id));
+    case GShape::GSCALAR:
+        return GRunArg(mag.slot<cv::Scalar>().at(ref.id),
+                       mag.meta<cv::Scalar>().at(ref.id));
     // Note: .at() is intentional for GArray and GOpaque as objects MUST be already there
     //   (and constructed by either bindIn/Out or resetInternal)
-    case GShape::GARRAY:  return GRunArg(mag.template slot<cv::detail::VectorRef>().at(ref.id));
-    case GShape::GOPAQUE: return GRunArg(mag.template slot<cv::detail::OpaqueRef>().at(ref.id));
-    case GShape::GFRAME:  return GRunArg(mag.template slot<cv::MediaFrame>().at(ref.id));
+    case GShape::GARRAY:
+        return GRunArg(mag.slot<cv::detail::VectorRef>().at(ref.id),
+                       mag.meta<cv::detail::VectorRef>().at(ref.id));
+    case GShape::GOPAQUE:
+        return GRunArg(mag.slot<cv::detail::OpaqueRef>().at(ref.id),
+                       mag.meta<cv::detail::OpaqueRef>().at(ref.id));
+    case GShape::GFRAME:
+        return GRunArg(mag.slot<cv::MediaFrame>().at(ref.id),
+                       mag.meta<cv::MediaFrame>().at(ref.id));
     default:
         util::throw_error(std::logic_error("Unsupported GShape type"));
         break;