feat: render pose roi and detection

homuler · Jan 28, 2021 · 026b6c8 · 026b6c8
1 parent 6a26384
commit 026b6c8
Show file tree

Hide file tree

Showing 4 changed files with 80 additions and 21 deletions.
diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Objects/HolisticDesktopGPU.prefab b/Assets/MediaPipe/Examples/Graphs/Holistic/Objects/HolisticDesktopGPU.prefab
@@ -59,6 +59,10 @@ MonoBehaviour:
   m_EditorClassIdentifier: 
   poseLandmarksPrefab: {fileID: 7166525400038315749, guid: 69cf28587972333cca1eb4c1a1ec7e6d,
     type: 3}
+  poseRoiPrefab: {fileID: 1405412484651109527, guid: c5c846cc7bbefdfa7af64f9d29912b70,
+    type: 3}
+  poseDetectionPrefab: {fileID: 1405412484651109527, guid: 3779864e7ead10e31b03fc362f4746de,
+    type: 3}
   faceLandmarksPrefab: {fileID: 2250723881323860951, guid: 570a388dca04100b8b491a9bc4ef1d51,
     type: 3}
   handLandmarksPrefab: {fileID: 3100741085041498678, guid: 4eb3a6d7cfea73afdb656b7166ae1a8d,

diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticAnnotationController.cs b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticAnnotationController.cs
@@ -3,39 +3,51 @@
 
 public class HolisticAnnotationController : AnnotationController {
   [SerializeField] GameObject poseLandmarksPrefab = null;
+  [SerializeField] GameObject poseRoiPrefab = null;
+  [SerializeField] GameObject poseDetectionPrefab = null;
   [SerializeField] GameObject faceLandmarksPrefab = null;
   [SerializeField] GameObject handLandmarksPrefab = null;
 
   private GameObject poseLandmarksAnnotation;
+  private GameObject poseRoiAnnotation;
+  private GameObject poseDetectionAnnotation;
   private GameObject faceLandmarksAnnotation;
   private GameObject leftHandLandmarksAnnotation;
   private GameObject rightHandLandmarksAnnotation;
 
   void Awake() {
     poseLandmarksAnnotation = Instantiate(poseLandmarksPrefab);
+    poseRoiAnnotation = Instantiate(poseRoiPrefab);
+    poseDetectionAnnotation = Instantiate(poseDetectionPrefab);
     faceLandmarksAnnotation = Instantiate(faceLandmarksPrefab);
     leftHandLandmarksAnnotation = Instantiate(handLandmarksPrefab);
     rightHandLandmarksAnnotation = Instantiate(handLandmarksPrefab);
   }
 
   void OnDestroy() {
     Destroy(poseLandmarksAnnotation);
+    Destroy(poseRoiAnnotation);
+    Destroy(poseDetectionAnnotation);
     Destroy(faceLandmarksAnnotation);
     Destroy(leftHandLandmarksAnnotation);
     Destroy(rightHandLandmarksAnnotation);
   }
 
   public override void Clear() {
     poseLandmarksAnnotation.GetComponent<FullBodyPoseLandmarkListAnnotationController>().Clear();
+    poseRoiAnnotation.GetComponent<RectAnnotationController>().Clear();
+    poseDetectionAnnotation.GetComponent<DetectionAnnotationController>().Clear();
     faceLandmarksAnnotation.GetComponent<FaceLandmarkListAnnotationController>().Clear();
     leftHandLandmarksAnnotation.GetComponent<HandLandmarkListAnnotationController>().Clear();
     rightHandLandmarksAnnotation.GetComponent<HandLandmarkListAnnotationController>().Clear();
   }
 
-  public void Draw(Transform screenTransform, NormalizedLandmarkList poseLandmarks, NormalizedLandmarkList faceLandmarks,
-                   NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks, bool isFlipped = false)
+  public void Draw(Transform screenTransform, NormalizedLandmarkList poseLandmarks, NormalizedRect poseRoi, Detection poseDetection,
+                   NormalizedLandmarkList faceLandmarks, NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks, bool isFlipped = false)
   {
     poseLandmarksAnnotation.GetComponent<FullBodyPoseLandmarkListAnnotationController>().Draw(screenTransform, poseLandmarks, isFlipped);
+    poseRoiAnnotation.GetComponent<RectAnnotationController>().Draw(screenTransform, poseRoi, isFlipped);
+    poseDetectionAnnotation.GetComponent<DetectionAnnotationController>().Draw(screenTransform, poseDetection, isFlipped);
     faceLandmarksAnnotation.GetComponent<FaceLandmarkListAnnotationController>().Draw(screenTransform, faceLandmarks, isFlipped);
     leftHandLandmarksAnnotation.GetComponent<HandLandmarkListAnnotationController>().Draw(screenTransform, leftHandLandmarks, isFlipped);
     rightHandLandmarksAnnotation.GetComponent<HandLandmarkListAnnotationController>().Draw(screenTransform, rightHandLandmarks, isFlipped);

diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticGraph.cs b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticGraph.cs
@@ -1,11 +1,18 @@
 using Mediapipe;
-using System.Collections.Generic;
 
 public class HolisticGraph : DemoGraph {
   private const string poseLandmarksStream = "pose_landmarks";
   private OutputStreamPoller<NormalizedLandmarkList> poseLandmarksStreamPoller;
   private NormalizedLandmarkListPacket poseLandmarksPacket;
 
+  private const string poseRoiStream = "pose_roi";
+  private OutputStreamPoller<NormalizedRect> poseRoiStreamPoller;
+  private NormalizedRectPacket poseRoiPacket;
+
+  private const string poseDetectionStream = "pose_detection";
+  private OutputStreamPoller<Detection> poseDetectionStreamPoller;
+  private DetectionPacket poseDetectionPacket;
+
   private const string faceLandmarksStream = "face_landmarks";
   private OutputStreamPoller<NormalizedLandmarkList> faceLandmarksStreamPoller;
   private NormalizedLandmarkListPacket faceLandmarksPacket;
@@ -18,14 +25,18 @@ public class HolisticGraph : DemoGraph {
   private OutputStreamPoller<NormalizedLandmarkList> rightHandLandmarksStreamPoller;
   private NormalizedLandmarkListPacket rightHandLandmarksPacket;
 
-  private const string poseDetectionStream = "pose_detection";
-  private OutputStreamPoller<Detection> poseDetectionStreamPoller;
-  private DetectionPacket poseDetectionPacket;
-
   private const string poseLandmarksPresenceStream = "pose_landmarks_presence";
   private OutputStreamPoller<bool> poseLandmarksPresenceStreamPoller;
   private BoolPacket poseLandmarksPresencePacket;
 
+  private const string poseRoiPresenceStream = "pose_roi_presence";
+  private OutputStreamPoller<bool> poseRoiPresenceStreamPoller;
+  private BoolPacket poseRoiPresencePacket;
+
+  private const string poseDetectionPresenceStream = "pose_detection_presence";
+  private OutputStreamPoller<bool> poseDetectionPresenceStreamPoller;
+  private BoolPacket poseDetectionPresencePacket;
+
   private const string faceLandmarksPresenceStream = "face_landmarks_presence";
   private OutputStreamPoller<bool> faceLandmarksPresenceStreamPoller;
   private BoolPacket faceLandmarksPresencePacket;
@@ -44,6 +55,12 @@ public override Status StartRun() {
     poseLandmarksStreamPoller = graph.AddOutputStreamPoller<NormalizedLandmarkList>(poseLandmarksStream).ConsumeValueOrDie();
     poseLandmarksPacket = new NormalizedLandmarkListPacket();
 
+    poseRoiStreamPoller = graph.AddOutputStreamPoller<NormalizedRect>(poseRoiStream).ConsumeValueOrDie();
+    poseRoiPacket = new NormalizedRectPacket();
+
+    poseDetectionStreamPoller = graph.AddOutputStreamPoller<Detection>(poseDetectionStream).ConsumeValueOrDie();
+    poseDetectionPacket = new DetectionPacket();
+
     faceLandmarksStreamPoller = graph.AddOutputStreamPoller<NormalizedLandmarkList>(faceLandmarksStream).ConsumeValueOrDie();
     faceLandmarksPacket = new NormalizedLandmarkListPacket();
 
@@ -53,12 +70,15 @@ public override Status StartRun() {
     rightHandLandmarksStreamPoller = graph.AddOutputStreamPoller<NormalizedLandmarkList>(rightHandLandmarksStream).ConsumeValueOrDie();
     rightHandLandmarksPacket = new NormalizedLandmarkListPacket();
 
-    poseDetectionStreamPoller = graph.AddOutputStreamPoller<Detection>(poseDetectionStream).ConsumeValueOrDie();
-    poseDetectionPacket = new DetectionPacket();
-
     poseLandmarksPresenceStreamPoller = graph.AddOutputStreamPoller<bool>(poseLandmarksPresenceStream).ConsumeValueOrDie();
     poseLandmarksPresencePacket = new BoolPacket();
 
+    poseRoiPresenceStreamPoller = graph.AddOutputStreamPoller<bool>(poseRoiPresenceStream).ConsumeValueOrDie();
+    poseRoiPresencePacket = new BoolPacket();
+
+    poseDetectionPresenceStreamPoller = graph.AddOutputStreamPoller<bool>(poseDetectionPresenceStream).ConsumeValueOrDie();
+    poseDetectionPresencePacket = new BoolPacket();
+
     faceLandmarksPresenceStreamPoller = graph.AddOutputStreamPoller<bool>(faceLandmarksPresenceStream).ConsumeValueOrDie();
     faceLandmarksPresencePacket = new BoolPacket();
 
@@ -81,24 +101,35 @@ public override void RenderOutput(WebCamScreenController screenController, Textu
   }
 
   private HolisticValue FetchNextHolisticValue() {
-    var isFaceLandmarksPresent = FetchNextFaceLandmarksPresence();
     var isPoseLandmarksPresent = FetchNextPoseLandmarksPresence();
+    var isPoseRoiPresent = FetchNextPoseRoiPresence();
+    var isPoseDetectionPresent = FetchNextPoseDetectionPresence();
+    var isFaceLandmarksPresent = FetchNextFaceLandmarksPresence();
     var isLeftHandLandmarksPresent = FetchNextLeftHandLandmarksPresence();
     var isRightHandLandmarksPresent = FetchNextRightHandLandmarksPresence();
 
     var poseLandmarks = isPoseLandmarksPresent ? FetchNextPoseLandmarks() : new NormalizedLandmarkList();
+    var poseRoi = isPoseRoiPresent ? FetchNextPoseRoi() : new NormalizedRect();
+    var poseDetection = isPoseDetectionPresent ? FetchNextPoseDetection() : new Detection();
     var faceLandmarks = isFaceLandmarksPresent ? FetchNextFaceLandmarks() : new NormalizedLandmarkList();
     var leftHandLandmarks = isLeftHandLandmarksPresent ? FetchNextLeftHandLandmarks() : new NormalizedLandmarkList();
     var rightHandLandmarks = isRightHandLandmarksPresent ? FetchNextRightHandLandmarks() : new NormalizedLandmarkList();
 
-    UnityEngine.Debug.Log(poseLandmarks.Landmark.Count);
-    return new HolisticValue(poseLandmarks, faceLandmarks, leftHandLandmarks, rightHandLandmarks);
+    return new HolisticValue(poseLandmarks, poseRoi, poseDetection, faceLandmarks, leftHandLandmarks, rightHandLandmarks);
   }
 
   private NormalizedLandmarkList FetchNextPoseLandmarks() {
     return FetchNext(poseLandmarksStreamPoller, poseLandmarksPacket, poseLandmarksStream);
   }
 
+  private NormalizedRect FetchNextPoseRoi() {
+    return FetchNext(poseRoiStreamPoller, poseRoiPacket, poseRoiStream);
+  }
+
+  private Detection FetchNextPoseDetection() {
+    return FetchNext(poseDetectionStreamPoller, poseDetectionPacket, poseDetectionStream);
+  }
+
   private NormalizedLandmarkList FetchNextFaceLandmarks() {
     return FetchNext(faceLandmarksStreamPoller, faceLandmarksPacket, faceLandmarksStream);
   }
@@ -115,6 +146,14 @@ private bool FetchNextPoseLandmarksPresence() {
     return FetchNext(poseLandmarksPresenceStreamPoller, poseLandmarksPresencePacket, poseLandmarksPresenceStream);
   }
 
+  private bool FetchNextPoseRoiPresence() {
+    return FetchNext(poseRoiPresenceStreamPoller, poseRoiPresencePacket, poseRoiPresenceStream);
+  }
+
+  private bool FetchNextPoseDetectionPresence() {
+    return FetchNext(poseDetectionPresenceStreamPoller, poseDetectionPresencePacket, poseDetectionPresenceStream);
+  }
+
   private bool FetchNextFaceLandmarksPresence() {
     return FetchNext(faceLandmarksPresenceStreamPoller, faceLandmarksPresencePacket, faceLandmarksPresenceStream);
   }
@@ -130,6 +169,7 @@ private bool FetchNextRightHandLandmarksPresence() {
   private void RenderAnnotation(WebCamScreenController screenController, HolisticValue value) {
     // NOTE: input image is flipped
     GetComponent<HolisticAnnotationController>().Draw(
-      screenController.transform, value.PoseLandmarks, value.FaceLandmarks, value.LeftHandLandmarks, value.RightHandLandmarks, true);
+      screenController.transform, value.PoseLandmarks, value.PoseRoi, value.PoseDetection,
+      value.FaceLandmarks, value.LeftHandLandmarks, value.RightHandLandmarks, true);
   }
 }
diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticValue.cs b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticValue.cs
@@ -1,17 +1,20 @@
 using Mediapipe;
-using System.Collections.Generic;
 
 class HolisticValue {
   public readonly NormalizedLandmarkList PoseLandmarks;
+  public readonly NormalizedRect PoseRoi;
+  public readonly Detection PoseDetection;
   public readonly NormalizedLandmarkList FaceLandmarks;
   public readonly NormalizedLandmarkList LeftHandLandmarks;
   public readonly NormalizedLandmarkList RightHandLandmarks;
 
-  public HolisticValue(NormalizedLandmarkList poseLandmarks, NormalizedLandmarkList faceLandmarks,
-                       NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks) {
-    PoseLandmarks = poseLandmarks;
-    FaceLandmarks = faceLandmarks;
-    LeftHandLandmarks = leftHandLandmarks;
-    RightHandLandmarks = rightHandLandmarks;
+  public HolisticValue(NormalizedLandmarkList PoseLandmarks, NormalizedRect PoseRoi, Detection PoseDetection,
+                       NormalizedLandmarkList FaceLandmarks, NormalizedLandmarkList LeftHandLandmarks, NormalizedLandmarkList RightHandLandmarks) {
+    this.PoseLandmarks = PoseLandmarks;
+    this.PoseRoi = PoseRoi;
+    this.PoseDetection = PoseDetection;
+    this.FaceLandmarks = FaceLandmarks;
+    this.LeftHandLandmarks = LeftHandLandmarks;
+    this.RightHandLandmarks = RightHandLandmarks;
   }
 }