From 026b6c833a35165e471616e7f621d974778b246c Mon Sep 17 00:00:00 2001 From: homuler Date: Thu, 28 Jan 2021 20:56:12 +0900 Subject: [PATCH] feat: render pose roi and detection --- .../Objects/HolisticDesktopGPU.prefab | 4 ++ .../Scripts/HolisticAnnotationController.cs | 16 ++++- .../Graphs/Holistic/Scripts/HolisticGraph.cs | 64 +++++++++++++++---- .../Graphs/Holistic/Scripts/HolisticValue.cs | 17 +++-- 4 files changed, 80 insertions(+), 21 deletions(-) diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Objects/HolisticDesktopGPU.prefab b/Assets/MediaPipe/Examples/Graphs/Holistic/Objects/HolisticDesktopGPU.prefab index ced841506..9d999dc1f 100644 --- a/Assets/MediaPipe/Examples/Graphs/Holistic/Objects/HolisticDesktopGPU.prefab +++ b/Assets/MediaPipe/Examples/Graphs/Holistic/Objects/HolisticDesktopGPU.prefab @@ -59,6 +59,10 @@ MonoBehaviour: m_EditorClassIdentifier: poseLandmarksPrefab: {fileID: 7166525400038315749, guid: 69cf28587972333cca1eb4c1a1ec7e6d, type: 3} + poseRoiPrefab: {fileID: 1405412484651109527, guid: c5c846cc7bbefdfa7af64f9d29912b70, + type: 3} + poseDetectionPrefab: {fileID: 1405412484651109527, guid: 3779864e7ead10e31b03fc362f4746de, + type: 3} faceLandmarksPrefab: {fileID: 2250723881323860951, guid: 570a388dca04100b8b491a9bc4ef1d51, type: 3} handLandmarksPrefab: {fileID: 3100741085041498678, guid: 4eb3a6d7cfea73afdb656b7166ae1a8d, diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticAnnotationController.cs b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticAnnotationController.cs index c8781153e..b15dd7d74 100644 --- a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticAnnotationController.cs +++ b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticAnnotationController.cs @@ -3,16 +3,22 @@ public class HolisticAnnotationController : AnnotationController { [SerializeField] GameObject poseLandmarksPrefab = null; + [SerializeField] GameObject poseRoiPrefab = null; + [SerializeField] GameObject poseDetectionPrefab = null; [SerializeField] GameObject faceLandmarksPrefab = null; [SerializeField] GameObject handLandmarksPrefab = null; private GameObject poseLandmarksAnnotation; + private GameObject poseRoiAnnotation; + private GameObject poseDetectionAnnotation; private GameObject faceLandmarksAnnotation; private GameObject leftHandLandmarksAnnotation; private GameObject rightHandLandmarksAnnotation; void Awake() { poseLandmarksAnnotation = Instantiate(poseLandmarksPrefab); + poseRoiAnnotation = Instantiate(poseRoiPrefab); + poseDetectionAnnotation = Instantiate(poseDetectionPrefab); faceLandmarksAnnotation = Instantiate(faceLandmarksPrefab); leftHandLandmarksAnnotation = Instantiate(handLandmarksPrefab); rightHandLandmarksAnnotation = Instantiate(handLandmarksPrefab); @@ -20,6 +26,8 @@ void Awake() { void OnDestroy() { Destroy(poseLandmarksAnnotation); + Destroy(poseRoiAnnotation); + Destroy(poseDetectionAnnotation); Destroy(faceLandmarksAnnotation); Destroy(leftHandLandmarksAnnotation); Destroy(rightHandLandmarksAnnotation); @@ -27,15 +35,19 @@ void OnDestroy() { public override void Clear() { poseLandmarksAnnotation.GetComponent().Clear(); + poseRoiAnnotation.GetComponent().Clear(); + poseDetectionAnnotation.GetComponent().Clear(); faceLandmarksAnnotation.GetComponent().Clear(); leftHandLandmarksAnnotation.GetComponent().Clear(); rightHandLandmarksAnnotation.GetComponent().Clear(); } - public void Draw(Transform screenTransform, NormalizedLandmarkList poseLandmarks, NormalizedLandmarkList faceLandmarks, - NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks, bool isFlipped = false) + public void Draw(Transform screenTransform, NormalizedLandmarkList poseLandmarks, NormalizedRect poseRoi, Detection poseDetection, + NormalizedLandmarkList faceLandmarks, NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks, bool isFlipped = false) { poseLandmarksAnnotation.GetComponent().Draw(screenTransform, poseLandmarks, isFlipped); + poseRoiAnnotation.GetComponent().Draw(screenTransform, poseRoi, isFlipped); + poseDetectionAnnotation.GetComponent().Draw(screenTransform, poseDetection, isFlipped); faceLandmarksAnnotation.GetComponent().Draw(screenTransform, faceLandmarks, isFlipped); leftHandLandmarksAnnotation.GetComponent().Draw(screenTransform, leftHandLandmarks, isFlipped); rightHandLandmarksAnnotation.GetComponent().Draw(screenTransform, rightHandLandmarks, isFlipped); diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticGraph.cs b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticGraph.cs index b054c946f..5dba6c6b1 100644 --- a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticGraph.cs +++ b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticGraph.cs @@ -1,11 +1,18 @@ using Mediapipe; -using System.Collections.Generic; public class HolisticGraph : DemoGraph { private const string poseLandmarksStream = "pose_landmarks"; private OutputStreamPoller poseLandmarksStreamPoller; private NormalizedLandmarkListPacket poseLandmarksPacket; + private const string poseRoiStream = "pose_roi"; + private OutputStreamPoller poseRoiStreamPoller; + private NormalizedRectPacket poseRoiPacket; + + private const string poseDetectionStream = "pose_detection"; + private OutputStreamPoller poseDetectionStreamPoller; + private DetectionPacket poseDetectionPacket; + private const string faceLandmarksStream = "face_landmarks"; private OutputStreamPoller faceLandmarksStreamPoller; private NormalizedLandmarkListPacket faceLandmarksPacket; @@ -18,14 +25,18 @@ public class HolisticGraph : DemoGraph { private OutputStreamPoller rightHandLandmarksStreamPoller; private NormalizedLandmarkListPacket rightHandLandmarksPacket; - private const string poseDetectionStream = "pose_detection"; - private OutputStreamPoller poseDetectionStreamPoller; - private DetectionPacket poseDetectionPacket; - private const string poseLandmarksPresenceStream = "pose_landmarks_presence"; private OutputStreamPoller poseLandmarksPresenceStreamPoller; private BoolPacket poseLandmarksPresencePacket; + private const string poseRoiPresenceStream = "pose_roi_presence"; + private OutputStreamPoller poseRoiPresenceStreamPoller; + private BoolPacket poseRoiPresencePacket; + + private const string poseDetectionPresenceStream = "pose_detection_presence"; + private OutputStreamPoller poseDetectionPresenceStreamPoller; + private BoolPacket poseDetectionPresencePacket; + private const string faceLandmarksPresenceStream = "face_landmarks_presence"; private OutputStreamPoller faceLandmarksPresenceStreamPoller; private BoolPacket faceLandmarksPresencePacket; @@ -44,6 +55,12 @@ public override Status StartRun() { poseLandmarksStreamPoller = graph.AddOutputStreamPoller(poseLandmarksStream).ConsumeValueOrDie(); poseLandmarksPacket = new NormalizedLandmarkListPacket(); + poseRoiStreamPoller = graph.AddOutputStreamPoller(poseRoiStream).ConsumeValueOrDie(); + poseRoiPacket = new NormalizedRectPacket(); + + poseDetectionStreamPoller = graph.AddOutputStreamPoller(poseDetectionStream).ConsumeValueOrDie(); + poseDetectionPacket = new DetectionPacket(); + faceLandmarksStreamPoller = graph.AddOutputStreamPoller(faceLandmarksStream).ConsumeValueOrDie(); faceLandmarksPacket = new NormalizedLandmarkListPacket(); @@ -53,12 +70,15 @@ public override Status StartRun() { rightHandLandmarksStreamPoller = graph.AddOutputStreamPoller(rightHandLandmarksStream).ConsumeValueOrDie(); rightHandLandmarksPacket = new NormalizedLandmarkListPacket(); - poseDetectionStreamPoller = graph.AddOutputStreamPoller(poseDetectionStream).ConsumeValueOrDie(); - poseDetectionPacket = new DetectionPacket(); - poseLandmarksPresenceStreamPoller = graph.AddOutputStreamPoller(poseLandmarksPresenceStream).ConsumeValueOrDie(); poseLandmarksPresencePacket = new BoolPacket(); + poseRoiPresenceStreamPoller = graph.AddOutputStreamPoller(poseRoiPresenceStream).ConsumeValueOrDie(); + poseRoiPresencePacket = new BoolPacket(); + + poseDetectionPresenceStreamPoller = graph.AddOutputStreamPoller(poseDetectionPresenceStream).ConsumeValueOrDie(); + poseDetectionPresencePacket = new BoolPacket(); + faceLandmarksPresenceStreamPoller = graph.AddOutputStreamPoller(faceLandmarksPresenceStream).ConsumeValueOrDie(); faceLandmarksPresencePacket = new BoolPacket(); @@ -81,24 +101,35 @@ public override void RenderOutput(WebCamScreenController screenController, Textu } private HolisticValue FetchNextHolisticValue() { - var isFaceLandmarksPresent = FetchNextFaceLandmarksPresence(); var isPoseLandmarksPresent = FetchNextPoseLandmarksPresence(); + var isPoseRoiPresent = FetchNextPoseRoiPresence(); + var isPoseDetectionPresent = FetchNextPoseDetectionPresence(); + var isFaceLandmarksPresent = FetchNextFaceLandmarksPresence(); var isLeftHandLandmarksPresent = FetchNextLeftHandLandmarksPresence(); var isRightHandLandmarksPresent = FetchNextRightHandLandmarksPresence(); var poseLandmarks = isPoseLandmarksPresent ? FetchNextPoseLandmarks() : new NormalizedLandmarkList(); + var poseRoi = isPoseRoiPresent ? FetchNextPoseRoi() : new NormalizedRect(); + var poseDetection = isPoseDetectionPresent ? FetchNextPoseDetection() : new Detection(); var faceLandmarks = isFaceLandmarksPresent ? FetchNextFaceLandmarks() : new NormalizedLandmarkList(); var leftHandLandmarks = isLeftHandLandmarksPresent ? FetchNextLeftHandLandmarks() : new NormalizedLandmarkList(); var rightHandLandmarks = isRightHandLandmarksPresent ? FetchNextRightHandLandmarks() : new NormalizedLandmarkList(); - UnityEngine.Debug.Log(poseLandmarks.Landmark.Count); - return new HolisticValue(poseLandmarks, faceLandmarks, leftHandLandmarks, rightHandLandmarks); + return new HolisticValue(poseLandmarks, poseRoi, poseDetection, faceLandmarks, leftHandLandmarks, rightHandLandmarks); } private NormalizedLandmarkList FetchNextPoseLandmarks() { return FetchNext(poseLandmarksStreamPoller, poseLandmarksPacket, poseLandmarksStream); } + private NormalizedRect FetchNextPoseRoi() { + return FetchNext(poseRoiStreamPoller, poseRoiPacket, poseRoiStream); + } + + private Detection FetchNextPoseDetection() { + return FetchNext(poseDetectionStreamPoller, poseDetectionPacket, poseDetectionStream); + } + private NormalizedLandmarkList FetchNextFaceLandmarks() { return FetchNext(faceLandmarksStreamPoller, faceLandmarksPacket, faceLandmarksStream); } @@ -115,6 +146,14 @@ private bool FetchNextPoseLandmarksPresence() { return FetchNext(poseLandmarksPresenceStreamPoller, poseLandmarksPresencePacket, poseLandmarksPresenceStream); } + private bool FetchNextPoseRoiPresence() { + return FetchNext(poseRoiPresenceStreamPoller, poseRoiPresencePacket, poseRoiPresenceStream); + } + + private bool FetchNextPoseDetectionPresence() { + return FetchNext(poseDetectionPresenceStreamPoller, poseDetectionPresencePacket, poseDetectionPresenceStream); + } + private bool FetchNextFaceLandmarksPresence() { return FetchNext(faceLandmarksPresenceStreamPoller, faceLandmarksPresencePacket, faceLandmarksPresenceStream); } @@ -130,6 +169,7 @@ private bool FetchNextRightHandLandmarksPresence() { private void RenderAnnotation(WebCamScreenController screenController, HolisticValue value) { // NOTE: input image is flipped GetComponent().Draw( - screenController.transform, value.PoseLandmarks, value.FaceLandmarks, value.LeftHandLandmarks, value.RightHandLandmarks, true); + screenController.transform, value.PoseLandmarks, value.PoseRoi, value.PoseDetection, + value.FaceLandmarks, value.LeftHandLandmarks, value.RightHandLandmarks, true); } } diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticValue.cs b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticValue.cs index 80865b86d..0de441ba5 100644 --- a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticValue.cs +++ b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticValue.cs @@ -1,17 +1,20 @@ using Mediapipe; -using System.Collections.Generic; class HolisticValue { public readonly NormalizedLandmarkList PoseLandmarks; + public readonly NormalizedRect PoseRoi; + public readonly Detection PoseDetection; public readonly NormalizedLandmarkList FaceLandmarks; public readonly NormalizedLandmarkList LeftHandLandmarks; public readonly NormalizedLandmarkList RightHandLandmarks; - public HolisticValue(NormalizedLandmarkList poseLandmarks, NormalizedLandmarkList faceLandmarks, - NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks) { - PoseLandmarks = poseLandmarks; - FaceLandmarks = faceLandmarks; - LeftHandLandmarks = leftHandLandmarks; - RightHandLandmarks = rightHandLandmarks; + public HolisticValue(NormalizedLandmarkList PoseLandmarks, NormalizedRect PoseRoi, Detection PoseDetection, + NormalizedLandmarkList FaceLandmarks, NormalizedLandmarkList LeftHandLandmarks, NormalizedLandmarkList RightHandLandmarks) { + this.PoseLandmarks = PoseLandmarks; + this.PoseRoi = PoseRoi; + this.PoseDetection = PoseDetection; + this.FaceLandmarks = FaceLandmarks; + this.LeftHandLandmarks = LeftHandLandmarks; + this.RightHandLandmarks = RightHandLandmarks; } }