Added Participant View, Added Daily custom track support, Added extra…

android/build.gradle

@@ -96,7 +96,8 @@ def kotlin_version = getExtOrDefault("kotlinVersion")
 dependencies {
   // For < 0.71, this will be from the local maven repo
   // For > 0.71, this will be replaced by `com.facebook.react:react-android:$version` by react gradle plugin
-
+  implementation 'co.daily:client:0.29.0'
+
   implementation 'com.google.code.gson:gson:2.11.0'
 
   //noinspection GradleDynamicVersion

android/src/main/java/com/tsmediapipe/PoseLandmarkerHelper.kt

@@ -1,19 +1,26 @@
 package com.tsmediapipe
 
 import android.content.Context
+import android.content.Intent
 import android.graphics.Bitmap
 import android.graphics.Matrix
+import android.os.Handler
+import android.os.HandlerThread
 import android.os.SystemClock
 import android.util.Log
 import androidx.annotation.VisibleForTesting
 import androidx.camera.core.ImageProxy
+import androidx.localbroadcastmanager.content.LocalBroadcastManager
 import com.google.mediapipe.framework.image.BitmapImageBuilder
 import com.google.mediapipe.framework.image.MPImage
 import com.google.mediapipe.tasks.core.BaseOptions
 import com.google.mediapipe.tasks.core.Delegate
 import com.google.mediapipe.tasks.vision.core.RunningMode
 import com.google.mediapipe.tasks.vision.poselandmarker.PoseLandmarker
 import com.google.mediapipe.tasks.vision.poselandmarker.PoseLandmarkerResult
+import java.io.ByteArrayOutputStream
+import co.daily.model.customtrack.CustomVideoSourceSurface
+import java.lang.reflect.Field
 
 class PoseLandmarkerHelper(
   var minPoseDetectionConfidence: Float = DEFAULT_POSE_DETECTION_CONFIDENCE,
@@ -31,13 +38,18 @@ class PoseLandmarkerHelper(
   // If the Pose Landmarker will not change, a lazy val would be preferable.
   private var poseLandmarker: PoseLandmarker? = null
 
+  private var referenceCount = 0
+  private var timestamp: Long = 0
+
   init {
     setupPoseLandmarker()
   }
 
   fun clearPoseLandmarker() {
     poseLandmarker?.close()
     poseLandmarker = null
+    referenceCount = 0
+    timestamp = 0
   }
 
   // Return running status of PoseLandmarkerHelper
@@ -133,6 +145,9 @@ class PoseLandmarkerHelper(
     imageProxy: ImageProxy,
     isFrontCamera: Boolean
   ) {
+    // Get timestamp Value
+    timestamp = System.currentTimeMillis() - SystemClock.elapsedRealtime() + (imageProxy.imageInfo.timestamp / 1_000_000)
+
     if (runningMode != RunningMode.LIVE_STREAM) {
       throw IllegalArgumentException(
         "Attempting to call detectLiveStream" +
@@ -171,12 +186,45 @@ class PoseLandmarkerHelper(
       matrix, true
     )
 
+    updateCustomVideoTrack(rotatedBitmap)
+
     // Convert the input Bitmap object to an MPImage object to run inference
     val mpImage = BitmapImageBuilder(rotatedBitmap).build()
 
     detectAsync(mpImage, frameTime)
   }
 
+  private val handlerThread = HandlerThread("CustomVideoTrackThread").apply { start() }
+  private val handler = Handler(handlerThread.looper)
+  private var customVideoSource: CustomVideoSourceSurface? = null
+
+  fun setCustomVideoSource(source: CustomVideoSourceSurface) {
+    this.customVideoSource = source
+  }
+
+  private fun updateCustomVideoTrack(bitmap: Bitmap) {
+    handler.removeCallbacksAndMessages(null)
+
+    handler.post {
+      try {
+        customVideoSource?.let { source ->
+          // Use source to draw on the canvas.
+          val surface = source.surfaceOrNull
+          if (surface == null || !surface.isValid) {
+            Log.e("Surface", "Surface is null or invalid")
+            return@post
+          }
+
+          val canvas = surface.lockHardwareCanvas()
+          canvas.drawBitmap(bitmap, 0f, 0f, null)
+          surface.unlockCanvasAndPost(canvas)
+        }
+      } catch (e: Exception) {
+        Log.e("Surface", "Error drawing bitmap: ${e.message}")
+      }
+    }
+  }
+
   // Run pose landmark using MediaPipe Pose Landmarker API
   @VisibleForTesting
   fun detectAsync(mpImage: MPImage, frameTime: Long) {
@@ -192,13 +240,18 @@ class PoseLandmarkerHelper(
   ) {
     val finishTimeMs = SystemClock.uptimeMillis()
     val inferenceTime = finishTimeMs - result.timestampMs()
+    val startTimestamp = System.currentTimeMillis()
+    referenceCount++
 
     poseLandmarkerHelperListener?.onResults(
       ResultBundle(
         listOf(result),
         inferenceTime,
         input.height,
-        input.width
+        input.width,
+        timestamp,
+        referenceCount,
+        startTimestamp
       )
     )
   }
@@ -232,6 +285,9 @@ class PoseLandmarkerHelper(
     val inferenceTime: Long,
     val inputImageHeight: Int,
     val inputImageWidth: Int,
+    val presentationTimeStamp: Long,
+    val frameNumber: Int,
+    val startTimestamp: Long
   )
 
   interface LandmarkerListener {

android/src/main/java/com/tsmediapipe/fragment/CameraFragment.kt

@@ -22,6 +22,7 @@ import androidx.camera.lifecycle.ProcessCameraProvider
 import androidx.core.content.ContextCompat
 import androidx.fragment.app.Fragment
 import androidx.fragment.app.activityViewModels
+import co.daily.model.customtrack.CustomVideoSourceSurface
 import com.facebook.react.modules.core.DeviceEventManagerModule
 import com.google.gson.Gson
 import com.google.mediapipe.tasks.vision.core.RunningMode
@@ -53,10 +54,15 @@ class CameraFragment : Fragment(), PoseLandmarkerHelper.LandmarkerListener {
   private var camera: Camera? = null
   private var cameraProvider: ProcessCameraProvider? = null
   private var cameraFacing = CameraSelector.LENS_FACING_FRONT
+  // private var cameraFacing = CameraSelector.LENS_FACING_BACK
 
   /** Blocking ML operations are performed using this executor */
   private lateinit var backgroundExecutor: ExecutorService
 
+  fun setCustomVideoSource(source: CustomVideoSourceSurface) {
+    poseLandmarkerHelper.setCustomVideoSource(source)
+  }
+
   fun hasPermissions(context: Context) = PERMISSIONS_REQUIRED.all {
     ContextCompat.checkSelfPermission(
       context,
@@ -298,9 +304,9 @@ class CameraFragment : Fragment(), PoseLandmarkerHelper.LandmarkerListener {
         val additionalData = mapOf(
           "height" to resultBundle.inputImageHeight,
           "width" to resultBundle.inputImageWidth,
-//          "presentationTimeStamp" to resultBundle.presentationTimeStamp,
-//          "frameNumber" to resultBundle.frameNumber,
-//          "startTimestamp" to resultBundle.startTimestamp
+          "presentationTimeStamp" to resultBundle.presentationTimeStamp,
+          "frameNumber" to resultBundle.frameNumber,
+          "startTimestamp" to resultBundle.startTimestamp
         )
 
         val swiftDict: MutableMap<String, Any> = mutableMapOf(

android/src/main/java/com/tsmediapipe/participantView/ParticipantOverlayManager.kt

@@ -0,0 +1,69 @@
+package com.tsmediapipe.participantView
+
+import android.annotation.SuppressLint
+import android.app.Activity
+import android.view.LayoutInflater
+import android.view.View
+import android.widget.FrameLayout
+import com.tsmediapipe.R
+import co.daily.model.Participant
+import co.daily.model.ParticipantId
+import co.daily.view.VideoView
+
+object ParticipantOverlayManager {
+
+    private val videoViews = mutableMapOf<ParticipantId, VideoView>()
+
+    @SuppressLint("MissingInflatedId")
+    fun showOverlay(activity: Activity, participant: Participant) {
+        activity.runOnUiThread {
+            val participantContainer = activity.findViewById<FrameLayout>(R.id.participant_overlay_container)
+            if (participantContainer != null) {
+                participantContainer.visibility = View.VISIBLE
+
+                // Inflate the participant video layout
+                val layoutInflater = LayoutInflater.from(activity)
+                val participantView = layoutInflater.inflate(R.layout.participant_view, participantContainer, false)
+
+                // Get the VideoView from the inflated layout
+                val videoView = participantView.findViewById<VideoView>(R.id.participant_video)
+
+                // Assign the track to the video view
+                videoView.track = participant.media?.camera?.track
+
+                // Store the video view for future reference
+                videoViews[participant.id] = videoView
+
+                // Add the participant view inside the overlay container
+                participantContainer.addView(participantView)
+            }
+        }
+    }
+
+    fun hideOverlay(activity: Activity, participant: Participant) {
+        activity.runOnUiThread {
+            val participantContainer = activity.findViewById<FrameLayout>(R.id.participant_overlay_container)
+            val videoView = videoViews.remove(participant.id)
+
+            if (participantContainer != null && videoView != null) {
+                participantContainer.removeView(videoView.parent as View)
+
+                // Hide container if empty
+                if (participantContainer.childCount == 0) {
+                    participantContainer.visibility = View.GONE
+                }
+            }
+        }
+    }
+
+    fun participantUpdated(activity: Activity, participant: Participant) {
+        activity.runOnUiThread {
+            val videoView = videoViews[participant.id]
+            if (videoView != null) {
+                videoView.track = participant.media?.camera?.track
+            } else {
+                videoView?.track = null
+            }
+        }
+    }
+}

android/src/main/res/layout/fragment_my_camera.xml

@@ -32,5 +32,12 @@
     android:layout_width="match_parent"
     android:layout_height="match_parent" />
 
+  <!-- New FrameLayout for Participant Video -->
+  <FrameLayout
+      android:id="@+id/participant_overlay_container"
+      android:layout_width="200dp"
+      android:layout_height="200dp"
+      android:visibility="gone"/>
+
 
 </androidx.coordinatorlayout.widget.CoordinatorLayout>

android/src/main/res/layout/participant_view.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:orientation="vertical">
+
+    <co.daily.view.VideoView
+        android:id="@+id/participant_video"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:layout_gravity="center"/>
+</LinearLayout>