feat: first draft of object detection on android

.eslintrc.js

@@ -28,7 +28,7 @@ module.exports = {
   rules: {
     // eslint
     semi: "off",
-    curly: ["warn", "multi-or-nest", "consistent"],
+    curly: ["error", "all"],
     "no-mixed-spaces-and-tabs": ["warn", "smart-tabs"],
     "no-async-promise-executor": "warn",
     "require-await": "warn",
@@ -56,7 +56,7 @@ module.exports = {
       },
     ],
     "@typescript-eslint/explicit-function-return-type": [
-      "warn",
+      "off",
       {
         allowExpressions: true,
       },
@@ -78,7 +78,7 @@ module.exports = {
       "error",
       {
         allowString: false,
-        allowNullableObject: false,
+        allowNullableObject: true,
         allowNumber: false,
         allowNullableBoolean: true,
       },

android/build.gradle

@@ -44,7 +44,7 @@ def supportsNamespace() {
 
 android {
   if (supportsNamespace()) {
-    namespace "com.mediapipe"
+    namespace "com.reactnativemediapipe"
 
     sourceSets {
       main {
@@ -90,5 +90,7 @@ dependencies {
   //noinspection GradleDynamicVersion
   implementation "com.facebook.react:react-native:+"
   implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+  implementation project(':react-native-vision-camera')
+  implementation 'com.google.mediapipe:tasks-vision:0.10.2'
 }
 

android/src/main/AndroidManifest.xml

@@ -1,3 +1,3 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
-          package="com.mediapipe">
+    package="com.reactnativemediapipe">
 </manifest>

android/src/main/java/com/reactnativemediapipe/MediapipePackage.kt

@@ -0,0 +1,25 @@
+package com.reactnativemediapipe
+
+import com.facebook.react.ReactPackage
+import com.facebook.react.bridge.NativeModule
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.uimanager.ViewManager
+import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry
+import com.reactnativemediapipe.objectdetection.ObjectDetectionFrameProcessorPlugin
+import com.reactnativemediapipe.objectdetection.ObjectDetectionModule
+
+class MediapipePackage : ReactPackage {
+  companion object {
+    init {
+      FrameProcessorPluginRegistry.addFrameProcessorPlugin("objectDetection") { _, _ ->
+        ObjectDetectionFrameProcessorPlugin()
+      }
+    }
+  } override fun createNativeModules(reactContext: ReactApplicationContext): List<NativeModule> {
+    return listOf(ObjectDetectionModule(reactContext))
+  }
+
+  override fun createViewManagers(reactContext: ReactApplicationContext): List<ViewManager<*, *>> {
+    return listOf(MediapipeViewManager())
+  }
+}

android/src/main/java/com/mediapipe/MediapipeViewManager.kt → android/src/main/java/com/reactnativemediapipe/MediapipeViewManager.kt

@@ -1,4 +1,4 @@
-package com.mediapipe
+package com.reactnativemediapipe
 
 import android.graphics.Color
 import android.view.View

android/src/main/java/com/reactnativemediapipe/objectdetection/ConvertHelpers.kt

@@ -0,0 +1,77 @@
+package com.reactnativemediapipe.objectdetection
+
+import android.graphics.RectF
+import com.facebook.react.bridge.Arguments
+import com.facebook.react.bridge.WritableArray
+import com.facebook.react.bridge.WritableMap
+import com.facebook.react.bridge.WritableNativeMap
+import com.google.mediapipe.tasks.components.containers.Category
+import com.google.mediapipe.tasks.components.containers.Detection
+import java.util.Optional
+
+// Assuming simplified representations based on your descriptions
+fun convertCategoryToWritableMap(category: Category): WritableMap {
+  val map = Arguments.createMap()
+  map.putDouble("score", category.score().toDouble())
+  map.putInt("index", category.index())
+  map.putString("categoryName", category.categoryName())
+  map.putString("displayName", category.displayName())
+  return map
+}
+
+fun convertDetectionToWritableMap(detection: Detection): WritableMap {
+  val map = Arguments.createMap()
+  val categoriesArray = Arguments.createArray()
+  detection.categories().forEach { category ->
+    categoriesArray.pushMap(convertCategoryToWritableMap(category))
+  }
+
+  val keypointsArray = Arguments.createArray()
+  detection.keypoints().ifPresent { keypoints ->
+    keypoints.forEach { keypoint ->
+      val keypointMap = Arguments.createMap()
+      keypointMap.putDouble("x", keypoint.x().toDouble())
+      keypointMap.putDouble("y", keypoint.y().toDouble())
+      keypoint.label().ifPresent { keypointMap.putString("label", it) }
+      keypoint.score().ifPresent{ keypointMap.putDouble("score",it.toDouble()) }
+      keypointsArray.pushMap(keypointMap)
+    }
+  }
+
+  map.putArray("categories", categoriesArray)
+  map.putMap("boundingBox", convertRectFToWritableMap(detection.boundingBox()))
+  map.putArray("keypoints", keypointsArray)
+  return map
+}
+
+fun convertRectFToWritableMap(rectF: RectF): WritableMap {
+  val map = Arguments.createMap()
+  map.putDouble("left", rectF.left.toDouble())
+  map.putDouble("top", rectF.top.toDouble())
+  map.putDouble("right", rectF.right.toDouble())
+  map.putDouble("bottom", rectF.bottom.toDouble())
+  return map
+}
+
+fun convertResultBundleToWritableMap(resultBundle: ObjectDetectorHelper.ResultBundle): WritableMap {
+  val map = Arguments.createMap()
+  val resultsArray = Arguments.createArray()
+
+  resultBundle.results.forEach { result ->
+    val resultMap = Arguments.createMap()
+    resultMap.putDouble("timestampMs", result.timestampMs().toDouble())
+    val detectionsArray = Arguments.createArray()
+    result.detections().forEach { detection ->
+      detectionsArray.pushMap(convertDetectionToWritableMap(detection))
+    }
+    resultMap.putArray("detections", detectionsArray)
+    resultsArray.pushMap(resultMap)
+  }
+
+  map.putArray("results", resultsArray)
+  map.putInt("inputImageHeight", resultBundle.inputImageHeight)
+  map.putInt("inputImageWidth", resultBundle.inputImageWidth)
+  map.putInt("inputImageRotation", resultBundle.inputImageRotation)
+  map.putDouble("inferenceTime", resultBundle.inferenceTime.toDouble())
+  return map
+}

android/src/main/java/com/reactnativemediapipe/objectdetection/ObjectDetectionFrameProcessorPlugin.kt

@@ -0,0 +1,18 @@
+package com.reactnativemediapipe.objectdetection
+
+import com.mrousavy.camera.frameprocessor.Frame
+import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
+import com.mrousavy.camera.frameprocessor.VisionCameraProxy
+
+class ObjectDetectionFrameProcessorPlugin() :
+  FrameProcessorPlugin() {
+
+  override fun callback(frame: Frame, params: MutableMap<String, Any>?): Any? {
+    val detectorHandle:Double = params!!["detectorHandle"] as Double
+    val detector = ObjectDetectorMap.detectorMap[detectorHandle.toInt()] ?: return false
+
+    val image = frame.image
+    detector.detectLivestreamFrame(image)
+    return true
+  }
+}

android/src/main/java/com/reactnativemediapipe/objectdetection/ObjectDetectionModule.kt

@@ -0,0 +1,94 @@
+package com.reactnativemediapipe.objectdetection
+
+import com.facebook.react.bridge.Arguments
+import com.facebook.react.bridge.Promise
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.bridge.ReactContextBaseJavaModule
+import com.facebook.react.bridge.ReactMethod
+import com.facebook.react.modules.core.DeviceEventManagerModule
+import com.google.mediapipe.tasks.vision.core.RunningMode
+
+object ObjectDetectorMap {
+  internal val detectorMap = mutableMapOf<Int, ObjectDetectorHelper>()
+
+}
+class ObjectDetectionModule(reactContext: ReactApplicationContext) :
+  ReactContextBaseJavaModule(reactContext) {
+
+  private var nextId = 22 // just not zero
+
+  override fun getName(): String {
+    return "ObjectDetection"
+  }
+
+  private class ObjectDetectorListener(
+    private val module: ObjectDetectionModule,
+    private val handle: Int
+  ) :
+    ObjectDetectorHelper.DetectorListener {
+    override fun onError(error: String, errorCode: Int) {
+      module.sendErrorEvent(handle, error, errorCode)
+    }
+
+    override fun onResults(resultBundle: ObjectDetectorHelper.ResultBundle) {
+      module.sendResultsEvent(handle, resultBundle)
+    }
+  }
+
+  @ReactMethod
+  fun createDetector(
+    threshold: Float,
+    maxResults: Int,
+    delegate: Int,
+    model: String,
+    runningMode: Int,
+    promise: Promise
+  ) {
+    val id = nextId++
+    val helper = ObjectDetectorHelper(
+      threshold = threshold,
+      maxResults = maxResults,
+      currentDelegate = delegate,
+      currentModel = model,
+      runningMode = enumValues<RunningMode>().first { it.ordinal == runningMode },
+      context = reactApplicationContext.applicationContext,
+      objectDetectorListener = ObjectDetectorListener(this,id)
+    )
+    ObjectDetectorMap.detectorMap[id] = helper
+    promise.resolve(id)
+  }
+
+  @ReactMethod
+  fun releaseDetector(handle: Int,promise: Promise) {
+    val entry = ObjectDetectorMap.detectorMap[handle]
+    if (entry != null) {
+      entry.clearObjectDetector()
+      ObjectDetectorMap.detectorMap.remove(handle)
+    }
+    promise.resolve(true)
+  }
+
+  @ReactMethod
+  fun addListener(eventName: String?) {
+    /* Required for RN built-in Event Emitter Calls. */
+  }
+
+  @ReactMethod
+  fun removeListeners(count: Int?) {
+    /* Required for RN built-in Event Emitter Calls. */
+  }
+  private fun sendErrorEvent(handle: Int, message: String, code: Int) {
+    val errorArgs =
+      Arguments.makeNativeMap(mapOf("handle" to handle, "message" to message, "code" to code))
+
+    reactApplicationContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java)
+      .emit("onError", errorArgs)
+  }
+
+  private fun sendResultsEvent(handle: Int, bundle: ObjectDetectorHelper.ResultBundle) {
+    val resultArgs = convertResultBundleToWritableMap(bundle)
+    resultArgs.putInt("handle", handle)
+    reactApplicationContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java)
+      .emit("onResults", resultArgs)
+  }
+}