feat: V3 tensorflow plugin

cpp/JSITypedArray.cpp

@@ -204,6 +204,12 @@ TypedArray<T>::TypedArray(jsi::Runtime &runtime, std::vector<ContentType<T>> dat
   update(runtime, data);
 }
 
+template <TypedArrayKind T>
+TypedArray<T>::TypedArray(jsi::Runtime &runtime, ContentType<T>* dataToCopy, size_t size)
+    : TypedArrayBase(runtime, size, T) {
+  updateUnsafe(runtime, dataToCopy, size);
+}
+
 template <TypedArrayKind T>
 TypedArray<T>::TypedArray(TypedArrayBase &&base) : TypedArrayBase(std::move(base)) {}
 

cpp/JSITypedArray.h

@@ -143,6 +143,7 @@ class TypedArray : public TypedArrayBase {
  public:
   explicit TypedArray(TypedArrayBase &&base);
   TypedArray(jsi::Runtime &runtime, size_t size);
+  TypedArray(jsi::Runtime &runtime, ContentType<T>* dataToCopy, size_t size);
   TypedArray(jsi::Runtime &runtime, std::vector<ContentType<T>> data);
   TypedArray(TypedArray &&) = default;
   TypedArray &operator=(TypedArray &&) = default;

example/ios/Podfile

@@ -37,6 +37,7 @@ target 'VisionCameraExample' do
   )
 
   pod 'VisionCamera', :path => '../..'
+  pod 'TensorFlowLiteObjC', :subspecs => ['Metal', 'CoreML']
 
   post_install do |installer|
     # https://github.com/facebook/react-native/blob/main/packages/react-native/scripts/react_native_pods.rb#L197-L202

example/ios/Podfile.lock

@@ -488,7 +488,22 @@ PODS:
   - RNVectorIcons (9.2.0):
     - React-Core
   - SocketRocket (0.6.1)
-  - VisionCamera (3.0.0-rc.2):
+  - TensorFlowLiteC (2.12.0):
+    - TensorFlowLiteC/Core (= 2.12.0)
+  - TensorFlowLiteC/Core (2.12.0)
+  - TensorFlowLiteC/CoreML (2.12.0):
+    - TensorFlowLiteC/Core
+  - TensorFlowLiteC/Metal (2.12.0):
+    - TensorFlowLiteC/Core
+  - TensorFlowLiteObjC/Core (2.12.0):
+    - TensorFlowLiteC (= 2.12.0)
+  - TensorFlowLiteObjC/CoreML (2.12.0):
+    - TensorFlowLiteC/CoreML (= 2.12.0)
+    - TensorFlowLiteObjC/Core (= 2.12.0)
+  - TensorFlowLiteObjC/Metal (2.12.0):
+    - TensorFlowLiteC/Metal (= 2.12.0)
+    - TensorFlowLiteObjC/Core (= 2.12.0)
+  - VisionCamera (3.0.0-rc.3):
     - React
     - React-callinvoker
     - React-Core
@@ -549,6 +564,8 @@ DEPENDENCIES:
   - RNScreens (from `../node_modules/react-native-screens`)
   - RNStaticSafeAreaInsets (from `../node_modules/react-native-static-safe-area-insets`)
   - RNVectorIcons (from `../node_modules/react-native-vector-icons`)
+  - TensorFlowLiteObjC/CoreML
+  - TensorFlowLiteObjC/Metal
   - VisionCamera (from `../..`)
   - Yoga (from `../node_modules/react-native/ReactCommon/yoga`)
 
@@ -557,6 +574,8 @@ SPEC REPOS:
     - fmt
     - libevent
     - SocketRocket
+    - TensorFlowLiteC
+    - TensorFlowLiteObjC
 
 EXTERNAL SOURCES:
   boost:
@@ -719,9 +738,11 @@ SPEC CHECKSUMS:
   RNStaticSafeAreaInsets: 055ddbf5e476321720457cdaeec0ff2ba40ec1b8
   RNVectorIcons: fcc2f6cb32f5735b586e66d14103a74ce6ad61f8
   SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
-  VisionCamera: 0ee46c2c5dd7b8aa3cc3152ff4deda60ac58276f
+  TensorFlowLiteC: 20785a69299185a379ba9852b6625f00afd7984a
+  TensorFlowLiteObjC: 9a46a29a76661c513172cfffd3bf712b11ef25c3
+  VisionCamera: 1b078887679d0376be1748f30117bff2b876e073
   Yoga: 65286bb6a07edce5e4fe8c90774da977ae8fc009
 
-PODFILE CHECKSUM: ab9c06b18c63e741c04349c0fd630c6d3145081c
+PODFILE CHECKSUM: b59fbebbb0e03033da45873b46690fdc1ee3d86f
 
 COCOAPODS: 1.12.1

example/metro.config.js

@@ -16,6 +16,7 @@ module.exports = {
   // We need to make sure that only one version is loaded for peerDependencies
   // So we block them at the root, and alias them to the versions in example's node_modules
   resolver: {
+    assetExts: ['tflite', 'png', 'jpg'],
     blacklistRE: exclusionList(
       modules.map(
         (m) =>

example/src/CameraPage.tsx

@@ -1,6 +1,6 @@
 import * as React from 'react';
 import { useRef, useState, useMemo, useCallback } from 'react';
-import { Platform, StyleSheet, Text, View } from 'react-native';
+import { StyleSheet, Text, View } from 'react-native';
 import { PinchGestureHandler, PinchGestureHandlerGestureEvent, TapGestureHandler } from 'react-native-gesture-handler';
 import {
   CameraDeviceFormat,
@@ -9,6 +9,7 @@ import {
   sortFormats,
   useCameraDevices,
   useFrameProcessor,
+  useTensorflowModel,
   VideoFile,
 } from 'react-native-vision-camera';
 import { Camera, frameRateIncluded } from 'react-native-vision-camera';
@@ -21,12 +22,10 @@ import { CaptureButton } from './views/CaptureButton';
 import { PressableOpacity } from 'react-native-pressable-opacity';
 import MaterialIcon from 'react-native-vector-icons/MaterialCommunityIcons';
 import IonIcon from 'react-native-vector-icons/Ionicons';
-import { examplePlugin } from './frame-processors/ExamplePlugin';
 import type { Routes } from './Routes';
 import type { NativeStackScreenProps } from '@react-navigation/native-stack';
 import { useIsFocused } from '@react-navigation/core';
-import { Skia } from '@shopify/react-native-skia';
-import { FACE_SHADER } from './Shaders';
+import { PaintStyle, Skia } from '@shopify/react-native-skia';
 
 const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera);
 Reanimated.addWhitelistedNativeProps({
@@ -198,36 +197,50 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
     console.log('re-rendering camera page without active camera');
   }
 
-  const radius = (format?.videoHeight ?? 1080) * 0.1;
-  const width = radius;
-  const height = radius;
-  const x = (format?.videoHeight ?? 1080) / 2 - radius / 2;
-  const y = (format?.videoWidth ?? 1920) / 2 - radius / 2;
-  const centerX = x + width / 2;
-  const centerY = y + height / 2;
-
-  const runtimeEffect = Skia.RuntimeEffect.Make(FACE_SHADER);
-  if (runtimeEffect == null) throw new Error('Shader failed to compile!');
-  const shaderBuilder = Skia.RuntimeShaderBuilder(runtimeEffect);
-  shaderBuilder.setUniform('r', [width]);
-  shaderBuilder.setUniform('x', [centerX]);
-  shaderBuilder.setUniform('y', [centerY]);
-  shaderBuilder.setUniform('resolution', [1920, 1080]);
-  const imageFilter = Skia.ImageFilter.MakeRuntimeShader(shaderBuilder, null, null);
+  const plugin = useTensorflowModel(require('../assets/object_detection_mobile_object_localizer_v1_1_default_1.tflite'));
+
+  if (plugin.state === 'loaded') console.log(JSON.stringify(plugin.model, null, 2));
 
   const paint = Skia.Paint();
-  paint.setImageFilter(imageFilter);
+  paint.setStyle(PaintStyle.Stroke);
+  paint.setStrokeWidth(20);
+  paint.setColor(Skia.Color('red'));
+
+  const inputShape = plugin.model?.inputs[0]?.shape;
+  const inputFrameSize = {
+    width: inputShape?.[1] ?? 0,
+    height: inputShape?.[2] ?? 0,
+  };
 
-  const isIOS = Platform.OS === 'ios';
   const frameProcessor = useFrameProcessor(
     (frame) => {
       'worklet';
-      console.log(`Width: ${frame.width}`);
 
-      if (isIOS) frame.render(paint);
-      else console.log('Drawing to the Frame is not yet available on Android. WIP PR');
+      if (plugin.state === 'loaded') {
+        const [boundingBoxes, classes, scores, count] = plugin.model.run(frame);
+
+        for (let i = 0; i < scores.length; i++) {
+          const confidence = scores[i] ?? 0;
+          if (confidence > 0.4) {
+            const scale = (1 / frame.width) * frame.height;
+
+            const top = boundingBoxes[i] / scale + 0.22;
+            const left = boundingBoxes[i + 1];
+            const bottom = boundingBoxes[i + 2] / scale + 0.22;
+            const right = boundingBoxes[i + 3];
+
+            frame.drawRect(
+              Skia.XYWHRect(left * frame.width, top * frame.height, (right - left) * frame.width, (bottom - top) * frame.height),
+              paint,
+            );
+            console.log(left, top, right, bottom);
+          }
+        }
+      } else {
+        console.log(`Model state: ${plugin.state}..`);
+      }
     },
-    [isIOS, paint],
+    [inputFrameSize.height, paint, plugin.model, plugin.state],
   );
 
   return (

ios/CameraViewManager.swift

@@ -102,14 +102,7 @@ final class CameraViewManager: RCTViewManager {
   final func getAvailableCameraDevices(_ resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock) {
     withPromise(resolve: resolve, reject: reject) {
       let discoverySession = AVCaptureDevice.DiscoverySession(deviceTypes: getAllDeviceTypes(), mediaType: .video, position: .unspecified)
-      let devices = discoverySession.devices.filter {
-        if #available(iOS 11.1, *) {
-          // exclude the true-depth camera. The True-Depth camera has YUV and Infrared, can't take photos!
-          return $0.deviceType != .builtInTrueDepthCamera
-        }
-        return true
-      }
-      return devices.map {
+      return discoverySession.devices.map {
         return [
           "id": $0.uniqueID,
           "devices": $0.physicalDevices.map(\.deviceType.descriptor),

ios/Frame Processor/FrameProcessorRuntimeManager.mm

@@ -27,6 +27,7 @@
 
 #import "FrameProcessorUtils.h"
 #import "FrameProcessorCallback.h"
+#import "../Tensorflow Plugin/TensorflowPlugin.h"
 #import "../React Utils/JSIUtils.h"
 #import "../../cpp/JSITypedArray.h"
 
@@ -120,6 +121,8 @@ - (void) setupWorkletContext:(jsi::Runtime&)runtime {
 
   // global.FrameProcessorPlugins Proxy
   runtime.global().setProperty(runtime, "FrameProcessorPlugins", frameProcessorPlugins);
+
+  TensorflowPlugin::installToRuntime(runtime, callInvoker);
 
   NSLog(@"FrameProcessorBindings: Frame Processor plugins installed!");
 }

ios/Tensorflow Plugin/FrameResizer.h

@@ -0,0 +1,55 @@
+//
+//  FrameResizer.h
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 29.06.23.
+//  Copyright © 2023 mrousavy. All rights reserved.
+//
+
+#pragma once
+
+#import <memory>
+#import <Foundation/Foundation.h>
+#import <Accelerate/Accelerate.h>
+#import <AVFoundation/AVFoundation.h>
+#import <TensorFlowLiteObjC/TFLTensorFlowLite.h>
+
+class FrameResizer {
+public:
+  /**
+   Create a new instance of the Frame Resizer. The Frame Resizer can resize incoming Frames to the given target width, height, and byte size.
+   It uses Accelerate to downscale, crop, and re-sample Frames to the correct pixelformat.
+   The constructor will allocate a few buffers used for internal steps.
+
+   `targetWidth`: The width to resize all incoming frames to
+   `targetHeight`: The height to resize all incoming frames to
+   `channels`: The number of channels to use for the output. E.g. for RGB this should be 3, for RGBA this should be 4.
+   `dataType`: The type of the output data as a Tensorflow type.
+
+   Examples:
+   - FrameResizer(192, 192, 3, TFLTensorDataTypeUInt8) will create a buffer with the size of 110.592
+   - FrameResizer(192, 192, 3, TFLTensorDataTypeFloat32) will create a buffer with the size of 442.368
+   */
+  explicit FrameResizer(size_t targetWidth, size_t targetHeight, size_t channels, TFLTensorDataType dataType);
+  ~FrameResizer();
+
+  /**
+   Resize the given Frame to the target dimensions and pixel formats.
+   */
+  const vImage_Buffer& resizeFrame(CVPixelBufferRef pixelBuffer);
+
+private:
+  vImage_Buffer _inputDownscaledBuffer;
+  vImage_Buffer _inputReformattedBuffer;
+
+  // target image with (e.g. 192)
+  size_t _targetWidth;
+  // target image height (e.g. 192)
+  size_t _targetHeight;
+  // target image bytes per row (e.g. 192*192*3)
+  size_t _targetBytesPerRow;
+  // target image channels (e.g. 3 for RGB, 4 for RGBA)
+  size_t _targetChannels;
+  // target image data type (e.g. UInt8)
+  TFLTensorDataType _targetDataType;
+};