feat: change code to avoid rerenders, add framerate option (#139)

cdiddy77 · Jul 21, 2024 · 0fd2e27 · 0fd2e27
1 parent 3799617
commit 0fd2e27
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 47 deletions.
diff --git a/examples/posedetection/babel.config.js b/examples/posedetection/babel.config.js
@@ -4,7 +4,6 @@ const pak = require("../../package.json");
 module.exports = {
   presets: ["module:@react-native/babel-preset"],
   plugins: [
-    ["react-native-worklets-core/plugin"],
     [
       "module-resolver",
       {
@@ -14,5 +13,7 @@ module.exports = {
         },
       },
     ],
+    ["react-native-reanimated/plugin"],
+    ["react-native-worklets-core/plugin"],
   ],
 };
diff --git a/examples/posedetection/src/CameraStream.tsx b/examples/posedetection/src/CameraStream.tsx
@@ -4,7 +4,6 @@ import { Pressable, StyleSheet, Text, View } from "react-native";
 import {
   MediapipeCamera,
   RunningMode,
-  type Point,
   usePoseDetection,
   KnownPoseLandmarkConnections,
   type DetectionError,
@@ -20,6 +19,8 @@ import type { RootTabParamList } from "./navigation";
 import type { BottomTabScreenProps } from "@react-navigation/bottom-tabs";
 import { useSettings } from "./app-settings";
 import { PoseDrawFrame } from "./Drawing";
+import { useSharedValue } from "react-native-reanimated";
+import { vec, type SkPoint } from "@shopify/react-native-skia";
 
 type Props = BottomTabScreenProps<RootTabParamList, "CameraStream">;
 
@@ -47,16 +48,7 @@ export const CameraStream: React.FC<Props> = () => {
     );
   };
 
-  const [landmarks, setLandmarks] = React.useState<Point[]>([]);
-
-  const connections = React.useMemo((): [Point, Point][] => {
-    return landmarks.length > 0
-      ? KnownPoseLandmarkConnections.map((connection) => [
-          landmarks[connection[0]],
-          landmarks[connection[1]],
-        ])
-      : [];
-  }, [landmarks]);
+  const connections = useSharedValue<SkPoint[]>([]);
 
   const onResults = React.useCallback(
     (results: PoseDetectionResultBundle, vc: ViewCoordinator): void => {
@@ -80,9 +72,17 @@ export const CameraStream: React.FC<Props> = () => {
       // );
       const frameDims = vc.getFrameDims(results);
       const pts = results.results[0].landmarks[0] ?? [];
-      setLandmarks(pts.map((landmark) => vc.convertPoint(frameDims, landmark)));
+      const newLines: SkPoint[] = [];
+      for (const connection of KnownPoseLandmarkConnections) {
+        const [a, b] = connection;
+        const pt1 = vc.convertPoint(frameDims, pts[a]);
+        const pt2 = vc.convertPoint(frameDims, pts[b]);
+        newLines.push(vec(pt1.x, pt1.y));
+        newLines.push(vec(pt2.x, pt2.y));
+      }
+      connections.value = newLines;
     },
-    []
+    [connections]
   );
   const onError = React.useCallback((error: DetectionError): void => {
     console.log(`error: ${error}`);
@@ -93,7 +93,8 @@ export const CameraStream: React.FC<Props> = () => {
       onError: onError,
     },
     RunningMode.LIVE_STREAM,
-    `${settings.model}.task`
+    `${settings.model}.task`,
+    { fpsMode: "none" } // supply a number instead to get a specific framerate
   );
 
   if (permsGranted.cam) {
@@ -105,11 +106,7 @@ export const CameraStream: React.FC<Props> = () => {
           activeCamera={active}
           resizeMode="cover"
         />
-        <PoseDrawFrame
-          points={landmarks}
-          lines={connections}
-          style={styles.box}
-        />
+        <PoseDrawFrame connections={connections} style={styles.box} />
         <Pressable style={styles.cameraSwitchButton} onPress={setActiveCamera}>
           <Text style={styles.cameraSwitchButtonText}>Switch Camera</Text>
         </Pressable>

diff --git a/examples/posedetection/src/Drawing.tsx b/examples/posedetection/src/Drawing.tsx
@@ -1,37 +1,30 @@
 import React from "react";
-import { Canvas, Circle, Line, vec } from "@shopify/react-native-skia";
+import { Canvas, Points, type SkPoint } from "@shopify/react-native-skia";
 import { type StyleProp, type ViewStyle } from "react-native";
-import { type Point } from "react-native-mediapipe";
+import type { SharedValue } from "react-native-reanimated";
 
 export interface PoseDrawFrameProps {
-  points: Point[];
-  lines: [Point, Point][];
+  connections: SharedValue<SkPoint[]>;
   style?: StyleProp<ViewStyle>;
 }
 export const PoseDrawFrame: React.FC<PoseDrawFrameProps> = (props) => {
   return (
     <Canvas style={props.style}>
-      {props.lines.map((segment, index) => (
-        <Line
-          key={index}
-          p1={vec(segment[0].x, segment[0].y)}
-          p2={vec(segment[1].x, segment[1].y)}
-          color="ForestGreen"
-          style="stroke"
-          strokeWidth={4}
-        />
-      ))}
-      {props.points.map((p, index) => (
-        <Circle
-          key={index}
-          cx={p.x}
-          cy={p.y}
-          r={5}
-          color="red"
-          strokeWidth={3}
-          style="stroke"
-        />
-      ))}
+      <Points
+        points={props.connections}
+        mode="lines"
+        color={"lightblue"}
+        style={"stroke"}
+        strokeWidth={3}
+      />
+      <Points
+        points={props.connections}
+        mode="points"
+        color={"red"}
+        style={"stroke"}
+        strokeWidth={10}
+        strokeCap={"round"}
+      />
     </Canvas>
   );
 };

diff --git a/src/poseDetection/index.ts b/src/poseDetection/index.ts
@@ -7,6 +7,7 @@ import {
 } from "react-native";
 import {
   VisionCameraProxy,
+  runAtTargetFps,
   useFrameProcessor,
   type CameraDevice,
   type Orientation,
@@ -79,6 +80,8 @@ export interface PoseLandmarkerResult {
 export type PoseDetectionResultBundle =
   DetectionResultBundle<PoseLandmarkerResult>;
 
+type FpsMode = "none" | number;
+
 export interface PoseDetectionOptions {
   numPoses: number;
   minPoseDetectionConfidence: number;
@@ -87,6 +90,7 @@ export interface PoseDetectionOptions {
   shouldOutputSegmentationMasks: boolean;
   delegate: Delegate;
   mirrorMode: "no-mirror" | "mirror" | "mirror-front-only";
+  fpsMode: FpsMode;
 }
 
 type PoseDetectionCallbackState =
@@ -240,12 +244,20 @@ export function usePoseDetection(
     options?.minTrackingConfidence,
     options?.shouldOutputSegmentationMasks,
   ]);
+
   const frameProcessor = useFrameProcessor(
     (frame) => {
       "worklet";
-      plugin?.call(frame, { detectorHandle });
+      const asyncMode = options?.fpsMode ?? "none";
+      if (asyncMode === "none") {
+        plugin?.call(frame, { detectorHandle });
+      } else {
+        runAtTargetFps(asyncMode, () => {
+          plugin?.call(frame, { detectorHandle });
+        });
+      }
     },
-    [detectorHandle]
+    [detectorHandle, options?.fpsMode]
   );
   return React.useMemo(
     (): MediaPipeSolution => ({