diff --git a/examples/posedetection/babel.config.js b/examples/posedetection/babel.config.js index 804f8df..1e618a7 100644 --- a/examples/posedetection/babel.config.js +++ b/examples/posedetection/babel.config.js @@ -4,7 +4,6 @@ const pak = require("../../package.json"); module.exports = { presets: ["module:@react-native/babel-preset"], plugins: [ - ["react-native-worklets-core/plugin"], [ "module-resolver", { @@ -14,5 +13,7 @@ module.exports = { }, }, ], + ["react-native-reanimated/plugin"], + ["react-native-worklets-core/plugin"], ], }; diff --git a/examples/posedetection/src/CameraStream.tsx b/examples/posedetection/src/CameraStream.tsx index 3b27166..c6ad3cb 100644 --- a/examples/posedetection/src/CameraStream.tsx +++ b/examples/posedetection/src/CameraStream.tsx @@ -4,7 +4,6 @@ import { Pressable, StyleSheet, Text, View } from "react-native"; import { MediapipeCamera, RunningMode, - type Point, usePoseDetection, KnownPoseLandmarkConnections, type DetectionError, @@ -20,6 +19,8 @@ import type { RootTabParamList } from "./navigation"; import type { BottomTabScreenProps } from "@react-navigation/bottom-tabs"; import { useSettings } from "./app-settings"; import { PoseDrawFrame } from "./Drawing"; +import { useSharedValue } from "react-native-reanimated"; +import { vec, type SkPoint } from "@shopify/react-native-skia"; type Props = BottomTabScreenProps; @@ -47,16 +48,7 @@ export const CameraStream: React.FC = () => { ); }; - const [landmarks, setLandmarks] = React.useState([]); - - const connections = React.useMemo((): [Point, Point][] => { - return landmarks.length > 0 - ? KnownPoseLandmarkConnections.map((connection) => [ - landmarks[connection[0]], - landmarks[connection[1]], - ]) - : []; - }, [landmarks]); + const connections = useSharedValue([]); const onResults = React.useCallback( (results: PoseDetectionResultBundle, vc: ViewCoordinator): void => { @@ -80,9 +72,17 @@ export const CameraStream: React.FC = () => { // ); const frameDims = vc.getFrameDims(results); const pts = results.results[0].landmarks[0] ?? []; - setLandmarks(pts.map((landmark) => vc.convertPoint(frameDims, landmark))); + const newLines: SkPoint[] = []; + for (const connection of KnownPoseLandmarkConnections) { + const [a, b] = connection; + const pt1 = vc.convertPoint(frameDims, pts[a]); + const pt2 = vc.convertPoint(frameDims, pts[b]); + newLines.push(vec(pt1.x, pt1.y)); + newLines.push(vec(pt2.x, pt2.y)); + } + connections.value = newLines; }, - [] + [connections] ); const onError = React.useCallback((error: DetectionError): void => { console.log(`error: ${error}`); @@ -93,7 +93,8 @@ export const CameraStream: React.FC = () => { onError: onError, }, RunningMode.LIVE_STREAM, - `${settings.model}.task` + `${settings.model}.task`, + { fpsMode: "none" } // supply a number instead to get a specific framerate ); if (permsGranted.cam) { @@ -105,11 +106,7 @@ export const CameraStream: React.FC = () => { activeCamera={active} resizeMode="cover" /> - + Switch Camera diff --git a/examples/posedetection/src/Drawing.tsx b/examples/posedetection/src/Drawing.tsx index 7889790..39efe38 100644 --- a/examples/posedetection/src/Drawing.tsx +++ b/examples/posedetection/src/Drawing.tsx @@ -1,37 +1,30 @@ import React from "react"; -import { Canvas, Circle, Line, vec } from "@shopify/react-native-skia"; +import { Canvas, Points, type SkPoint } from "@shopify/react-native-skia"; import { type StyleProp, type ViewStyle } from "react-native"; -import { type Point } from "react-native-mediapipe"; +import type { SharedValue } from "react-native-reanimated"; export interface PoseDrawFrameProps { - points: Point[]; - lines: [Point, Point][]; + connections: SharedValue; style?: StyleProp; } export const PoseDrawFrame: React.FC = (props) => { return ( - {props.lines.map((segment, index) => ( - - ))} - {props.points.map((p, index) => ( - - ))} + + ); }; diff --git a/src/poseDetection/index.ts b/src/poseDetection/index.ts index 408ffe6..f4ad793 100644 --- a/src/poseDetection/index.ts +++ b/src/poseDetection/index.ts @@ -7,6 +7,7 @@ import { } from "react-native"; import { VisionCameraProxy, + runAtTargetFps, useFrameProcessor, type CameraDevice, type Orientation, @@ -79,6 +80,8 @@ export interface PoseLandmarkerResult { export type PoseDetectionResultBundle = DetectionResultBundle; +type FpsMode = "none" | number; + export interface PoseDetectionOptions { numPoses: number; minPoseDetectionConfidence: number; @@ -87,6 +90,7 @@ export interface PoseDetectionOptions { shouldOutputSegmentationMasks: boolean; delegate: Delegate; mirrorMode: "no-mirror" | "mirror" | "mirror-front-only"; + fpsMode: FpsMode; } type PoseDetectionCallbackState = @@ -240,12 +244,20 @@ export function usePoseDetection( options?.minTrackingConfidence, options?.shouldOutputSegmentationMasks, ]); + const frameProcessor = useFrameProcessor( (frame) => { "worklet"; - plugin?.call(frame, { detectorHandle }); + const asyncMode = options?.fpsMode ?? "none"; + if (asyncMode === "none") { + plugin?.call(frame, { detectorHandle }); + } else { + runAtTargetFps(asyncMode, () => { + plugin?.call(frame, { detectorHandle }); + }); + } }, - [detectorHandle] + [detectorHandle, options?.fpsMode] ); return React.useMemo( (): MediaPipeSolution => ({