Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Facemesh v2 #1437

Merged
merged 4 commits into from
May 10, 2023
Merged

Facemesh v2 #1437

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cf89b6e
feat(Facemesh): it now looks in the same direction as the face
abernier May 9, 2023
2a2497d
fix: avoiding creating disposable objects imperatively
abernier May 9, 2023
9e85de5
feat: origin landmark prop
abernier May 10, 2023
7531949
fix: initial useState value as function initializer
abernier May 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 14 additions & 15 deletions src/core/Facemesh.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import * as React from 'react'
import * as THREE from 'three'
import { useThree } from '@react-three/fiber'
import { Line } from '../../src/core'

export type MediaPipeFaceMesh = typeof FacemeshDatas.SAMPLE_FACE

Expand All @@ -25,6 +26,9 @@ export type FacemeshApi = {
outerRef: React.RefObject<THREE.Group>
}

const defaultLookAt = new THREE.Vector3(0, 0, -1)
const origin = new THREE.Vector3(0, 0, 0)

export const Facemesh = React.forwardRef<FacemeshApi, FacemeshProps>(
(
{
Expand All @@ -41,17 +45,15 @@ export const Facemesh = React.forwardRef<FacemeshApi, FacemeshProps>(
) => {
const outerRef = React.useRef<THREE.Group>(null)
const meshRef = React.useRef<THREE.Mesh>(null)
const [geometry] = React.useState(() => new THREE.BufferGeometry().setIndex(FacemeshDatas.TRIANGULATION))

const [sightDir] = React.useState(new THREE.Vector3())
const [sightDirQuaternion] = React.useState(new THREE.Quaternion())

const { invalidate } = useThree()

const [defaultLookAt] = React.useState(new THREE.Vector3(0, 0, -1))
const [origin] = React.useState(new THREE.Vector3(0, 0, 0))

const [lineGeometry] = React.useState(() => new THREE.BufferGeometry().setFromPoints([origin, defaultLookAt]))
React.useEffect(() => {
meshRef.current?.geometry.setIndex(FacemeshDatas.TRIANGULATION)
}, [])

const [a] = React.useState(new THREE.Vector3())
const [b] = React.useState(new THREE.Vector3())
Expand All @@ -60,6 +62,9 @@ export const Facemesh = React.forwardRef<FacemeshApi, FacemeshProps>(
const [ac] = React.useState(new THREE.Vector3())
const [bboxSize] = React.useState(new THREE.Vector3())
React.useEffect(() => {
const geometry = meshRef.current?.geometry
if (!geometry) return

geometry.setFromPoints(face.keypoints as THREE.Vector3[])

//
Expand Down Expand Up @@ -107,9 +112,7 @@ export const Facemesh = React.forwardRef<FacemeshApi, FacemeshProps>(
height,
depth,
verticalTri,
geometry,
debug,
lineGeometry,
invalidate,
sightDir,
sightDirQuaternion,
Expand All @@ -118,8 +121,6 @@ export const Facemesh = React.forwardRef<FacemeshApi, FacemeshProps>(
c,
ab,
ac,
origin,
defaultLookAt,
bboxSize,
])

Expand All @@ -140,16 +141,14 @@ export const Facemesh = React.forwardRef<FacemeshApi, FacemeshProps>(
<group {...props}>
<group ref={outerRef}>
<mesh ref={meshRef}>
<primitive object={geometry} attach="geometry" />
{children}

{debug ? (
<>
{geometry?.boundingBox && <box3Helper args={[geometry.boundingBox]} />}
<line>
<primitive object={lineGeometry} attach="geometry" />
<lineBasicMaterial color={0x00ffff} />
</line>
{meshRef.current?.geometry?.boundingBox && (
<box3Helper args={[meshRef.current?.geometry.boundingBox]} />
)}
<Line points={[origin, defaultLookAt]} color={0x00ffff} />
</>
) : null}
</mesh>
Expand Down