Skip to content

Commit 7fb6f3f

Browse files
Danial-KordDanial-Kord
committed
[feat]: Facial landmarks to blendshape values system completed
1 parent 7abe71a commit 7fb6f3f

File tree

4 files changed

+356
-118
lines changed

4 files changed

+356
-118
lines changed

‎Backend/blendshapes/blendshape_calculator.py

+43-21
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,6 @@ def _calculate_mouth_landmarks(self):
119119
mouth_open = self._remap_blendshape(FaceBlendShape.MouthOpen, mouth_open_dist/mouth_width) #mouth aspect ratio
120120
self._face_data.set_blendshape(FaceBlendShape.MouthOpen, mouth_open)
121121

122-
123122
#Simle
124123
mouth_smile_left,mouth_smile_right = self.detect_smile(upper_lip,mouth_corner_left,mouth_corner_right)
125124

@@ -133,24 +132,17 @@ def _calculate_mouth_landmarks(self):
133132

134133

135134
uppest_lip = self._get_landmark(0)
136-
137-
#Jaw left right
138-
self.detect_Jaw_direction(nose_tip,lowest_chin)
139-
140-
#
141135
lowest_lip = self._get_landmark(self.blend_shape_config.CanonicalPoints.lowest_lip)
142136
under_lip = self._get_landmark(self.blend_shape_config.CanonicalPoints.under_lip)
143137

144-
outer_lip_dist = self.dist(lower_lip, lowest_lip)
145-
upper_lip_dist = self.dist(upper_lip, upper_outer_lip)
146-
self._face_data.set_blendshape(
147-
FaceBlendShape.MouthRollLower, 1 - self._remap_blendshape(FaceBlendShape.MouthRollLower, outer_lip_dist))
148-
self._face_data.set_blendshape(
149-
FaceBlendShape.MouthRollUpper, 1 - self._remap_blendshape(FaceBlendShape.MouthRollUpper, upper_lip_dist))
138+
#Jaw left right
139+
self.detect_Jaw_direction(nose_tip,lowest_chin)
150140

141+
#mouth roll
142+
self.detect_Mouth_Roll(lower_lip,upper_lip,upper_outer_lip,lowest_lip)
151143

152144
#mouth pucker
153-
self.detect_mouth_pucker(mouth_width)
145+
self.detect_mouth_pucker(mouth_width,mouth_open_dist)
154146

155147
#Mouth shrug
156148
self.detect_mouth_shrug(nose_tip,uppest_lip,lowest_lip)
@@ -180,6 +172,7 @@ def detect_smile(self,upper_lip,mouth_corner_left,mouth_corner_right):
180172
mouth_smile_right = 1 - \
181173
self._remap_blendshape(FaceBlendShape.MouthSmileRight, smile_right)
182174

175+
print(mouth_smile_right)
183176
self._face_data.set_blendshape(
184177
FaceBlendShape.MouthSmileLeft, mouth_smile_left)
185178
self._face_data.set_blendshape(
@@ -233,6 +226,8 @@ def detect_mouth_Stretch(self,mouth_center,mouth_corner_left,mouth_corner_right,
233226
FaceBlendShape.MouthLeft, mouth_left)
234227
self._face_data.set_blendshape(
235228
FaceBlendShape.MouthRight, mouth_right)
229+
230+
236231
# self._live_link_face.set_blendshape(ARKitFace.MouthRight, 1 - remap(mouth_left_right, -1.5, 0.0))
237232
#-------------------------------------------------------
238233

@@ -254,6 +249,19 @@ def detect_mouth_Stretch(self,mouth_center,mouth_corner_left,mouth_corner_right,
254249
mouth_right_stretch_final = self._remap(mouth_right_stretch, stretch_normal_right, stretch_max_right)
255250
self._face_data.set_blendshape(FaceBlendShape.MouthStretchRight, mouth_right_stretch_final)
256251

252+
253+
# when your mouth roll and shrink this will be near to 1
254+
def detect_Mouth_Roll(self,lower_lip,upper_lip,upper_outer_lip,lowest_lip):
255+
256+
257+
outer_lip_dist = self.dist(lower_lip, lowest_lip)
258+
upper_lip_dist = self.dist(upper_lip, upper_outer_lip)
259+
self._face_data.set_blendshape(
260+
FaceBlendShape.MouthRollLower, 1 - self._remap_blendshape(FaceBlendShape.MouthRollLower, outer_lip_dist))
261+
self._face_data.set_blendshape(
262+
FaceBlendShape.MouthRollUpper, 1 - self._remap_blendshape(FaceBlendShape.MouthRollUpper, upper_lip_dist))
263+
264+
257265
def detect_Jaw_direction(self,nose_tip,lowest_chin):
258266
#Jaw left right
259267
# jaw only interesting on x yxis
@@ -268,6 +276,7 @@ def detect_Jaw_direction(self,nose_tip,lowest_chin):
268276
self._face_data.set_blendshape(FaceBlendShape.JawRight, jaw_right)
269277
#-------------------------------
270278

279+
# will appear in the time of mouth press(when you hide your lips by pulling it inside your mouth!)
271280
def detect_mouth_press(self):
272281
left_upper_press = self.dist(
273282
self._get_landmark(self.blend_shape_config.CanonicalPoints.left_upper_press[0]),
@@ -292,13 +301,15 @@ def detect_mouth_press(self):
292301
mouth_press_left_final = 1 - self._remap_blendshape(FaceBlendShape.MouthPressLeft, mouth_press_left)
293302
mouth_press_right_final = 1 - self._remap_blendshape(FaceBlendShape.MouthPressRight, mouth_press_right)
294303

304+
# print(mouth_press_left_final)
295305
self._face_data.set_blendshape(
296306
FaceBlendShape.MouthPressLeft, mouth_press_left_final)
297307
self._face_data.set_blendshape(
298308
FaceBlendShape.MouthPressRight,
299309
mouth_press_right_final
300310
)
301311

312+
#will appear when your whole mouth goes to corner left down or right direction
302313
def detect_mouth_lower_direction(self,mouth_open_dist):
303314
lower_down_left = self.dist(self._get_landmark(
304315
424), self._get_landmark(319)) + mouth_open_dist * 0.5
@@ -307,9 +318,9 @@ def detect_mouth_lower_direction(self,mouth_open_dist):
307318

308319
lower_down_left_final = 1 - self._remap_blendshape(FaceBlendShape.MouthLowerDownLeft, lower_down_left)
309320
lower_down_right_final = 1 -self._remap_blendshape(FaceBlendShape.MouthLowerDownRight,lower_down_right)
310-
321+
# print(lower_down_right_final)
311322
self._face_data.set_blendshape(FaceBlendShape.MouthLowerDownLeft, lower_down_left_final)
312-
self._face_data.set_blendshape(FaceBlendShape.MouthLowerDownRight, lower_down_right_final )
323+
self._face_data.set_blendshape(FaceBlendShape.MouthLowerDownRight, lower_down_right_final)
313324

314325
def detect_mouth_shrug(self,nose_tip,uppest_lip,lowest_lip):
315326
#mouth shrug up will be near 1 if upper mouth is near nose!
@@ -327,11 +338,16 @@ def detect_mouth_shrug(self,nose_tip,uppest_lip,lowest_lip):
327338

328339
#------------------------------------------------------------------
329340

330-
def detect_mouth_pucker(self,mouth_width):
341+
def detect_mouth_pucker(self,mouth_width,mouth_open_dist):
331342
mouth_pucker = self._remap_blendshape(
332-
FaceBlendShape.MouthPucker, mouth_width)
343+
FaceBlendShape.MouthPucker, mouth_open_dist/mouth_width)
344+
# self._face_data.set_blendshape(
345+
# FaceBlendShape.MouthPucker, 1 - mouth_pucker)
333346
self._face_data.set_blendshape(
334-
FaceBlendShape.MouthPucker, 1 - mouth_pucker)
347+
FaceBlendShape.MouthPucker, mouth_pucker)
348+
349+
# print(mouth_pucker)
350+
335351
# mouth funnel only can be seen if mouth pucker is really small
336352
if self._face_data.get_blendshape(FaceBlendShape.MouthPucker) < 0.5:
337353
self._face_data.set_blendshape(
@@ -368,14 +384,16 @@ def get_eye_open_ration(points):
368384
self._get_landmark(self.blend_shape_config.CanonicalPoints.squint_left[1])
369385
)
370386
self._face_data.set_blendshape(
371-
FaceBlendShape.EyeSquintLeft, 1 - self._remap_blendshape(FaceBlendShape.EyeSquintLeft, squint_left))
387+
FaceBlendShape.EyeSquintLeft, self._remap_blendshape(FaceBlendShape.EyeSquintLeft, squint_left))
372388

373389
squint_right = self.dist(
374390
self._get_landmark(self.blend_shape_config.CanonicalPoints.squint_right[0]),
375391
self._get_landmark(self.blend_shape_config.CanonicalPoints.squint_right[1])
376392
)
377393
self._face_data.set_blendshape(
378-
FaceBlendShape.EyeSquintRight, 1 - self._remap_blendshape(FaceBlendShape.EyeSquintRight, squint_right))
394+
FaceBlendShape.EyeSquintRight, self._remap_blendshape(FaceBlendShape.EyeSquintRight, squint_right))
395+
396+
# print(self._remap_blendshape(FaceBlendShape.EyeSquintRight, squint_right))
379397

380398
#Brow
381399
self.detect_brow_actions()
@@ -403,6 +421,7 @@ def detect_blinks(self,get_eye_open_ration):
403421
FaceBlendShape.EyeWideLeft, eye_open_ratio_left))
404422
self._face_data.set_blendshape(FaceBlendShape.EyeWideRight, self._remap_blendshape(
405423
FaceBlendShape.EyeWideRight, eye_open_ratio_right))
424+
406425
# ----------------------------------------
407426

408427
def detect_brow_actions(self):
@@ -447,8 +466,9 @@ def detect_brow_actions(self):
447466
self._face_data.set_blendshape(FaceBlendShape.BrowInnerUp, brow_inner_up)
448467
# print(brow_inner_up)
449468

469+
450470
def detect_cheek(self):
451-
# Cheek is turned over left or right
471+
# Cheek is turned over left or right (will be higher when goes right up or left up) (will be 1 when nose also turn over)
452472

453473
cheek_squint_left = self.dist(
454474
self._get_landmark(self.blend_shape_config.CanonicalPoints.cheek_squint_left[0]),
@@ -466,6 +486,8 @@ def detect_cheek(self):
466486
cheek_squint_right_final = 1 - self._remap_blendshape(FaceBlendShape.CheekSquintRight, cheek_squint_right)
467487
self._face_data.set_blendshape(FaceBlendShape.CheekSquintRight, cheek_squint_right_final)
468488

489+
# print(cheek_squint_right_final)
490+
469491
# ----------------------------------------------
470492

471493
# just use the same values for cheeksquint for nose sneer, mediapipe deosn't seem to have a separate value for nose sneer

‎Backend/blendshapes/blendshape_config.py

+7-6
Original file line numberDiff line numberDiff line change
@@ -49,23 +49,24 @@ class CanonicalPoints:
4949
# FaceBlendShape.EyeLookInLeft : (-0.4, 0.0),
5050
# FaceBlendShape.EyeLookOutLeft : (-0.4, 0.0),
5151
# FaceBlendShape.EyeLookUpLeft : (-0.4, 0.0),
52-
FaceBlendShape.EyeSquintLeft : (0.37, 0.44),
52+
FaceBlendShape.EyeSquintLeft : (0.37, 0.6),
5353
FaceBlendShape.EyeWideLeft : (0.9, 1.2),
5454
FaceBlendShape.EyeBlinkRight : (0.40, 0.70),
5555
# FaceBlendShape.EyeLookDownRight : (-0.4, 0.0),
5656
# FaceBlendShape.EyeLookInRight : (-0.4, 0.0),
5757
# FaceBlendShape.EyeLookOutRight : (-0.4, 0.0),
5858
# FaceBlendShape.EyeLookUpRight : (-0.4, 0.0),
59-
FaceBlendShape.EyeSquintRight : (0.37, 0.44),
59+
FaceBlendShape.EyeSquintRight : (0.37, 0.6),
6060
FaceBlendShape.EyeWideRight : (0.9, 1.2),
6161
# FaceBlendShape.JawForward : (-0.4, 0.0),
6262
FaceBlendShape.JawLeft : (-0.4, 0.0),
6363
FaceBlendShape.JawRight : (0.0, 0.4),
6464
FaceBlendShape.JawOpen : (0.50, 0.55),
6565
FaceBlendShape.MouthClose : (3.0, 4.5),
66-
FaceBlendShape.MouthOpen : (0.02, 0.9),
66+
FaceBlendShape.MouthOpen : (0.02, 0.62),
6767
FaceBlendShape.MouthFunnel : (4.0, 4.8),
68-
FaceBlendShape.MouthPucker : (3.46, 4.92),
68+
# FaceBlendShape.MouthPucker : (3.46, 4.92),
69+
FaceBlendShape.MouthPucker : (0, 1.2),
6970
FaceBlendShape.MouthLeft : (-3.4, -2.3),
7071
FaceBlendShape.MouthRight : ( 1.5, 3.0),
7172
FaceBlendShape.MouthSmileLeft : (-0.25, 0.0),
@@ -82,8 +83,8 @@ class CanonicalPoints:
8283
FaceBlendShape.MouthShrugUpper : (1.4, 2.4),
8384
FaceBlendShape.MouthPressLeft : (0.4, 0.5),
8485
FaceBlendShape.MouthPressRight : (0.4, 0.5),
85-
FaceBlendShape.MouthLowerDownLeft : (1.7, 2.1),
86-
FaceBlendShape.MouthLowerDownRight : (1.7, 2.1),
86+
FaceBlendShape.MouthLowerDownLeft : (1.7, 2),
87+
FaceBlendShape.MouthLowerDownRight : (1.7, 2),
8788
# FaceBlendShape.MouthUpperUpLeft : (-0.4, 0.0),
8889
# FaceBlendShape.MouthUpperUpRight : (-0.4, 0.0),
8990
FaceBlendShape.BrowDownLeft : (1.0, 1.2),

0 commit comments

Comments
 (0)