indigo
e2f2e6668a
Maya API 2.0 plugin that maps a skeleton to OpenPose_full keypoints (BODY_25 or COCO body, 21+21 hand, 70 face) via an openposeCharacter node, with a live Viewport 2.0 draw override, an "OpenPose" viewport renderer plus openposeRenderSequence for PNG output, and a PySide2 mapping editor. Face can also be driven procedurally from ARKit's 52 blendshape weights instead of facial joints, for rigs with no facial skeleton. Includes VS Code debug configs and Maya test scenes/renders. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
236 lines
11 KiB
Python
236 lines
11 KiB
Python
"""Drives the 70-point OpenPose_full face part from ARKit's 52 facial
|
|
blendshape weights, for rigs that animate the face with blendShape targets
|
|
(or live ARKit/iPhone TrueDepth mocap) instead of facial joints.
|
|
|
|
``ARKIT_BLENDSHAPE_NAMES`` (order, spelling) is verbatim from Apple's own
|
|
``ARFaceAnchor.BlendShapeLocation`` enumeration.
|
|
|
|
There is no published, authoritative "which ARKit weight moves which
|
|
OpenPose/iBUG landmark, by how much" table anywhere -- unlike BODY_25/COCO,
|
|
which OpenPose's own source defines precisely, this is a genuinely
|
|
unstandardized problem. ``NEUTRAL_FACE_TEMPLATE`` and ``BLENDSHAPE_OFFSETS``
|
|
below are therefore a hand-authored, geometrically-reasoned *approximation*
|
|
(each shape nudges only the anatomically obvious nearby landmarks, in an
|
|
intuitive direction) -- good enough to produce a recognizable, responsive
|
|
OpenPose-style face for ControlNet conditioning, but not a biomechanically
|
|
accurate face simulation. A few shapes with no representable effect on a
|
|
frontal 2D landmark set (``jawForward``, ``tongueOut``, ``mouthFunnel`` --
|
|
visually redundant with ``mouthPucker`` on this coarse a template) are
|
|
intentionally no-ops.
|
|
"""
|
|
|
|
import math
|
|
|
|
from . import constants
|
|
|
|
# Verbatim from Apple's ARFaceAnchor.BlendShapeLocation.
|
|
ARKIT_BLENDSHAPE_NAMES = [
|
|
# Left eye
|
|
"eyeBlinkLeft", "eyeLookDownLeft", "eyeLookInLeft", "eyeLookOutLeft",
|
|
"eyeLookUpLeft", "eyeSquintLeft", "eyeWideLeft",
|
|
# Right eye
|
|
"eyeBlinkRight", "eyeLookDownRight", "eyeLookInRight", "eyeLookOutRight",
|
|
"eyeLookUpRight", "eyeSquintRight", "eyeWideRight",
|
|
# Jaw
|
|
"jawForward", "jawLeft", "jawRight", "jawOpen",
|
|
# Mouth
|
|
"mouthClose", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight",
|
|
"mouthSmileLeft", "mouthSmileRight", "mouthFrownLeft", "mouthFrownRight",
|
|
"mouthDimpleLeft", "mouthDimpleRight", "mouthStretchLeft", "mouthStretchRight",
|
|
"mouthRollLower", "mouthRollUpper", "mouthShrugLower", "mouthShrugUpper",
|
|
"mouthPressLeft", "mouthPressRight", "mouthLowerDownLeft", "mouthLowerDownRight",
|
|
"mouthUpperUpLeft", "mouthUpperUpRight",
|
|
# Brow
|
|
"browDownLeft", "browDownRight", "browInnerUp", "browOuterUpLeft", "browOuterUpRight",
|
|
# Cheek
|
|
"cheekPuff", "cheekSquintLeft", "cheekSquintRight",
|
|
# Nose
|
|
"noseSneerLeft", "noseSneerRight",
|
|
# Tongue
|
|
"tongueOut",
|
|
]
|
|
assert len(ARKIT_BLENDSHAPE_NAMES) == 52
|
|
BLENDSHAPE_COUNT = 52
|
|
BLENDSHAPE_NAME_TO_INDEX = {name: i for i, name in enumerate(ARKIT_BLENDSHAPE_NAMES)}
|
|
|
|
_FACE_INDEX = {name: i for i, name in enumerate(constants.FACE_NAMES)}
|
|
|
|
# Mouth outer contour = Mouth0..11 (12 pts), inner contour = Mouth12..19 (8
|
|
# pts). Within the outer contour: 0 = left corner, 1-5 = upper lip (left to
|
|
# right), 6 = right corner, 7-11 = lower lip (right to left).
|
|
_MOUTH_UPPER_OUTER = [0, 1, 2, 3, 4, 5, 6]
|
|
_MOUTH_LOWER_OUTER = [6, 7, 8, 9, 10, 11, 0]
|
|
|
|
|
|
def _mouth_outer(i):
|
|
return "Mouth{0}".format(i)
|
|
|
|
|
|
def _mouth_inner(i):
|
|
return "Mouth{0}".format(12 + i)
|
|
|
|
|
|
def _arc(cx, cy, rx, ry, start_deg, end_deg, count):
|
|
points = []
|
|
for i in range(count):
|
|
t = i / (count - 1) if count > 1 else 0.0
|
|
angle = math.radians(start_deg + (end_deg - start_deg) * t)
|
|
points.append((cx + rx * math.cos(angle), cy + ry * math.sin(angle)))
|
|
return points
|
|
|
|
|
|
def _build_neutral_template():
|
|
"""70 (x, y) points in a local, camera-facing "face plane" (roughly a
|
|
unit square, +X = screen-right, +Y = up, origin ~ between the eyes), in
|
|
the same order as constants.FACE_NAMES. Proportions are a plausible
|
|
generic face, not a specific published dataset's exact mean shape.
|
|
"""
|
|
template = [None] * constants.FACE_COUNT
|
|
|
|
# Jaw0..16: jawline, ear to ear around the chin.
|
|
for i in range(17):
|
|
angle = math.radians(180 + 180 * i / 16.0)
|
|
x = 1.05 * math.cos(angle)
|
|
y = -0.15 + 0.85 * math.sin(angle)
|
|
template[_FACE_INDEX["Jaw{0}".format(i)]] = (x, y)
|
|
|
|
# Eyebrows: shallow arcs above each eye. Right eyebrow sits at negative
|
|
# local X, left eyebrow mirrors at positive X (an arbitrary but
|
|
# internally consistent left/right convention, same idea as BODY_25).
|
|
for i, (x, y) in enumerate(_arc(-0.5, 0.62, 0.32, 0.10, 200, 340, 5)):
|
|
template[_FACE_INDEX["REyebrow{0}".format(i)]] = (x, y)
|
|
for i, (x, y) in enumerate(_arc(0.5, 0.62, 0.32, 0.10, -20, 140, 5)):
|
|
template[_FACE_INDEX["LEyebrow{0}".format(i)]] = (x, y)
|
|
|
|
# Nose: bridge (top to tip) then the nostril-base arc.
|
|
for i in range(4):
|
|
t = i / 3.0
|
|
template[_FACE_INDEX["Nose{0}".format(i)]] = (0.0, 0.42 - 0.38 * t)
|
|
for i, (x, y) in enumerate(_arc(0.0, -0.02, 0.20, 0.06, 200, 340, 5)):
|
|
template[_FACE_INDEX["Nose{0}".format(4 + i)]] = (x, y)
|
|
|
|
# Eyes: 6 points each (outer corner, 2x upper lid, inner corner, 2x lower lid).
|
|
for i, (x, y) in enumerate(_arc(-0.5, 0.32, 0.30, 0.14, 0, 300, 6)):
|
|
template[_FACE_INDEX["REye{0}".format(i)]] = (x, y)
|
|
for i, (x, y) in enumerate(_arc(0.5, 0.32, 0.30, 0.14, 180, -120, 6)):
|
|
template[_FACE_INDEX["LEye{0}".format(i)]] = (x, y)
|
|
|
|
# Mouth: 12-point outer contour, then 8-point inner contour.
|
|
for i, (x, y) in enumerate(_arc(0.0, -0.55, 0.42, 0.20, 0, 300, 12)):
|
|
template[_FACE_INDEX["Mouth{0}".format(i)]] = (x, y)
|
|
for i, (x, y) in enumerate(_arc(0.0, -0.55, 0.28, 0.11, 0, 315, 8)):
|
|
template[_FACE_INDEX["Mouth{0}".format(12 + i)]] = (x, y)
|
|
|
|
template[_FACE_INDEX["RPupil"]] = (-0.5, 0.32)
|
|
template[_FACE_INDEX["LPupil"]] = (0.5, 0.32)
|
|
|
|
assert all(p is not None for p in template)
|
|
return template
|
|
|
|
|
|
NEUTRAL_FACE_TEMPLATE = _build_neutral_template()
|
|
|
|
|
|
def _sign(value):
|
|
return 1.0 if value >= 0 else -1.0
|
|
|
|
|
|
# {blendshape_name: [(face_point_name, dx, dy), ...]} -- displacement applied
|
|
# at weight 1.0, linearly scaled by the live weight. See module docstring for
|
|
# the honesty caveat on precision.
|
|
BLENDSHAPE_OFFSETS = {
|
|
# --- Eyes: eyelid closure / widening (upper lid moves toward or away
|
|
# from the lower lid; lower lid nudges slightly the opposite way).
|
|
"eyeBlinkLeft": [("LEye1", 0.0, -0.10), ("LEye2", 0.0, -0.10)],
|
|
"eyeBlinkRight": [("REye1", 0.0, -0.10), ("REye2", 0.0, -0.10)],
|
|
"eyeWideLeft": [("LEye1", 0.0, 0.05), ("LEye2", 0.0, 0.05), ("LEye4", 0.0, -0.03), ("LEye5", 0.0, -0.03)],
|
|
"eyeWideRight": [("REye1", 0.0, 0.05), ("REye2", 0.0, 0.05), ("REye4", 0.0, -0.03), ("REye5", 0.0, -0.03)],
|
|
"eyeSquintLeft": [("LEye1", 0.0, -0.04), ("LEye2", 0.0, -0.04), ("LEye4", 0.0, 0.02), ("LEye5", 0.0, 0.02)],
|
|
"eyeSquintRight": [("REye1", 0.0, -0.04), ("REye2", 0.0, -0.04), ("REye4", 0.0, 0.02), ("REye5", 0.0, 0.02)],
|
|
# Gaze direction moves the pupil point within the eye socket.
|
|
"eyeLookDownLeft": [("LPupil", 0.0, -0.06)],
|
|
"eyeLookUpLeft": [("LPupil", 0.0, 0.06)],
|
|
"eyeLookInLeft": [("LPupil", -0.08, 0.0)],
|
|
"eyeLookOutLeft": [("LPupil", 0.08, 0.0)],
|
|
"eyeLookDownRight": [("RPupil", 0.0, -0.06)],
|
|
"eyeLookUpRight": [("RPupil", 0.0, 0.06)],
|
|
"eyeLookInRight": [("RPupil", 0.08, 0.0)],
|
|
"eyeLookOutRight": [("RPupil", -0.08, 0.0)],
|
|
|
|
# --- Brows.
|
|
"browDownLeft": [("LEyebrow{0}".format(i), 0.0, -0.08) for i in range(5)],
|
|
"browDownRight": [("REyebrow{0}".format(i), 0.0, -0.08) for i in range(5)],
|
|
"browInnerUp": [("LEyebrow0", 0.0, 0.09), ("REyebrow4", 0.0, 0.09)],
|
|
"browOuterUpLeft": [("LEyebrow3", 0.0, 0.08), ("LEyebrow4", 0.0, 0.08)],
|
|
"browOuterUpRight": [("REyebrow0", 0.0, 0.08), ("REyebrow1", 0.0, 0.08)],
|
|
|
|
# --- Jaw. jawForward has no representable 2D effect (frontal view) --
|
|
# intentionally omitted below.
|
|
"jawOpen": (
|
|
[("Jaw{0}".format(i), 0.0, -0.30) for i in range(6, 11)]
|
|
+ [(_mouth_outer(i), 0.0, -0.22) for i in _MOUTH_LOWER_OUTER]
|
|
+ [(_mouth_inner(i), 0.0, -0.16) for i in range(4, 7)]
|
|
),
|
|
"jawLeft": [("Jaw{0}".format(i), 0.06, 0.0) for i in range(6, 11)],
|
|
"jawRight": [("Jaw{0}".format(i), -0.06, 0.0) for i in range(6, 11)],
|
|
|
|
# --- Mouth shape.
|
|
"mouthSmileLeft": [(_mouth_outer(0), 0.05, 0.10), (_mouth_outer(1), 0.03, 0.06), (_mouth_outer(11), 0.03, 0.06)],
|
|
"mouthSmileRight": [(_mouth_outer(6), -0.05, 0.10), (_mouth_outer(5), -0.03, 0.06), (_mouth_outer(7), -0.03, 0.06)],
|
|
"mouthFrownLeft": [(_mouth_outer(0), 0.02, -0.08)],
|
|
"mouthFrownRight": [(_mouth_outer(6), -0.02, -0.08)],
|
|
"mouthLeft": [(_mouth_outer(i), 0.08, 0.0) for i in range(12)],
|
|
"mouthRight": [(_mouth_outer(i), -0.08, 0.0) for i in range(12)],
|
|
"mouthDimpleLeft": [(_mouth_outer(0), 0.02, 0.02)],
|
|
"mouthDimpleRight": [(_mouth_outer(6), -0.02, 0.02)],
|
|
"mouthStretchLeft": [(_mouth_outer(0), 0.10, 0.0)],
|
|
"mouthStretchRight": [(_mouth_outer(6), -0.10, 0.0)],
|
|
"mouthPucker": [
|
|
(_mouth_outer(i), -0.10 * _sign(NEUTRAL_FACE_TEMPLATE[_FACE_INDEX[_mouth_outer(i)]][0]), 0.0)
|
|
for i in range(12)
|
|
],
|
|
"mouthFunnel": [], # visually redundant with mouthPucker on this coarse template
|
|
"mouthClose": (
|
|
[(_mouth_outer(i), 0.0, -0.02) for i in _MOUTH_UPPER_OUTER]
|
|
+ [(_mouth_outer(i), 0.0, 0.02) for i in _MOUTH_LOWER_OUTER]
|
|
),
|
|
"mouthRollLower": [(_mouth_outer(i), 0.0, 0.03) for i in _MOUTH_LOWER_OUTER],
|
|
"mouthRollUpper": [(_mouth_outer(i), 0.0, -0.03) for i in _MOUTH_UPPER_OUTER],
|
|
"mouthShrugLower": [(_mouth_outer(i), 0.0, -0.04) for i in _MOUTH_LOWER_OUTER],
|
|
"mouthShrugUpper": [(_mouth_outer(i), 0.0, 0.04) for i in _MOUTH_UPPER_OUTER],
|
|
"mouthPressLeft": [(_mouth_outer(0), 0.02, -0.01)],
|
|
"mouthPressRight": [(_mouth_outer(6), -0.02, -0.01)],
|
|
"mouthLowerDownLeft": [(_mouth_outer(7), 0.0, -0.06), (_mouth_outer(8), 0.0, -0.06)],
|
|
"mouthLowerDownRight": [(_mouth_outer(10), 0.0, -0.06), (_mouth_outer(11), 0.0, -0.06)],
|
|
"mouthUpperUpLeft": [(_mouth_outer(1), 0.0, 0.05), (_mouth_outer(2), 0.0, 0.05)],
|
|
"mouthUpperUpRight": [(_mouth_outer(4), 0.0, 0.05), (_mouth_outer(5), 0.0, 0.05)],
|
|
|
|
# --- Cheeks / nose: subtle.
|
|
"cheekPuff": [("Jaw{0}".format(i), 0.03, 0.0) for i in (3, 4)] + [("Jaw{0}".format(i), -0.03, 0.0) for i in (12, 13)],
|
|
"cheekSquintLeft": [("LEye4", 0.0, 0.02), ("LEye5", 0.0, 0.02)],
|
|
"cheekSquintRight": [("REye4", 0.0, 0.02), ("REye5", 0.0, 0.02)],
|
|
"noseSneerLeft": [("Nose7", 0.0, 0.03), ("Nose8", 0.0, 0.03)],
|
|
"noseSneerRight": [("Nose4", 0.0, 0.03), ("Nose5", 0.0, 0.03)],
|
|
|
|
# --- No 2D-representable effect on a frontal landmark set.
|
|
"jawForward": [],
|
|
"tongueOut": [],
|
|
}
|
|
assert set(BLENDSHAPE_OFFSETS.keys()) == set(ARKIT_BLENDSHAPE_NAMES)
|
|
|
|
|
|
def compute_face_local_points(weights):
|
|
"""``weights``: {blendshape_name: 0..1}. Returns 70 (x, y) local points
|
|
(constants.FACE_NAMES order), the neutral template plus every active
|
|
blendshape's weighted displacement.
|
|
"""
|
|
points = list(NEUTRAL_FACE_TEMPLATE)
|
|
for name, weight in weights.items():
|
|
if not weight:
|
|
continue
|
|
for point_name, dx, dy in BLENDSHAPE_OFFSETS.get(name, ()):
|
|
idx = _FACE_INDEX[point_name]
|
|
x, y = points[idx]
|
|
points[idx] = (x + dx * weight, y + dy * weight)
|
|
return points
|