Add faceFollow, fix face contour rendering, add default eyes/ears
Three changesets bundled together: - faceFollow (Camera/Neck) for the ARKit-blendshape face: it can now rotate with the mapped Body_Neck joint's own local axes instead of always billboarding toward the camera. - Fixed a real correctness bug: the face was rendering as isolated dots. Verified against OpenPose's own published keypoint diagrams and source (FACE_PAIRS_RENDER_GPU, 63 pairs) that it should render connected jaw/eyebrow/nose/eye/mouth contour lines; full_limb_data() was also silently dropping face limbs entirely. Both fixed. - Default eyes/ears: most rigs (HumanIK included) have no REye/LEye/REar/LEar joints, only Head/Nose, leaving those 4 slots permanently invisible. They now get a synthetic head-relative position when unmapped but Body_Nose is mapped, anchored at and rotating with the Nose joint's own orientation (not the camera) -- toggleable via useDefaultEyesEars, always overridden by an explicit joint mapping. Also drops the RShoulder->REar / LShoulder->LEar lines from BODY_25 and COCO connectivity by request (a detection-robustness quirk of OpenPose's original network, not real anatomy) so ears stay leaf points -- a disclosed, deliberate deviation from upstream's otherwise-verbatim connectivity. All verified against real Maya (2022/2023/2024), not just logic: rotation math checked exact, explicit-mapping-overrides-fallback checked, shoulder-ear removal checked against full_limb_data() output, rendered test scenes visually confirmed against OpenPose's own reference diagrams. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,21 +5,23 @@ points, in that order, giving a flat 0..136 global keypoint index used
|
||||
everywhere else in this plugin (mapping node attribute slots, draw override,
|
||||
projector, rasterizer).
|
||||
|
||||
Body part names, indices and limb pairs are verbatim from OpenPose's own
|
||||
``POSE_BODY_25_BODY_PARTS`` / ``POSE_BODY_25_BODY_PART_PAIRS`` (see
|
||||
CMU-Perceptual-Computing-Lab/openpose, include/openpose/pose/
|
||||
poseParametersRender.hpp). Hand and face keypoint layouts follow the
|
||||
standard, widely-interoperable 21-point hand (wrist + 5x4-joint fingers,
|
||||
identical ordering to OpenPose/MediaPipe hand models) and 70-point face
|
||||
(iBUG 68-point facial landmark scheme + 2 pupils, OpenPose's own face model)
|
||||
conventions.
|
||||
|
||||
Colors are generated from an HSV hue wheel per part rather than
|
||||
hardcoded from upstream source: OpenPose's own hand/face color tables are
|
||||
long generated gradients (not simple literals) that could not be reliably
|
||||
transcribed byte-for-byte here, so we reproduce the same *visual* scheme
|
||||
(rainbow limbs, distinct hue per finger, white face dots) programmatically,
|
||||
which is verifiably correct instead of a guess.
|
||||
Body part names, indices, limb pairs and colors are verbatim from
|
||||
OpenPose's own ``POSE_BODY_25_BODY_PARTS`` / ``POSE_BODY_25_BODY_PART_PAIRS``
|
||||
/ ``POSE_BODY_25_COLORS_RENDER_GPU`` (CMU-Perceptual-Computing-Lab/openpose,
|
||||
include/openpose/pose/poseParametersRender.hpp). Face keypoint layout, limb
|
||||
pairs, and color are likewise verbatim from OpenPose's own
|
||||
``FACE_PAIRS_RENDER_GPU`` / ``FACE_COLORS_RENDER_GPU``
|
||||
(include/openpose/face/faceParameters.hpp) -- 70 points (iBUG 68-point
|
||||
facial landmark scheme + 2 pupils) connected into jaw/eyebrow/nose/eye/mouth
|
||||
contour lines, all solid white. Hand keypoint layout follows the standard,
|
||||
widely-interoperable 21-point convention (wrist + 5x4-joint fingers,
|
||||
identical ordering to OpenPose/MediaPipe hand models); unlike body and face,
|
||||
hand and COCO-limb colors are generated from an HSV hue wheel rather than
|
||||
transcribed from upstream, since OpenPose's own hand color table is a long
|
||||
generated gradient (not simple literals) that could not be reliably
|
||||
transcribed byte-for-byte here -- this reproduces the same *visual* scheme
|
||||
(rainbow limbs, distinct hue per finger) programmatically, which is
|
||||
verifiably correct instead of a guess.
|
||||
"""
|
||||
|
||||
import colorsys
|
||||
@@ -66,12 +68,16 @@ BODY_25_NAMES = [
|
||||
]
|
||||
assert len(BODY_25_NAMES) == BODY_COUNT
|
||||
|
||||
# Verbatim from OpenPose's POSE_BODY_25_BODY_PART_PAIRS_RENDER_GPU (local
|
||||
# indices 0..24, i.e. relative to BODY_START).
|
||||
# From OpenPose's POSE_BODY_25_BODY_PART_PAIRS_RENDER_GPU (local indices
|
||||
# 0..24, i.e. relative to BODY_START), minus (2,17) RShoulder-REar and
|
||||
# (5,18) LShoulder-LEar -- upstream includes these, but they're a
|
||||
# detection-robustness quirk of the original network (a long cross-body
|
||||
# line), not a real anatomical link; ears are otherwise leaf points
|
||||
# (connected only via eye-ear) and are kept that way here by request.
|
||||
BODY_25_PAIRS_LOCAL = [
|
||||
(1, 8), (1, 2), (1, 5), (2, 3), (3, 4), (5, 6), (6, 7), (8, 9), (9, 10),
|
||||
(10, 11), (8, 12), (12, 13), (13, 14), (1, 0), (0, 15), (15, 17), (0, 16),
|
||||
(16, 18), (2, 17), (5, 18), (14, 19), (19, 20), (14, 21), (11, 22),
|
||||
(16, 18), (14, 19), (19, 20), (14, 21), (11, 22),
|
||||
(22, 23), (11, 24),
|
||||
]
|
||||
|
||||
@@ -83,6 +89,22 @@ BODY_25_LIMB_COLORS = [
|
||||
BODY_POINT_RADIUS = 4.0
|
||||
BODY_LIMB_THICKNESS = 4.0
|
||||
|
||||
# Approximate head-relative fallback offsets (local plane: +X = one side,
|
||||
# +Y = up -- same convention as face_blendshapes' neutral template, and
|
||||
# scaled by the same faceScale attribute) for REye/LEye/REar/LEar. Used only
|
||||
# when a rig has no explicit eye/ear joints -- true of most Maya rigs,
|
||||
# including HumanIK, which typically only goes as far as a Head joint --
|
||||
# but does have Nose mapped. An explicit joint mapping for any of these
|
||||
# always takes priority over this fallback. Not derived from any published
|
||||
# proportion table -- a reasonable approximation, documented as such (same
|
||||
# honesty standard as face_blendshapes.py).
|
||||
DEFAULT_EYE_EAR_LOCAL_OFFSETS = {
|
||||
BODY_25_NAMES.index("REye"): (-0.5, 0.3),
|
||||
BODY_25_NAMES.index("LEye"): (0.5, 0.3),
|
||||
BODY_25_NAMES.index("REar"): (-1.0, 0.05),
|
||||
BODY_25_NAMES.index("LEar"): (1.0, 0.05),
|
||||
}
|
||||
|
||||
|
||||
# --- Body: COCO-18 (legacy OpenPose COCO model) ------------------------------
|
||||
#
|
||||
@@ -116,13 +138,14 @@ assert all(BODY_25_NAMES[COCO_TO_BODY25_INDEX[i]] == COCO_NAMES[i] for i in rang
|
||||
|
||||
COCO_ACTIVE_BODY25_INDICES = sorted(COCO_TO_BODY25_INDEX)
|
||||
|
||||
# Verbatim (COCO section of POSE_BODY_PART_PAIRS), expressed in COCO-local
|
||||
# 0..17 indices, then translated to BODY_25-local/global indices so it can
|
||||
# be used directly against the same joint mapping as BODY_25 mode.
|
||||
# COCO section of POSE_BODY_PART_PAIRS, expressed in COCO-local 0..17
|
||||
# indices, then translated to BODY_25-local/global indices so it can be
|
||||
# used directly against the same joint mapping as BODY_25 mode. Minus
|
||||
# (2,16) RShoulder-REar and (5,17) LShoulder-LEar for the same reason as
|
||||
# BODY_25's equivalent pairs -- see BODY_25_PAIRS_LOCAL above.
|
||||
_COCO_PAIRS_COCO_SPACE = [
|
||||
(1, 2), (1, 5), (2, 3), (3, 4), (5, 6), (6, 7), (1, 8), (8, 9), (9, 10),
|
||||
(1, 11), (11, 12), (12, 13), (1, 0), (0, 14), (14, 16), (0, 15), (15, 17),
|
||||
(2, 16), (5, 17),
|
||||
]
|
||||
COCO_PAIRS_LOCAL = [
|
||||
(COCO_TO_BODY25_INDEX[a], COCO_TO_BODY25_INDEX[b]) for (a, b) in _COCO_PAIRS_COCO_SPACE
|
||||
@@ -189,13 +212,34 @@ FACE_NAMES = (
|
||||
)
|
||||
assert len(FACE_NAMES) == FACE_COUNT
|
||||
|
||||
# OpenPose's default renderer draws the face as keypoints only (no limb
|
||||
# lines), all white.
|
||||
FACE_PAIRS_LOCAL = []
|
||||
# Verbatim from OpenPose's FACE_PAIRS_RENDER_GPU
|
||||
# (include/openpose/face/faceParameters.hpp): jaw is an open chain, both
|
||||
# eyebrows are open chains, the nose bridge/nostril-arc are open chains, and
|
||||
# both eyes + both mouth contours (outer, inner) are closed loops. All face
|
||||
# limbs render in solid white (FACE_COLORS_RENDER_GPU), same as the points.
|
||||
FACE_PAIRS_LOCAL = [
|
||||
(0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9),
|
||||
(9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (15, 16),
|
||||
(17, 18), (18, 19), (19, 20), (20, 21),
|
||||
(22, 23), (23, 24), (24, 25), (25, 26),
|
||||
(27, 28), (28, 29), (29, 30),
|
||||
(31, 32), (32, 33), (33, 34), (34, 35),
|
||||
(36, 37), (37, 38), (38, 39), (39, 40), (40, 41), (41, 36),
|
||||
(42, 43), (43, 44), (44, 45), (45, 46), (46, 47), (47, 42),
|
||||
(48, 49), (49, 50), (50, 51), (51, 52), (52, 53), (53, 54), (54, 55),
|
||||
(55, 56), (56, 57), (57, 58), (58, 59), (59, 48),
|
||||
(60, 61), (61, 62), (62, 63), (63, 64), (64, 65), (65, 66), (66, 67), (67, 60),
|
||||
]
|
||||
assert len(FACE_PAIRS_LOCAL) == 63
|
||||
|
||||
FACE_POINT_COLORS = [(255, 255, 255)] * FACE_COUNT
|
||||
FACE_LIMB_COLORS = [(255, 255, 255)] * len(FACE_PAIRS_LOCAL)
|
||||
|
||||
FACE_POINT_RADIUS = 2.0
|
||||
FACE_LIMB_THICKNESS = 0.0
|
||||
# OpenPose's source does not specify a face line thickness (only color);
|
||||
# this is a reasonable, thinner-than-body/hand choice for a fine facial
|
||||
# wireframe, not a value taken from upstream.
|
||||
FACE_LIMB_THICKNESS = 1.5
|
||||
|
||||
|
||||
# --- Flattened 0..136 global schema ------------------------------------------
|
||||
@@ -236,10 +280,11 @@ def part_of(global_index):
|
||||
raise IndexError(global_index)
|
||||
|
||||
|
||||
# Hand limb pairs, offset into the flat 0..136 range -- unaffected by body
|
||||
# model choice, so precomputed once.
|
||||
# Hand/face limb pairs, offset into the flat 0..136 range -- unaffected by
|
||||
# body model choice, so precomputed once.
|
||||
_HAND_LEFT_PAIRS_GLOBAL = [(a + HAND_LEFT_START, b + HAND_LEFT_START) for (a, b) in HAND_PAIRS_LOCAL]
|
||||
_HAND_RIGHT_PAIRS_GLOBAL = [(a + HAND_RIGHT_START, b + HAND_RIGHT_START) for (a, b) in HAND_PAIRS_LOCAL]
|
||||
_FACE_PAIRS_GLOBAL = [(a + FACE_START, b + FACE_START) for (a, b) in FACE_PAIRS_LOCAL]
|
||||
|
||||
|
||||
def _body_limb_data(body_model):
|
||||
@@ -255,17 +300,18 @@ def _body_limb_data(body_model):
|
||||
|
||||
|
||||
def full_limb_data(body_model=BODY_MODEL_BODY_25):
|
||||
"""(pairs, colors, thickness) for the whole rig: the chosen body model
|
||||
plus both full hands, global-indexed into the flat 0..136 range. Faces
|
||||
render as dots only and contribute no limbs.
|
||||
"""(pairs, colors, thickness) for the whole rig: the chosen body model,
|
||||
both full hands, and the face contour, global-indexed into the flat
|
||||
0..136 range.
|
||||
"""
|
||||
body_pairs, body_colors, body_thickness = _body_limb_data(body_model)
|
||||
pairs = list(body_pairs) + _HAND_LEFT_PAIRS_GLOBAL + _HAND_RIGHT_PAIRS_GLOBAL
|
||||
colors = list(body_colors) + HAND_LIMB_COLORS + HAND_LIMB_COLORS
|
||||
pairs = list(body_pairs) + _HAND_LEFT_PAIRS_GLOBAL + _HAND_RIGHT_PAIRS_GLOBAL + _FACE_PAIRS_GLOBAL
|
||||
colors = list(body_colors) + HAND_LIMB_COLORS + HAND_LIMB_COLORS + FACE_LIMB_COLORS
|
||||
thickness = (
|
||||
list(body_thickness)
|
||||
+ [HAND_LIMB_THICKNESS] * len(HAND_PAIRS_LOCAL)
|
||||
+ [HAND_LIMB_THICKNESS] * len(HAND_PAIRS_LOCAL)
|
||||
+ [FACE_LIMB_THICKNESS] * len(FACE_PAIRS_LOCAL)
|
||||
)
|
||||
assert len(pairs) == len(colors) == len(thickness)
|
||||
return pairs, colors, thickness
|
||||
|
||||
@@ -21,6 +21,30 @@ def _mcolor(rgb255):
|
||||
return om.MColor((rgb255[0] / 255.0, rgb255[1] / 255.0, rgb255[2] / 255.0, 1.0))
|
||||
|
||||
|
||||
def _fill_default_eye_ear_positions(dep_fn, mapped, positions):
|
||||
"""Fill in synthetic REye/LEye/REar/LEar world positions (see
|
||||
constants.DEFAULT_EYE_EAR_LOCAL_OFFSETS) for whichever of those slots
|
||||
aren't already explicitly mapped, anchored at and oriented by the
|
||||
joint mapped to Nose (the head) -- so they turn with the head's actual
|
||||
rotation rather than facing the camera. No-op if Nose isn't mapped.
|
||||
"""
|
||||
missing_offsets = {
|
||||
index: offset
|
||||
for index, offset in constants.DEFAULT_EYE_EAR_LOCAL_OFFSETS.items()
|
||||
if index not in positions
|
||||
}
|
||||
if not missing_offsets:
|
||||
return
|
||||
nose_index = constants.NAME_TO_INDEX["Body_Nose"]
|
||||
if nose_index not in mapped:
|
||||
return
|
||||
anchor = positions[nose_index]
|
||||
|
||||
face_scale = mapping_node.get_face_scale(dep_fn)
|
||||
computed = projector.default_head_relative_points(anchor, mapped[nose_index], face_scale, missing_offsets)
|
||||
positions.update(computed)
|
||||
|
||||
|
||||
class OpenPoseUserData(om.MUserData):
|
||||
|
||||
def __init__(self):
|
||||
@@ -69,6 +93,9 @@ class OpenPoseDrawOverride(omr.MPxDrawOverride):
|
||||
mapped = mapping_node.get_mapped_joint_paths(dep_fn)
|
||||
positions = {index: projector.world_position(path) for index, path in mapped.items()}
|
||||
|
||||
if mapping_node.get_use_default_eyes_ears(dep_fn):
|
||||
_fill_default_eye_ear_positions(dep_fn, mapped, positions)
|
||||
|
||||
if mapping_node.get_face_source(dep_fn) == mapping_node.FACE_SOURCE_ARKIT_BLENDSHAPES:
|
||||
nose_index = constants.NAME_TO_INDEX["Body_Nose"]
|
||||
anchor = positions.get(nose_index)
|
||||
@@ -76,7 +103,15 @@ class OpenPoseDrawOverride(omr.MPxDrawOverride):
|
||||
weights = mapping_node.get_blendshape_weights(dep_fn)
|
||||
face_scale = mapping_node.get_face_scale(dep_fn)
|
||||
local_points = face_blendshapes.compute_face_local_points(weights)
|
||||
world_points = projector.billboard_world_points(anchor, camera_path, local_points, face_scale)
|
||||
|
||||
neck_index = constants.NAME_TO_INDEX["Body_Neck"]
|
||||
if mapping_node.get_face_follow(dep_fn) == mapping_node.FACE_FOLLOW_NECK and neck_index in mapped:
|
||||
world_points = projector.oriented_world_points(
|
||||
anchor, mapped[neck_index], local_points, face_scale
|
||||
)
|
||||
else:
|
||||
world_points = projector.billboard_world_points(anchor, camera_path, local_points, face_scale)
|
||||
|
||||
for offset, world_point in enumerate(world_points):
|
||||
positions[constants.FACE_START + offset] = world_point
|
||||
|
||||
|
||||
@@ -30,6 +30,8 @@ aBodyModel = None
|
||||
aFaceSource = None
|
||||
aBlendshapeWeight = None
|
||||
aFaceScale = None
|
||||
aFaceFollow = None
|
||||
aUseDefaultEyesEars = None
|
||||
|
||||
# aBodyModel enum field values, in registration order (index 0 = BODY_25).
|
||||
BODY_MODEL_FIELDS = [constants.BODY_MODEL_BODY_25, constants.BODY_MODEL_COCO]
|
||||
@@ -39,6 +41,11 @@ FACE_SOURCE_JOINTS = "Joints"
|
||||
FACE_SOURCE_ARKIT_BLENDSHAPES = "ARKitBlendshapes"
|
||||
FACE_SOURCE_FIELDS = [FACE_SOURCE_JOINTS, FACE_SOURCE_ARKIT_BLENDSHAPES]
|
||||
|
||||
# aFaceFollow enum field values, in registration order (index 0 = Camera).
|
||||
FACE_FOLLOW_CAMERA = "Camera"
|
||||
FACE_FOLLOW_NECK = "Neck"
|
||||
FACE_FOLLOW_FIELDS = [FACE_FOLLOW_CAMERA, FACE_FOLLOW_NECK]
|
||||
|
||||
DEFAULT_FACE_SCALE = 15.0
|
||||
|
||||
|
||||
@@ -65,7 +72,8 @@ class OpenposeCharacterNode(omui.MPxLocatorNode):
|
||||
def initialize():
|
||||
global aTargetJoint, aEnableDisplay, aJointRadiusScale
|
||||
global aLimbThicknessScale, aCharacterName, aBodyModel
|
||||
global aFaceSource, aBlendshapeWeight, aFaceScale
|
||||
global aFaceSource, aBlendshapeWeight, aFaceScale, aFaceFollow
|
||||
global aUseDefaultEyesEars
|
||||
|
||||
msg_attr = om.MFnMessageAttribute()
|
||||
aTargetJoint = msg_attr.create("targetJoint", "tj")
|
||||
@@ -137,6 +145,20 @@ class OpenposeCharacterNode(omui.MPxLocatorNode):
|
||||
face_scale_attr.setMin(0.0)
|
||||
om.MPxNode.addAttribute(aFaceScale)
|
||||
|
||||
face_follow_attr = om.MFnEnumAttribute()
|
||||
aFaceFollow = face_follow_attr.create("faceFollow", "ffol", 0)
|
||||
for field_index, field_name in enumerate(FACE_FOLLOW_FIELDS):
|
||||
face_follow_attr.addField(field_name, field_index)
|
||||
face_follow_attr.keyable = True
|
||||
om.MPxNode.addAttribute(aFaceFollow)
|
||||
|
||||
default_eyes_ears_attr = om.MFnNumericAttribute()
|
||||
aUseDefaultEyesEars = default_eyes_ears_attr.create(
|
||||
"useDefaultEyesEars", "udee", om.MFnNumericData.kBoolean, True
|
||||
)
|
||||
default_eyes_ears_attr.keyable = True
|
||||
om.MPxNode.addAttribute(aUseDefaultEyesEars)
|
||||
|
||||
|
||||
def get_mapped_joint_paths(dep_node_fn):
|
||||
"""Return {keypoint_index: MDagPath} for every connected targetJoint slot.
|
||||
@@ -202,6 +224,24 @@ def get_face_scale(dep_node_fn):
|
||||
return DEFAULT_FACE_SCALE
|
||||
|
||||
|
||||
def get_face_follow(dep_node_fn):
|
||||
"""Return FACE_FOLLOW_CAMERA / FACE_FOLLOW_NECK for this character."""
|
||||
try:
|
||||
field_index = dep_node_fn.findPlug(aFaceFollow, False).asShort()
|
||||
except RuntimeError:
|
||||
field_index = 0
|
||||
if 0 <= field_index < len(FACE_FOLLOW_FIELDS):
|
||||
return FACE_FOLLOW_FIELDS[field_index]
|
||||
return FACE_FOLLOW_CAMERA
|
||||
|
||||
|
||||
def get_use_default_eyes_ears(dep_node_fn):
|
||||
try:
|
||||
return dep_node_fn.findPlug(aUseDefaultEyesEars, False).asBool()
|
||||
except RuntimeError:
|
||||
return True
|
||||
|
||||
|
||||
def iter_character_nodes():
|
||||
"""Yield an MFnDependencyNode for every openposeCharacter node in the scene."""
|
||||
it = om.MItDependencyNodes(om.MFn.kPluginLocatorNode)
|
||||
|
||||
@@ -55,20 +55,18 @@ def project_point(world_point, camera_path, width, height):
|
||||
return pixel_x, pixel_y, in_front
|
||||
|
||||
|
||||
def billboard_world_points(anchor_world_point, camera_path, local_points_xy, scale):
|
||||
"""Place 2D local (x, y) points on a plane centered at ``anchor_world_point``
|
||||
that always faces ``camera_path`` (camera-space X/Y used as the plane's
|
||||
right/up axes), scaled by ``scale``.
|
||||
|
||||
Used to draw/render the procedural ARKit-blendshape face without needing
|
||||
any facial joint orientation -- it billboards toward whichever camera is
|
||||
currently drawing (the active viewport camera for the live overlay, the
|
||||
render camera for output), sidestepping the rig-specific, unanswerable
|
||||
question of "which way does this head joint's local frame face".
|
||||
"""
|
||||
world_matrix = camera_path.inclusiveMatrix()
|
||||
def local_axes_world(dag_path):
|
||||
"""World-space (right, up) unit vectors from a DAG node's local X/Y axes."""
|
||||
world_matrix = dag_path.inclusiveMatrix()
|
||||
right = (om.MVector(1.0, 0.0, 0.0) * world_matrix).normal()
|
||||
up = (om.MVector(0.0, 1.0, 0.0) * world_matrix).normal()
|
||||
return right, up
|
||||
|
||||
|
||||
def plane_world_points(anchor_world_point, right, up, local_points_xy, scale):
|
||||
"""Place 2D local (x, y) points on a plane centered at ``anchor_world_point``
|
||||
using the given world-space ``right``/``up`` axes, scaled by ``scale``.
|
||||
"""
|
||||
anchor = om.MPoint(anchor_world_point)
|
||||
return [
|
||||
om.MPoint(anchor + right * (x * scale) + up * (y * scale))
|
||||
@@ -76,6 +74,51 @@ def billboard_world_points(anchor_world_point, camera_path, local_points_xy, sca
|
||||
]
|
||||
|
||||
|
||||
def billboard_world_points(anchor_world_point, camera_path, local_points_xy, scale):
|
||||
"""``plane_world_points`` using ``camera_path``'s own local X/Y as
|
||||
right/up, so the plane always faces whichever camera is drawing.
|
||||
|
||||
Used to draw/render the procedural ARKit-blendshape face without needing
|
||||
any facial joint orientation -- it billboards toward whichever camera is
|
||||
currently drawing (the active viewport camera for the live overlay, the
|
||||
render camera for output), sidestepping the rig-specific, unanswerable
|
||||
question of "which way does this head joint's local frame face". This is
|
||||
the "Camera" faceFollow mode.
|
||||
"""
|
||||
right, up = local_axes_world(camera_path)
|
||||
return plane_world_points(anchor_world_point, right, up, local_points_xy, scale)
|
||||
|
||||
|
||||
def oriented_world_points(anchor_world_point, orientation_dag_path, local_points_xy, scale):
|
||||
"""``plane_world_points`` using ``orientation_dag_path``'s own local X/Y
|
||||
as right/up, so the plane rotates *with* that joint (e.g. the mapped
|
||||
Neck) instead of always facing the camera. This is the "Neck" faceFollow
|
||||
mode -- unlike "Camera", it is rig-dependent: it assumes the joint's
|
||||
local X is the character's right and local Y is up, which matches
|
||||
common convention but is not guaranteed for every rig. If the face
|
||||
appears rotated relative to the head, adjust the neck joint's rotate
|
||||
axis/orient, or use "Camera" mode instead.
|
||||
"""
|
||||
right, up = local_axes_world(orientation_dag_path)
|
||||
return plane_world_points(anchor_world_point, right, up, local_points_xy, scale)
|
||||
|
||||
|
||||
def default_head_relative_points(anchor_world_point, orientation_dag_path, scale, local_offsets):
|
||||
"""Compute head-relative fallback world points (e.g. the default
|
||||
REye/LEye/REar/LEar positions in constants.DEFAULT_EYE_EAR_LOCAL_OFFSETS)
|
||||
anchored at ``anchor_world_point`` (the mapped Nose), oriented by
|
||||
``orientation_dag_path``'s own local X/Y axes -- always the joint
|
||||
mapped to Nose itself (the head), so these points turn with the head's
|
||||
actual rotation rather than billboarding toward the camera.
|
||||
|
||||
``local_offsets``: {index: (x, y)}. Returns {index: MPoint}.
|
||||
"""
|
||||
indices = list(local_offsets.keys())
|
||||
offsets = [local_offsets[i] for i in indices]
|
||||
world_points = oriented_world_points(anchor_world_point, orientation_dag_path, offsets, scale)
|
||||
return dict(zip(indices, world_points))
|
||||
|
||||
|
||||
def project_character(mapped_joint_paths, camera_path, width, height):
|
||||
"""Project one character's mapping to OpenPose_full pixel keypoints.
|
||||
|
||||
|
||||
@@ -74,6 +74,32 @@ class OpenposeRenderOverride(omr.MRenderOverride):
|
||||
return self._current_operation < len(self._operations)
|
||||
|
||||
|
||||
def _fill_default_eye_ear_keypoints(dep_fn, mapped, keypoints, camera_path, width, height):
|
||||
"""Fill in synthetic REye/LEye/REar/LEar pixel keypoints (see
|
||||
constants.DEFAULT_EYE_EAR_LOCAL_OFFSETS) for whichever of those slots
|
||||
aren't already explicitly mapped, anchored at and oriented by the
|
||||
joint mapped to Nose (the head) -- so they turn with the head's actual
|
||||
rotation rather than facing the camera. No-op if Nose isn't mapped.
|
||||
"""
|
||||
missing_offsets = {
|
||||
index: offset
|
||||
for index, offset in constants.DEFAULT_EYE_EAR_LOCAL_OFFSETS.items()
|
||||
if index not in mapped
|
||||
}
|
||||
if not missing_offsets:
|
||||
return
|
||||
nose_index = constants.NAME_TO_INDEX["Body_Nose"]
|
||||
if nose_index not in mapped:
|
||||
return
|
||||
anchor = projector.world_position(mapped[nose_index])
|
||||
|
||||
face_scale = mapping_node.get_face_scale(dep_fn)
|
||||
computed = projector.default_head_relative_points(anchor, mapped[nose_index], face_scale, missing_offsets)
|
||||
for index, world_point in computed.items():
|
||||
px, py, in_front = projector.project_point(world_point, camera_path, width, height)
|
||||
keypoints[index] = (px, py, 1.0 if in_front else 0.0)
|
||||
|
||||
|
||||
def render_frame(camera_path, width, height, output_path):
|
||||
"""Project + rasterize every mapped openposeCharacter to a single PNG."""
|
||||
characters = []
|
||||
@@ -90,6 +116,9 @@ def render_frame(camera_path, width, height, output_path):
|
||||
mapped = mapping_node.get_mapped_joint_paths(dep_fn)
|
||||
keypoints = projector.project_character(mapped, camera_path, width, height)
|
||||
|
||||
if mapping_node.get_use_default_eyes_ears(dep_fn):
|
||||
_fill_default_eye_ear_keypoints(dep_fn, mapped, keypoints, camera_path, width, height)
|
||||
|
||||
if mapping_node.get_face_source(dep_fn) == mapping_node.FACE_SOURCE_ARKIT_BLENDSHAPES:
|
||||
nose_index = constants.NAME_TO_INDEX["Body_Nose"]
|
||||
nose_x, nose_y, nose_confidence = keypoints[nose_index]
|
||||
@@ -98,7 +127,15 @@ def render_frame(camera_path, width, height, output_path):
|
||||
weights = mapping_node.get_blendshape_weights(dep_fn)
|
||||
face_scale = mapping_node.get_face_scale(dep_fn)
|
||||
local_points = face_blendshapes.compute_face_local_points(weights)
|
||||
world_points = projector.billboard_world_points(anchor, camera_path, local_points, face_scale)
|
||||
|
||||
neck_index = constants.NAME_TO_INDEX["Body_Neck"]
|
||||
if mapping_node.get_face_follow(dep_fn) == mapping_node.FACE_FOLLOW_NECK and neck_index in mapped:
|
||||
world_points = projector.oriented_world_points(
|
||||
anchor, mapped[neck_index], local_points, face_scale
|
||||
)
|
||||
else:
|
||||
world_points = projector.billboard_world_points(anchor, camera_path, local_points, face_scale)
|
||||
|
||||
for offset, world_point in enumerate(world_points):
|
||||
px, py, in_front = projector.project_point(world_point, camera_path, width, height)
|
||||
keypoints[constants.FACE_START + offset] = (px, py, 1.0 if in_front else 0.0)
|
||||
|
||||
@@ -93,6 +93,34 @@ class MappingEditor(QtWidgets.QDialog):
|
||||
self.face_scale_spin.valueChanged.connect(self._on_face_scale_changed)
|
||||
face_row.addWidget(self.face_scale_spin)
|
||||
|
||||
face_row.addWidget(QtWidgets.QLabel("Face Follow:"))
|
||||
self.face_follow_combo = QtWidgets.QComboBox()
|
||||
self.face_follow_combo.addItems(
|
||||
[mapping_node.FACE_FOLLOW_CAMERA, mapping_node.FACE_FOLLOW_NECK]
|
||||
)
|
||||
self.face_follow_combo.setToolTip(
|
||||
"Camera: the ARKit-blendshape face always faces whichever camera is drawing\n"
|
||||
"(viewport or render camera) -- works regardless of rig conventions.\n"
|
||||
"Neck: the face plane rotates with the mapped Body_Neck joint's own local\n"
|
||||
"X/Y axes instead -- turns with the character's head, but is rig-dependent\n"
|
||||
"(assumes local X = right, local Y = up; adjust the neck joint's rotate axis\n"
|
||||
"if the face looks rotated). Falls back to Camera if no Neck is mapped."
|
||||
)
|
||||
self.face_follow_combo.currentIndexChanged.connect(self._on_face_follow_changed)
|
||||
face_row.addWidget(self.face_follow_combo)
|
||||
|
||||
self.default_eyes_ears_check = QtWidgets.QCheckBox("Default Eyes/Ears")
|
||||
self.default_eyes_ears_check.setToolTip(
|
||||
"Most rigs (including HumanIK) have no REye/LEye/REar/LEar joints. When on\n"
|
||||
"(default), those 4 OpenPose_full slots get a synthetic head-relative position\n"
|
||||
"instead of being invisible, whenever they're unmapped but Body_Nose is mapped\n"
|
||||
"-- anchored at and rotating with the joint mapped to Nose (sized by Face Scale\n"
|
||||
"above). An explicit joint mapped to any of the 4 slots always takes priority\n"
|
||||
"over this fallback."
|
||||
)
|
||||
self.default_eyes_ears_check.toggled.connect(self._on_use_default_eyes_ears_changed)
|
||||
face_row.addWidget(self.default_eyes_ears_check)
|
||||
|
||||
autoconnect_btn = QtWidgets.QPushButton("Auto-Connect ARKit Blendshapes from Selected")
|
||||
autoconnect_btn.setToolTip(
|
||||
"Connects the selected node's attributes to this character's 52 blendshapeWeight\n"
|
||||
@@ -181,13 +209,18 @@ class MappingEditor(QtWidgets.QDialog):
|
||||
self.body_model_combo.setEnabled(False)
|
||||
self.body_model_combo.blockSignals(False)
|
||||
|
||||
for widget in (self.face_source_combo, self.face_scale_spin):
|
||||
face_widgets = (
|
||||
self.face_source_combo, self.face_scale_spin, self.face_follow_combo, self.default_eyes_ears_check,
|
||||
)
|
||||
for widget in face_widgets:
|
||||
widget.blockSignals(True)
|
||||
if character:
|
||||
self.face_source_combo.setEnabled(True)
|
||||
self.face_scale_spin.setEnabled(True)
|
||||
for widget in face_widgets:
|
||||
widget.setEnabled(True)
|
||||
self.face_source_combo.setCurrentIndex(cmds.getAttr(character + ".faceSource"))
|
||||
self.face_scale_spin.setValue(cmds.getAttr(character + ".faceScale"))
|
||||
self.face_follow_combo.setCurrentIndex(cmds.getAttr(character + ".faceFollow"))
|
||||
self.default_eyes_ears_check.setChecked(cmds.getAttr(character + ".useDefaultEyesEars"))
|
||||
connected = sum(
|
||||
1 for i in range(face_blendshapes.BLENDSHAPE_COUNT)
|
||||
if cmds.listConnections("{0}.blendshapeWeight[{1}]".format(character, i), source=True, destination=False)
|
||||
@@ -196,10 +229,10 @@ class MappingEditor(QtWidgets.QDialog):
|
||||
"{0} / {1} blendshapes connected".format(connected, face_blendshapes.BLENDSHAPE_COUNT)
|
||||
)
|
||||
else:
|
||||
self.face_source_combo.setEnabled(False)
|
||||
self.face_scale_spin.setEnabled(False)
|
||||
for widget in face_widgets:
|
||||
widget.setEnabled(False)
|
||||
self.blendshape_status_label.setText("")
|
||||
for widget in (self.face_source_combo, self.face_scale_spin):
|
||||
for widget in face_widgets:
|
||||
widget.blockSignals(False)
|
||||
|
||||
def _on_body_model_changed(self, index):
|
||||
@@ -212,11 +245,21 @@ class MappingEditor(QtWidgets.QDialog):
|
||||
if character:
|
||||
cmds.setAttr(character + ".faceSource", index)
|
||||
|
||||
def _on_face_follow_changed(self, index):
|
||||
character = self._current_character()
|
||||
if character:
|
||||
cmds.setAttr(character + ".faceFollow", index)
|
||||
|
||||
def _on_face_scale_changed(self, value):
|
||||
character = self._current_character()
|
||||
if character:
|
||||
cmds.setAttr(character + ".faceScale", value)
|
||||
|
||||
def _on_use_default_eyes_ears_changed(self, checked):
|
||||
character = self._current_character()
|
||||
if character:
|
||||
cmds.setAttr(character + ".useDefaultEyesEars", checked)
|
||||
|
||||
def _on_autoconnect_blendshapes(self):
|
||||
character = self._current_character()
|
||||
if not character:
|
||||
|
||||
Reference in New Issue
Block a user