Skip to content

Commit

Permalink
Merge pull request #6 from RokokoElectronics/f/RSMLY-30-31-32
Browse files Browse the repository at this point in the history 
F/rsmly 30 31 32 34 35 36
  • Loading branch information
karavias authored Mar 10, 2020
2 parents 9c2019b + dc85cbc commit 00acaf7
Show file tree
Hide file tree
Showing 20 changed files with 931 additions and 111 deletions.
2 changes: 2 additions & 0 deletions resources.qrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,7 @@
<file>resources/createHIKForActor.mel</file>
<file>resources/setOrCreateRSIdAttribute.mel</file>
<file>resources/deleteMappingAttribute.mel</file>
<file>resources/createFaceAttributes.mel</file>
<file>resources/fetchBlendShapes.mel</file>
</qresource>
</RCC>
87 changes: 87 additions & 0 deletions resources/createFaceAttributes.mel
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
string $bsHostNode = "MAYA_OBJECT_NAME";


proc _createFaceAttrs(string $bsNode) {
if(!`objExists $bsNode`) {
print("Node not found" + $bsNode + "\n");
return;
}

string $faceMappingName = "FACE_MAPPING_FILED_NAME";
int $faceAttrExists = `attributeExists $faceMappingName $bsNode`;

if ($faceAttrExists) {
// already been created
print("Face mapping already been created!\n");
return;
} else {
// create compound attribute with 53 child attributes

addAttr -longName $faceMappingName -numberOfChildren 53 -attributeType compound $bsNode;
addAttr -dt "string" -longName "FaceId" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeBlinkLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeLookDownLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeLookInLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeLookOutLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeLookUpLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeSquintLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeWideLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeBlinkRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeLookDownRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeLookInRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeLookOutRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeLookUpRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeSquintRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "eyeWideRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "jawForward" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "jawLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "jawRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "jawOpen" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthClose" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthFunnel" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthPucker" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthSmileLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthSmileRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthFrownLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthFrownRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthDimpleLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthDimpleRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthStretchLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthStretchRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthRollLower" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthRollUpper" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthShrugLower" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthShrugUpper" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthPressLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthPressRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthLowerDownLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthLowerDownRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthUpperUpLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "mouthUpperUpRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "browDownLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "browDownRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "browInnerUp" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "browOuterUpLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "browOuterUpRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "cheekPuff" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "cheekSquintLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "cheekSquintRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "noseSneerLeft" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "noseSneerRight" -parent $faceMappingName $bsNode;
addAttr -dt "string" -longName "tongueOut" -parent $faceMappingName $bsNode;
}
}

proc createFaceMappingOnNode( string $targetNode ) {
string $connections[] = `listHistory -bf $targetNode`;
string $bsNodes[] = `ls -type "blendShape" $connections`;

for ($bsNode in $bsNodes) {
// create face attributes for each BS node
_createFaceAttrs($bsNode);
}
}

createFaceMappingOnNode($bsHostNode);
10 changes: 10 additions & 0 deletions resources/fetchBlendShapes.mel
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
proc string getConnectedBlendShapes(string $targetNode, string $separator) {
if(!`objExists $targetNode`) {
return "";
}
string $connections[] = `listHistory -bf $targetNode`;
string $bsNodes[] = `ls -type "blendShape" $connections`;

return stringArrayToString($bsNodes, $separator);
}
getConnectedBlendShapes("MAYA_OBJECT_NAME", "SEPARATOR");
2 changes: 2 additions & 0 deletions rslm.pro
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ SOURCES += \
src/main.cpp \
src/mapping.cpp \
src/receiverworker.cpp \
src/recorder.cpp \
src/ui/button.cpp \
src/ui/categoryheader.cpp \
src/ui/commandapicontent.cpp \
Expand All @@ -47,6 +48,7 @@ HEADERS += \
src/constants.h \
src/mapping.h \
src/receiverworker.h \
src/recorder.h \
src/singleton.h \
src/ui/button.h \
src/ui/categoryheader.h \
Expand Down
198 changes: 148 additions & 50 deletions src/animations.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,25 @@
#include <iostream>

#include "animations.h"
#include "recorder.h"
#include "utils.h"
#include "mapping.h"

#include <QJsonObject>
#include <QJsonValue>

#include <maya/MObject.h>
#include <maya/MSelectionList.h>
#include <maya/MEulerRotation.h>
#include <maya/MObjectArray.h>
#include <maya/MFnTransform.h>
#include <maya/MFnIkJoint.h>
#include <maya/MFnBlendShapeDeformer.h>
#include <maya/MDagPath.h>
#include <maya/MItDag.h>
#include <maya/MPlug.h>
#include <maya/MMatrix.h>

#ifdef _WINDOWS
#pragma comment(lib,"OpenMayaAnim.lib")
#endif


_Animations::_Animations()
{
Expand Down Expand Up @@ -70,59 +72,78 @@ void _Animations::applyAnimationsToMappedObjects()
const QHash<QString, QString> mayaToRsBoneNames = Mapping::get()->getBoneMapping();
const QHash<QString, MQuaternion> studioTPose = Mapping::get()->getStudioTPose();

struct Local {
static void animatePropOrTracker(QJsonObject obj, MDagPath dagPath) {
// bool isLive = obj["isLive"].toBool();

// do nothing is not live
// if(!isLive) return;

QJsonObject postitionObject = obj["position"].toObject();
QJsonObject rotationObject = obj["rotation"].toObject();
MVector rsPosition(postitionObject["x"].toDouble(),
postitionObject["y"].toDouble(),
postitionObject["z"].toDouble());

MQuaternion rsRotation(rotationObject["x"].toDouble(),
rotationObject["y"].toDouble(),
rotationObject["z"].toDouble(),
rotationObject["w"].toDouble());

// convert RS transform to Maya transform
MVector mayaPosition = Utils::rsToMaya(rsPosition) * Animations::get()->sceneScale();
MQuaternion mayaRotation = Utils::rsToMaya(rsRotation);
MTransformationMatrix finalTransform(MTransformationMatrix::identity);
finalTransform.setTranslation(mayaPosition, MSpace::kWorld);
finalTransform.setRotationQuaternion(mayaRotation.x, mayaRotation.y, mayaRotation.z, mayaRotation.w);

// apply converted transform onto mapped object
MFnTransform fn(dagPath);
fn.set(finalTransform);
}
};

QList<QString> allIds = objectMapping.keys();
for(QString rsId : allIds) {
auto it = objectMapping.find(rsId);
// std::cout << "cnt: " << objectMapping.count(rsId) << std::endl;
if(it != objectMapping.end()) {
while(it != objectMapping.end()) {
while(it != objectMapping.end())
{
QString rsId = it.key();

if(rsId.isEmpty()) {
++it;
continue;
}

MObject object = it.value();
MDagPath dagPath;
MDagPath::getAPathTo(object, dagPath);

// apply props animations
if(propsMap.contains(rsId)) {
// apply props animations

QJsonObject propObject = propsMap[rsId];
Local::animatePropOrTracker(propObject, dagPath);
// apply trackers animations
animatePropOrTracker(propObject, dagPath);
// printf("prop animated %s - %s\n", dagPath.partialPathName().asChar(), rsId.toStdString().c_str());

} else if(trackersMap.contains(rsId)) {

// apply trackers animations
QJsonObject trackerObject = trackersMap[rsId];
Local::animatePropOrTracker(trackerObject, dagPath);
// apply faces animations
animatePropOrTracker(trackerObject, dagPath);

} else if(facesMap.contains(rsId)) {
// apply actor animations
// apply face animations

const QStringList faceShapeNames = Mapping::get()->getFaceShapeNames();
// get input data for this face
QJsonObject faceObject = facesMap[rsId];

// access mapped blendshape node
MFnBlendShapeDeformer faceFn(object);

// TODO: run this once when receiver stared
// prepare weight weight name to attribute plug map
QHash <QString, MPlug> weightsMap;
Utils::fillFaceWeightsMap(faceFn, weightsMap);

// for each studio shape name set weight value
for(QString studioShapeName : faceShapeNames) {
double weight = faceObject[studioShapeName].toDouble() * 0.01;

MString bsFieldName = BLEND_SHAPE_PREFIX + studioShapeName.toStdString().c_str();
MStatus fieldPlugStatus;
MPlug weightAttr = faceFn.findPlug(bsFieldName, true, &fieldPlugStatus);
if(fieldPlugStatus == MStatus::kSuccess) {
MString mappedName;
weightAttr.getValue(mappedName);
if(weightsMap.contains(mappedName.asChar())) {
MPlug weightPlug = weightsMap[mappedName.asChar()];
weightPlug.setDouble(weight);
if(recordingEnabled) {

// this functions will be called later
Recorder::get()->recordFace(timestamp, [object, weight, weightPlug](int frame) {
Recorder::get()->keyframeNumericAttribute(frame, weightPlug, weight);
});
}
}
}
}

} else if(actorsMap.contains(rsId)) {
// apply actor animations
QJsonObject actorObject = actorsMap[rsId];

// Hips joint mapped implicitly for user
Expand All @@ -131,7 +152,7 @@ void _Animations::applyAnimationsToMappedObjects()
// We transform HIK source skeleton. To see results on custom character user should create
// Another character and define it, than set it as target for our generated character.

MItDag it;
MItDag jIt;
MFnDagNode hipDagNode(dagPath);
MVector referenceOffset;
MQuaternion referenceQuat;
Expand All @@ -145,10 +166,10 @@ void _Animations::applyAnimationsToMappedObjects()
referenceQuat.normalizeIt();
}

it.reset(dagPath, MItDag::kBreadthFirst, MFn::kJoint);
while(!it.isDone()) {
jIt.reset(dagPath, MItDag::kBreadthFirst, MFn::kJoint);
while(!jIt.isDone()) {
MDagPath jointPath;
it.getPath(jointPath);
jIt.getPath(jointPath);

QString jointPathString(jointPath.fullPathName().asChar());
// this name contains character name CHARNAME_BONENAME
Expand Down Expand Up @@ -188,24 +209,101 @@ void _Animations::applyAnimationsToMappedObjects()
boneQuat *= referenceQuat;
boneQuat.normalizeIt();
fnTr.setRotation(boneQuat, MSpace::kWorld);

if(recordingEnabled) {
MVector recordLocation = fnTr.getTranslation(MSpace::kTransform);
MQuaternion recordRotation;
fnTr.getRotation(recordRotation);
Recorder::get()->recordBone(timestamp, [recordLocation, jointPath, recordRotation](int frame) {
Recorder::get()->keyframeNumericAttribute("tx", frame, jointPath, recordLocation.x);
Recorder::get()->keyframeNumericAttribute("ty", frame, jointPath, recordLocation.y);
Recorder::get()->keyframeNumericAttribute("tz", frame, jointPath, recordLocation.z);

MEulerRotation euAngle = recordRotation.asEulerRotation();
Recorder::get()->keyframeNumericAttribute("rx", frame, jointPath, euAngle.x);
Recorder::get()->keyframeNumericAttribute("ry", frame, jointPath, euAngle.y);
Recorder::get()->keyframeNumericAttribute("rz", frame, jointPath, euAngle.z);
});
}
}

it.next();
jIt.next();
}

} else {
// this should never happen
std::cout << "MAPPED OBJECTS NOT FOUND IN DATA STREAM!!!!\n";
}
// this block executes only if rsId is not found in mapped objects
// for example user opened maya scene that doesn't corresponds to opened studio project
// TODO: tell user about problematic objects

// std::cout << "MAPPED OBJECTS NOT FOUND IN DATA STREAM!!!!\n";
}

++it;
}
}
}
}

void _Animations::recordingToggled(bool enabled)
{
recordingEnabled = enabled;
Recorder::get()->recordingToggled(enabled);
if(!recordingEnabled) {
// put cached data into timeline
Recorder::get()->finalizeRecording();
}

}

void _Animations::setSceneScale(float scale)
{
_sceneScale = scale;
}

void _Animations::animatePropOrTracker(QJsonObject obj, const MDagPath &dagPath)
{
// bool isLive = obj["isLive"].toBool();

// do nothing if not live
// if(!isLive) return;

QJsonObject postitionObject = obj["position"].toObject();
QJsonObject rotationObject = obj["rotation"].toObject();
MVector rsPosition(postitionObject["x"].toDouble(),
postitionObject["y"].toDouble(),
postitionObject["z"].toDouble());

MQuaternion rsRotation(rotationObject["x"].toDouble(),
rotationObject["y"].toDouble(),
rotationObject["z"].toDouble(),
rotationObject["w"].toDouble());

// convert RS transform to Maya transform
MVector mayaPosition = Utils::rsToMaya(rsPosition) * Animations::get()->sceneScale();
MQuaternion mayaRotation = Utils::rsToMaya(rsRotation);
MTransformationMatrix finalTransform(MTransformationMatrix::identity);
finalTransform.setTranslation(mayaPosition, MSpace::kWorld);
finalTransform.setRotationQuaternion(mayaRotation.x, mayaRotation.y, mayaRotation.z, mayaRotation.w);

// apply converted transform onto mapped object
MFnTransform fn(dagPath);
fn.set(finalTransform);

if(recordingEnabled) {
Recorder::get()->recordPropOrTracker(timestamp, [mayaPosition, mayaRotation, dagPath](int frame){
Recorder::get()->keyframeNumericAttribute("tx", frame, dagPath, mayaPosition.x);
Recorder::get()->keyframeNumericAttribute("ty", frame, dagPath, mayaPosition.y);
Recorder::get()->keyframeNumericAttribute("tz", frame, dagPath, mayaPosition.z);

MEulerRotation euAngle = mayaRotation.asEulerRotation();
Recorder::get()->keyframeNumericAttribute("rx", frame, dagPath, euAngle.x);
Recorder::get()->keyframeNumericAttribute("ry", frame, dagPath, euAngle.y);
Recorder::get()->keyframeNumericAttribute("rz", frame, dagPath, euAngle.z);
});
}
}





Loading

0 comments on commit 00acaf7

Please sign in to comment.