Merge pull request #7 from readyplayerme/feature/emotions-control

Feature/emotions control

src/App/App.component.tsx

@@ -7,8 +7,8 @@ function App() {
     <div className="App">
       <div>localhost playground</div>
       <div className="container">
-        <div className="card">
-          <Avatar modelUrl="/male.glb" animationUrl="/maleIdle.glb" backgroundColor="#fafafa" shadows />
+        <div className="card" style={{ width: '100%' }}>
+          <Avatar modelUrl="/male.glb" poseUrl="/male-pose-standing.glb" backgroundColor="#fafafa" shadows />
         </div>
       </div>
     </div>

src/components/Avatar/Avatar.component.tsx

@@ -4,7 +4,7 @@ import { PresetsType } from '@react-three/drei/helpers/environment-assets';
 import { Vector3 } from 'three';
 import { CameraLighting } from 'src/components/SceneControls/CameraLighting.component';
 import { AnimationModel } from 'src/components/Models/AnimationModel/AnimationModel.component';
-import { LightingProps } from 'src/types';
+import { Emotion, LightingProps } from 'src/types';
 import { BaseCanvas } from 'src/components/BaseCanvas';
 import { HalfBodyModel, StaticModel, PoseModel } from 'src/components/Models';
 import { isValidGlbUrl } from 'src/services';
@@ -82,6 +82,10 @@ export interface AvatarProps extends LightingProps {
    * Applies an idle rotation to the animated and half-body models.
    */
   idleRotation?: boolean;
+  /**
+   * Applies a face emotion of the model.
+   */
+  emotion?: Emotion;
 }
 
 /**
@@ -107,6 +111,7 @@ export const Avatar: FC<AvatarProps> = ({
   cameraTarget = CAMERA.TARGET.FULL_BODY,
   cameraInitialDistance = CAMERA.INITIAL_DISTANCE.FULL_BODY,
   style,
+  emotion,
   idleRotation = false
 }) => {
   const AvatarModel = useMemo(() => {
@@ -121,15 +126,15 @@ export const Avatar: FC<AvatarProps> = ({
     }
 
     if (halfBody) {
-      return <HalfBodyModel modelUrl={modelUrl} scale={scale} idleRotation={idleRotation} />;
+      return <HalfBodyModel emotion={emotion} modelUrl={modelUrl} scale={scale} idleRotation={idleRotation} />;
     }
 
     if (isValidGlbUrl(poseUrl)) {
-      return <PoseModel modelUrl={modelUrl} scale={scale} poseUrl={poseUrl!} />;
+      return <PoseModel emotion={emotion} modelUrl={modelUrl} scale={scale} poseUrl={poseUrl!} />;
     }
 
     return <StaticModel modelUrl={modelUrl} scale={scale} />;
-  }, [halfBody, animationUrl, modelUrl, scale, poseUrl, idleRotation]);
+  }, [halfBody, animationUrl, modelUrl, scale, poseUrl, idleRotation, emotion]);
 
   return (
     <BaseCanvas background={backgroundColor} position={new Vector3(0, 0, 3)} fov={50} style={style}>

src/components/Avatar/Avatar.stories.tsx

@@ -23,7 +23,8 @@ Static.args = {
   spotLightColor: '#fff5b6',
   spotLightAngle: 0.314,
   cameraTarget: CAMERA.TARGET.FULL_BODY,
-  cameraInitialDistance: CAMERA.CONTROLS.FULL_BODY.MAX_DISTANCE
+  cameraInitialDistance: CAMERA.CONTROLS.FULL_BODY.MAX_DISTANCE,
+  emotion: 'idle'
 };
 
 export default {
@@ -68,3 +69,13 @@ Posing.args = {
   cameraTarget: CAMERA.TARGET.FULL_BODY,
   cameraInitialDistance: CAMERA.CONTROLS.FULL_BODY.MAX_DISTANCE
 };
+
+export const Emotions = Template.bind({});
+Emotions.args = {
+  ...Static.args,
+  modelUrl: getStoryAssetPath('male.glb'),
+  poseUrl: getStoryAssetPath('male-pose-standing.glb'),
+  emotion: 'angry',
+  cameraTarget: CAMERA.TARGET.FULL_BODY,
+  cameraInitialDistance: CAMERA.CONTROLS.FULL_BODY.MAX_DISTANCE
+};

src/components/Models/HalfBodyModel/HalfBodyModel.component.tsx

@@ -2,14 +2,16 @@ import React, { FC, useRef } from 'react';
 import { useFrame, useGraph, useLoader } from '@react-three/fiber';
 import { GLTFLoader } from 'three-stdlib';
 import { Model } from 'src/components/Models/Model';
-import { useHeadMovement } from 'src/services';
+import { useEmotion, useHeadMovement } from 'src/services';
 import { Group } from 'three';
+import { Emotion } from '../../../types';
 
 interface HalfBodyModelProps {
   modelUrl: string;
   rotation?: number;
   scale?: number;
   idleRotation?: boolean;
+  emotion?: Emotion;
 }
 
 let currentRotation = 0;
@@ -18,7 +20,8 @@ export const HalfBodyModel: FC<HalfBodyModelProps> = ({
   modelUrl,
   scale = 1,
   rotation = 20 * (Math.PI / 180),
-  idleRotation = false
+  idleRotation = false,
+  emotion = 'idle'
 }) => {
   const ref = useRef<Group>();
   const { scene } = useLoader(GLTFLoader, modelUrl);
@@ -51,6 +54,7 @@ export const HalfBodyModel: FC<HalfBodyModelProps> = ({
   });
 
   useHeadMovement(nodes, true);
+  useEmotion(nodes, emotion);
 
   return <Model modelRef={ref} scene={scene} scale={scale} />;
 };

src/components/Models/PoseModel/PoseModel.component.tsx

@@ -3,22 +3,25 @@ import { useGraph, useLoader } from '@react-three/fiber';
 import { GLTFLoader } from 'three-stdlib';
 import { Model } from 'src/components/Models/Model';
 import { Group } from 'three';
-import { mutatePose } from 'src/services';
+import { mutatePose, useEmotion } from 'src/services';
+import { Emotion } from '../../../types';
 
 interface PoseModelProps {
   modelUrl: string;
   poseUrl: string;
   modelRef?: MutableRefObject<Group | undefined>;
   scale?: number;
+  emotion?: Emotion;
 }
 
-export const PoseModel: FC<PoseModelProps> = ({ modelUrl, poseUrl, modelRef, scale = 1 }) => {
+export const PoseModel: FC<PoseModelProps> = ({ modelUrl, poseUrl, modelRef, scale = 1, emotion = 'happy' }) => {
   const { scene } = useLoader(GLTFLoader, modelUrl);
   const { nodes } = useGraph(scene);
   const pose = useLoader(GLTFLoader, poseUrl);
   const { nodes: sourceNodes } = useGraph(pose.scene);
 
   mutatePose(nodes, sourceNodes);
+  useEmotion(nodes, emotion);
 
   return <Model modelRef={modelRef} scene={scene} scale={scale} />;
 };

src/services/Models.service.ts

@@ -1,6 +1,8 @@
-import { LinearFilter, MeshStandardMaterial, Material, Vector2, Object3D } from 'three';
+import { LinearFilter, MeshStandardMaterial, Material, Vector2, Object3D, SkinnedMesh } from 'three';
 import { useFrame } from '@react-three/fiber';
 import type { ObjectMap } from '@react-three/fiber';
+import { useMemo } from 'react';
+import { Emotions, Emotion } from '../types';
 
 export const getStoryAssetPath = (publicAsset: string) =>
   `${process.env.NODE_ENV === 'production' ? '/visage' : ''}/${publicAsset}`;
@@ -111,3 +113,75 @@ export const mutatePose = (targetNodes?: ObjectMap['nodes'], sourceNodes?: Objec
     });
   }
 };
+
+const emotions: Emotions = {
+  idle: {},
+  impressed: {
+    mouthOpen: 0.7,
+    mouthSmile: 0.3,
+    mouthDimpleLeft: 0.4,
+    mouthDimpleRight: 0.4,
+    eyeWideLeft: 0.75,
+    eyeWideRight: 0.75,
+    browInnerUp: 0.3
+  },
+  sad: {
+    mouthSmile: -0.35,
+    browDownLeft: -0.45,
+    browDownRight: -0.45,
+    noseSneerLeft: -0.35,
+    noseSneerRight: -0.35
+  },
+  angry: {
+    browDownLeft: 0.95,
+    browDownRight: 0.95,
+    mouthDimpleLeft: -0.5,
+    mouthDimpleRight: -0.5,
+    mouthLowerDownRight: -0.3,
+    mouthLowerDownLeft: -0.3,
+    noseSneerLeft: 0.35,
+    noseSneerRight: 0.35,
+    eyeBlinkLeft: 0.15,
+    eyeBlinkRight: 0.15
+  },
+  happy: {
+    mouthSmile: 0.6,
+    mouthOpen: 0.3,
+    browDownRight: -0.5,
+    browDownLeft: -0.5,
+    mouthDimpleLeft: 0.7,
+    mouthDimpleRight: 0.7,
+    noseSneerLeft: 0.45,
+    noseSneerRight: 0.45
+  }
+};
+
+export const useEmotion = (nodes: ObjectMap['nodes'], emotion: Emotion) => {
+  const headMesh = (nodes.Wolf3D_Head || nodes.Wolf3D_Avatar) as SkinnedMesh;
+  const selectedEmotion = useMemo(() => emotions[emotion], [emotion]);
+
+  const resetEmotions = () =>
+    headMesh?.morphTargetInfluences?.forEach((_, index) => {
+      headMesh!.morphTargetInfluences![index] = 0;
+    });
+
+  useFrame(() => {
+    if (!headMesh) {
+      return;
+    }
+
+    if (emotion !== 'idle') {
+      resetEmotions();
+
+      Object.entries(selectedEmotion).forEach(([shape, value]) => {
+        const shapeId = headMesh!.morphTargetDictionary?.[shape];
+
+        if (shapeId) {
+          headMesh!.morphTargetInfluences![shapeId] = value;
+        }
+      });
+    } else {
+      resetEmotions();
+    }
+  });
+};

src/types.ts

@@ -20,3 +20,76 @@ export interface LightingProps {
   spotLightColor?: string;
   spotLightAngle?: number;
 }
+
+export type Emotion = 'idle' | 'sad' | 'angry' | 'happy' | 'impressed';
+
+export type HeadBlendShapeType =
+  | 'browDownLeft'
+  | 'browDownRight'
+  | 'browInnerUp'
+  | 'browOuterUpLeft'
+  | 'browOuterUpRight'
+  | 'cheekPuff'
+  | 'cheekSquintLeft'
+  | 'cheekSquintRight'
+  | 'eyeBlinkLeft'
+  | 'eyeBlinkRight'
+  | 'eyeSquintLeft'
+  | 'eyeSquintRight'
+  | 'eyeWideLeft'
+  | 'eyeWideRight'
+  | 'eyesClosed'
+  | 'eyesLookDown'
+  | 'eyesLookUp'
+  | 'jawForward'
+  | 'jawLeft'
+  | 'jawOpen'
+  | 'jawRight'
+  | 'mouthClose'
+  | 'mouthDimpleLeft'
+  | 'mouthDimpleRight'
+  | 'mouthFrownLeft'
+  | 'mouthFrownRight'
+  | 'mouthFunnel'
+  | 'mouthLeft'
+  | 'mouthLowerDownLeft'
+  | 'mouthLowerDownRight'
+  | 'mouthOpen'
+  | 'mouthPressLeft'
+  | 'mouthPressRight'
+  | 'mouthPucker'
+  | 'mouthRight'
+  | 'mouthRollLower'
+  | 'mouthRollUpper'
+  | 'mouthShrugLower'
+  | 'mouthShrugUpper'
+  | 'mouthSmile'
+  | 'mouthSmileLeft'
+  | 'mouthSmileRight'
+  | 'mouthStretchLeft'
+  | 'mouthStretchRight'
+  | 'mouthUpperUpLeft'
+  | 'mouthUpperUpRight'
+  | 'noseSneerLeft'
+  | 'noseSneerRight'
+  | 'viseme_CH'
+  | 'viseme_DD'
+  | 'viseme_E'
+  | 'viseme_FF'
+  | 'viseme_I'
+  | 'viseme_O'
+  | 'viseme_PP'
+  | 'viseme_RR'
+  | 'viseme_SS'
+  | 'viseme_TH'
+  | 'viseme_U'
+  | 'viseme_aa'
+  | 'viseme_kk'
+  | 'viseme_nn'
+  | 'viseme_sil';
+
+export type EmotionConfig = Record<HeadBlendShapeType, number>;
+
+export type Emotions = {
+  [name in Emotion]: Partial<EmotionConfig>;
+};