diff --git a/src/App.tsx b/src/App.tsx index df8c36c..ebeab69 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -10,6 +10,8 @@ import { Download } from "@mui/icons-material" function App(): JSX.Element { const [pose, setPose] = useState(null) const [face, setFace] = useState(null) + const [leftHand, setLeftHand] = useState(null) + const [rightHand, setRightHand] = useState(null) const [lerpFactor, setLerpFactor] = useState(0.5) const [fps, setFps] = useState(0) @@ -38,8 +40,21 @@ function App(): JSX.Element { - - + + ) } diff --git a/src/MMDScene.tsx b/src/MMDScene.tsx index aba14d5..2c20fb5 100644 --- a/src/MMDScene.tsx +++ b/src/MMDScene.tsx @@ -31,11 +31,15 @@ const availableModels = ["深空之眼-托特", "深空之眼-托特2", "鸣潮- function MMDScene({ pose, face, + leftHand, + rightHand, lerpFactor, setFps, }: { pose: NormalizedLandmark[] | null face: NormalizedLandmark[] | null + leftHand: NormalizedLandmark[] | null + rightHand: NormalizedLandmark[] | null lerpFactor: number setFps: (fps: number) => void }): JSX.Element { @@ -656,6 +660,8 @@ function MMDScene({ updateMMDFace(mmdModelRef.current, face) } }, [face]) + + useEffect(() => {}, [leftHand, rightHand]) return ( <> diff --git a/src/Video.tsx b/src/Video.tsx index d9ef7a5..9f33277 100644 --- a/src/Video.tsx +++ b/src/Video.tsx @@ -39,10 +39,14 @@ const VisuallyHiddenInput = styled("input")({ function Video({ setPose, setFace, + setLeftHand, + setRightHand, setLerpFactor, }: { setPose: (pose: NormalizedLandmark[]) => void setFace: (face: NormalizedLandmark[]) => void + setLeftHand: (leftHand: NormalizedLandmark[]) => void + setRightHand: (rightHand: NormalizedLandmark[]) => void setLerpFactor: (lerpFactor: number) => void }): JSX.Element { const videoRef = useRef(null) @@ -216,6 +220,16 @@ function Video({ } else { setFace([]) } + if (result.leftHandWorldLandmarks && result.leftHandWorldLandmarks.length > 0) { + setLeftHand(result.leftHandWorldLandmarks[0]) + } else { + setLeftHand([]) + } + if (result.rightHandWorldLandmarks && result.rightHandWorldLandmarks.length > 0) { + setRightHand(result.rightHandWorldLandmarks[0]) + } else { + setRightHand([]) + } }) } else if ( imgRef.current && @@ -225,14 +239,24 @@ function Video({ ) { lastImgSrc = imgRef.current.src holisticLandmarkerRef.current!.detect(imgRef.current, (result) => { - if (result.poseLandmarks[0]) { - setPose(result.poseLandmarks[0]) + if (result.poseWorldLandmarks[0]) { + setPose(result.poseWorldLandmarks[0]) } else { setPose([]) } if (result.faceLandmarks && result.faceLandmarks.length > 0) { setFace(result.faceLandmarks[0]) } + if (result.leftHandWorldLandmarks && result.leftHandWorldLandmarks.length > 0) { + setLeftHand(result.leftHandWorldLandmarks[0]) + } else { + setLeftHand([]) + } + if (result.rightHandWorldLandmarks && result.rightHandWorldLandmarks.length > 0) { + setRightHand(result.rightHandWorldLandmarks[0]) + } else { + setRightHand([]) + } }) } requestAnimationFrame(detect) @@ -240,7 +264,7 @@ function Video({ detect() } ) - }, [setPose, setFace, imgRef, videoRef, isRecordingRef]) + }, [setPose, setFace, setLeftHand, setRightHand, imgRef, videoRef, isRecordingRef]) const replayCallback = (fps: number) => { setIsReplaying(true)