diff --git a/src/Video.tsx b/src/Video.tsx index 7aeeaf9..4041e09 100644 --- a/src/Video.tsx +++ b/src/Video.tsx @@ -1,8 +1,13 @@ import { useEffect, useRef, useState } from "react" -import { FilesetResolver, NormalizedLandmark, HolisticLandmarker } from "@mediapipe/tasks-vision" +import { + FilesetResolver, + NormalizedLandmark, + HolisticLandmarker, + HolisticLandmarkerResult, +} from "@mediapipe/tasks-vision" import { FormControlLabel, IconButton, Switch, Tooltip } from "@mui/material" -import { Videocam, CloudUpload } from "@mui/icons-material" +import { Videocam, CloudUpload, Replay } from "@mui/icons-material" import { styled } from "@mui/material/styles" const defaultVideoSrc = "./video/flash.mp4" @@ -30,7 +35,6 @@ function Video({ }): JSX.Element { const videoRef = useRef(null) const imgRef = useRef(null) - const canvasRef = useRef(null) const [videoSrc, setVideoSrc] = useState(defaultVideoSrc) const [imgSrc, setImgSrc] = useState("") const [isCameraActive, setIsCameraActive] = useState(false) @@ -39,6 +43,8 @@ function Video({ const holisticLandmarkerRef = useRef(null) const [lastMedia, setLastMedia] = useState("VIDEO") + const landmarkHistoryRef = useRef([]) + const handleFileUpload = (event: React.ChangeEvent) => { const file = event.target.files?.[0] if (file) { @@ -76,6 +82,7 @@ function Video({ } const toggleCamera = async () => { + landmarkHistoryRef.current = [] if (isCameraActive) { if (videoRef.current && videoRef.current.srcObject) { const tracks = (videoRef.current.srcObject as MediaStream).getTracks() @@ -127,6 +134,10 @@ function Video({ if (videoRef.current && lastTime != videoRef.current.currentTime && videoRef.current.videoWidth > 0) { lastTime = videoRef.current.currentTime holisticLandmarkerRef.current!.detectForVideo(videoRef.current, performance.now(), (result) => { + if (!isCameraActive) { + landmarkHistoryRef.current.push(result) + } + if (isPoseDetectionEnabled.current && result.poseWorldLandmarks[0]) { setPose(result.poseWorldLandmarks[0]) } else { @@ -161,45 +172,55 @@ function Video({ detect() } ) - }, [setPose, setFace, imgRef, videoRef]) + }, [setPose, setFace, imgRef, videoRef, isCameraActive]) - useEffect(() => { - const resizeCanvas = () => { - if (videoRef.current && canvasRef.current) { - const videoWidth = videoRef.current.videoWidth - const videoHeight = videoRef.current.videoHeight - const containerWidth = videoRef.current.clientWidth - const containerHeight = videoRef.current.clientHeight + const replayCallback = () => { + let currentIndex = 0 + const frameInterval = 1000 / 24 // 24 FPS + + const playNextFrame = () => { + if (currentIndex < landmarkHistoryRef.current.length) { + const result = landmarkHistoryRef.current[currentIndex] - const scale = Math.min(containerWidth / videoWidth, containerHeight / videoHeight) - const scaledWidth = videoWidth * scale - const scaledHeight = videoHeight * scale + if (isPoseDetectionEnabled.current && result.poseWorldLandmarks && result.poseWorldLandmarks[0]) { + setPose(result.poseWorldLandmarks[0]) + } else { + setPose([]) + } + + if (isFaceDetectionEnabled.current && result.faceLandmarks && result.faceLandmarks.length > 0) { + setFace(result.faceLandmarks[0]) + } else { + setFace([]) + } - canvasRef.current.width = scaledWidth - canvasRef.current.height = scaledHeight - canvasRef.current.style.left = `${(containerWidth - scaledWidth) / 2}px` - canvasRef.current.style.top = `${(containerHeight - scaledHeight) / 2}px` + currentIndex++ + setTimeout(() => requestAnimationFrame(playNextFrame), frameInterval) } - if (imgRef.current && imgRef.current.complete && canvasRef.current) { - const containerWidth = imgRef.current.clientWidth - const containerHeight = imgRef.current.clientHeight - console.log("containerWidth", containerWidth) - canvasRef.current.width = containerWidth - canvasRef.current.height = containerHeight - canvasRef.current.style.width = `${containerWidth}px` - canvasRef.current.style.height = `${containerHeight}px` - canvasRef.current.style.left = "0" - canvasRef.current.style.top = "0" + } + + requestAnimationFrame(playNextFrame) + } + + useEffect(() => { + const clearLandmarkHistory = () => { + landmarkHistoryRef.current = [] + } + + const videoElement = videoRef.current + if (videoElement) { + videoElement.addEventListener("play", clearLandmarkHistory) + return () => { + videoElement.removeEventListener("play", clearLandmarkHistory) } } - resizeCanvas() - }, [videoSrc, imgSrc]) + }, []) return ( <>
- + - + + + + +