From 442abfcc5610389f0692413d23b70f83cc1e37ed Mon Sep 17 00:00:00 2001 From: James Harvey Date: Mon, 2 Dec 2024 11:23:10 +0000 Subject: [PATCH] fix(memory): ACT-1185 Visage memory optimisations (#104) * fix(memory): ACT-1185 dispose gltf on fallback * fix(memory): ACT-1185 dispose after build * refactor(memory): ACT-1185 memory leak rework * refactor(memory): ACT-1185 gltf loader refactor --- src/App/components/Nova.tsx | 33 +-- .../Models/Model/Model.component.tsx | 8 +- .../MultipleAnimationModel.component.tsx | 12 +- .../MultipleAnimationModel.container.tsx | 1 + src/services/Models.service.tsx | 193 ++++++++++++++---- 5 files changed, 182 insertions(+), 65 deletions(-) diff --git a/src/App/components/Nova.tsx b/src/App/components/Nova.tsx index a047b6d1..0f414439 100644 --- a/src/App/components/Nova.tsx +++ b/src/App/components/Nova.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useMemo } from 'react'; import { StatsGl } from '@react-three/drei'; import { Avatar, CAMERA } from 'src/components/Avatar'; @@ -8,25 +8,26 @@ import { SettingsPanel } from './SettingsPanel'; const idleUrl = 'https://readyplayerme-assets.s3.amazonaws.com/animations/nova-male-idle.glb'; const victoryUrl = 'https://readyplayerme-assets.s3.amazonaws.com/animations/nova-victory-03.glb'; -const animations: Record = { - idle: idleUrl, - victory: victoryUrl -}; - -const modelOneUrl = - 'https://api.readyplayer.dev/v3/avatars/66fa76b8fdea89a183c01341.glb?meshCompression=true&textureQuality=low&meshLOD=1&morphTargetsGroup=Basic expressions'; -const modelTwoUrl = - 'https://api.readyplayer.dev/v3/avatars/66fa77cbfdea89a183c0134d.glb?meshCompression=true&textureQuality=low&meshLOD=1&morphTargetsGroup=Basic expressions'; - const models: Record = { - one: modelOneUrl, - two: modelTwoUrl + one: 'https://avatars.readyplayer.me/673dc84d6874800e1db46095.glb?meshCompression=true&textureQuality=low&meshLOD=1&morphTargetsGroup=Basic expressions', + two: 'https://avatars.readyplayer.me/673f109553f9ed9312fafe70.glb?meshCompression=true&textureQuality=low&meshLOD=1&morphTargetsGroup=Basic expressions', + three: + 'https://avatars.readyplayer.me/6745ca78c6e2d65e7b99d8e1.glb?meshCompression=true&textureQuality=low&meshLOD=1&morphTargetsGroup=Basic expressions', + four: 'https://avatars.readyplayer.me/6745df2bd7ae3ba1d340d019.glb?meshCompression=true&textureQuality=low&meshLOD=1&morphTargetsGroup=Basic expressions' }; export const AvatarNova: React.FC = () => { const [activeAnimation, setActiveAnimation] = React.useState('idle'); const [modelSrc, setModelSrc] = React.useState(models.one); + const animations = useMemo( + () => ({ + idle: idleUrl, + victory: victoryUrl + }), + [] + ); + return ( <> @@ -42,6 +43,12 @@ export const AvatarNova: React.FC = () => { + + = ({ bloom, materialConfig }) => { - const { materials } = useGraph(scene); const { gl } = useThree(); const [isTouching, setIsTouching] = useState(false); const [touchEvent, setTouchEvent] = useState(null); @@ -61,7 +60,8 @@ export const Model: FC = ({ [isTouching, touchEvent, scene] ); - normaliseMaterialsConfig(materials, bloom, materialConfig); + normaliseMaterialsConfig(scene, bloom, materialConfig); + scene.traverse((object) => { const node = object; @@ -74,7 +74,7 @@ export const Model: FC = ({ } }); - useEffect(() => triggerCallback(onLoaded), [scene, materials, onLoaded]); + useEffect(() => triggerCallback(onLoaded), [scene, onLoaded]); useEffect(() => { gl.domElement.addEventListener('mousedown', setTouchingOn); diff --git a/src/components/Models/MultipleAnimationModel/MultipleAnimationModel.component.tsx b/src/components/Models/MultipleAnimationModel/MultipleAnimationModel.component.tsx index bfb3d0b5..b6b605d1 100644 --- a/src/components/Models/MultipleAnimationModel/MultipleAnimationModel.component.tsx +++ b/src/components/Models/MultipleAnimationModel/MultipleAnimationModel.component.tsx @@ -1,10 +1,10 @@ import React, { FC, useEffect, useRef } from 'react'; -import { useFrame, useGraph } from '@react-three/fiber'; +import { useFrame } from '@react-three/fiber'; import { AnimationAction, AnimationMixer } from 'three'; import { Model } from 'src/components/Models/Model'; import { AnimationConfiguration, BaseModelProps } from 'src/types'; -import { useEmotion, useFallback, useGltfCachedLoader, useIdleExpression } from 'src/services'; +import { useEmotion, useFallbackScene, useGltfCachedLoader, useIdleExpression } from 'src/services'; import { Emotion } from 'src/components/Avatar/Avatar.component'; import { useAnimations } from 'src/services/Animation.service'; @@ -35,7 +35,6 @@ export const MultipleAnimationModel: FC = ({ const loadedAnimations = useAnimations(animations); const { scene } = useGltfCachedLoader(modelSrc); - const { nodes } = useGraph(scene); useEffect(() => { if (scene) { @@ -55,6 +54,7 @@ export const MultipleAnimationModel: FC = ({ return () => { mixerRef.current?.stopAllAction(); mixerRef.current?.uncacheRoot(scene); + mixerRef.current = null; }; }, [scene]); @@ -83,9 +83,9 @@ export const MultipleAnimationModel: FC = ({ animationTimeRef.current = activeActionRef.current?.time || 0; }); - useEmotion(nodes, emotion); - useIdleExpression('blink', nodes); - useFallback(nodes, setModelFallback); + useEmotion(scene, emotion); + useIdleExpression('blink', scene); + useFallbackScene(scene, setModelFallback); return ; }; diff --git a/src/components/Models/MultipleAnimationModel/MultipleAnimationModel.container.tsx b/src/components/Models/MultipleAnimationModel/MultipleAnimationModel.container.tsx index 046477fc..d71ad4b1 100644 --- a/src/components/Models/MultipleAnimationModel/MultipleAnimationModel.container.tsx +++ b/src/components/Models/MultipleAnimationModel/MultipleAnimationModel.container.tsx @@ -1,4 +1,5 @@ import React, { FC, Suspense, useState } from 'react'; + import { MultipleAnimationModel, MultipleAnimationModelProps } from './MultipleAnimationModel.component'; /** diff --git a/src/services/Models.service.tsx b/src/services/Models.service.tsx index 430bb590..6ea94fe1 100644 --- a/src/services/Models.service.tsx +++ b/src/services/Models.service.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useCallback, useRef } from 'react'; +import React, { useCallback, useEffect, useRef } from 'react'; import { LinearFilter, MeshStandardMaterial, @@ -10,7 +10,10 @@ import { Vector3, BufferGeometry, Skeleton, - Group + Group, + Texture, + Mesh, + Object3DEventMap } from 'three'; import { useFrame } from '@react-three/fiber'; import type { ObjectMap, SkinnedMeshProps } from '@react-three/fiber'; @@ -72,15 +75,49 @@ export const clamp = (value: number, max: number, min: number): number => Math.m export const lerp = (start: number, end: number, time = 0.05): number => start * (1 - time) + end * time; +function traverseMaterials(object: Object3D, callback: (material: Material) => void) { + object.traverse((node) => { + const mesh = node as Mesh; + if (!mesh.geometry) return; + const materials = Array.isArray(mesh.material) ? mesh.material : [mesh.material]; + materials.forEach(callback); + }); +} + +const disposeGltfScene = (scene: Group) => { + scene.traverse((node) => { + if (node instanceof SkinnedMesh && node.skeleton) { + node.geometry.dispose(); + node.skeleton.dispose(); + } + + if (node instanceof Mesh) { + node.geometry.dispose(); + } + }); + + traverseMaterials(scene, (material: Material) => { + Object.values(material).forEach((value) => { + if (value instanceof Texture) { + value.dispose(); + } + }); + + material.dispose(); + }); + + scene.clear(); +}; + /** * Avoid texture pixelation and add depth effect. */ export const normaliseMaterialsConfig = ( - materials: Record, + scene: Group, bloomConfig?: BloomConfiguration, materialConfig?: MaterialConfiguration ) => { - Object.values(materials).forEach((material) => { + traverseMaterials(scene, (material: Material) => { const mat = material as MeshStandardMaterial; if (mat.map) { mat.map.minFilter = LinearFilter; @@ -184,21 +221,31 @@ export const mutatePose = (targetNodes?: ObjectMap['nodes'], sourceNodes?: Objec } }; -export const useEmotion = (nodes: ObjectMap['nodes'], emotion?: Emotion) => { - // @ts-ignore - const meshes = Object.values(nodes).filter((item: SkinnedMesh) => item?.morphTargetInfluences) as SkinnedMesh[]; +export const useEmotion = (nodes: ObjectMap['nodes'] | Group, emotion?: Emotion) => { + useEffect(() => { + let meshes: SkinnedMesh[] = []; - const resetEmotions = (resetMeshes: Array) => { - resetMeshes.forEach((mesh) => { - mesh?.morphTargetInfluences?.forEach((_, index) => { - mesh!.morphTargetInfluences![index] = 0; + if (nodes instanceof Group) { + nodes.traverse((object) => { + if (object instanceof SkinnedMesh && object.morphTargetInfluences) { + meshes.push(object); + } }); - }); - }; + } else { + // @ts-ignore + meshes = Object.values(nodes).filter((item: SkinnedMesh) => item?.morphTargetInfluences) as SkinnedMesh[]; + } + + const resetEmotions = () => { + meshes.forEach((mesh) => { + if (mesh.morphTargetInfluences) { + mesh.morphTargetInfluences.fill(0); + } + }); + }; - useFrame(() => { if (emotion) { - resetEmotions(meshes); + resetEmotions(); meshes.forEach((mesh) => { Object.entries(emotion).forEach(([shape, value]) => { @@ -210,9 +257,9 @@ export const useEmotion = (nodes: ObjectMap['nodes'], emotion?: Emotion) => { }); }); } else { - resetEmotions(meshes); + resetEmotions(); } - }); + }, [emotion, nodes]); }; const loader = new GLTFLoader(); @@ -222,38 +269,57 @@ const dracoLoader = new DRACOLoader(); dracoLoader.setDecoderPath('https://www.gstatic.com/draco/versioned/decoders/1.5.5/'); loader.setDRACOLoader(dracoLoader); +async function loadGltf(source: Blob | string): Promise { + let gltf: GLTF; + + if (source instanceof Blob) { + const url = URL.createObjectURL(source); + try { + gltf = await loader.loadAsync(url); + } finally { + URL.revokeObjectURL(url); + } + } else { + gltf = await loader.loadAsync(source); + } + + return gltf; +} + export const useGltfLoader = (source: Blob | string): GLTF => - suspend( - async () => { - if (source instanceof Blob) { - const buffer = await source.arrayBuffer(); - return (await loader.parseAsync(buffer, '')) as unknown as GLTF; + suspend(async () => loadGltf(source), [source], { lifespan: 100 }); + +export const useGltfCachedLoader = (source: Blob | string): GLTF => { + const cachedGltf = useRef(null); + const prevSource = useRef(null); + + const scene = suspend( + async (): Promise => { + if (source === prevSource.current && cachedGltf.current) { + return cachedGltf.current; } - return loader.loadAsync(source); + const gltf = await loadGltf(source); + + cachedGltf.current = gltf; + prevSource.current = source; + + return gltf; }, [source], { lifespan: 100 } ); -export const useGltfCachedLoader = (source: Blob | string): GLTF => { - const cachedGltf = useRef>(new Map()); - - return suspend(async (): Promise => { - if (cachedGltf.current.has(source as string)) { - return cachedGltf.current.get(source as string)!; - } - let result: GLTF; - if (source instanceof Blob) { - const buffer = await source.arrayBuffer(); - result = (await loader.parseAsync(buffer, '')) as GLTF; - } else { - result = await loader.loadAsync(source); - } + useEffect( + () => () => { + if (scene) { + disposeGltfScene(scene.scene); + } + }, + [scene] + ); - cachedGltf.current.set(source as string, result); - return result; - }, [source]); + return scene; }; export function usePersistantRotation(scene: Group) { @@ -282,6 +348,40 @@ export class Transform { position: Vector3; } +/** + * Builds a fallback model for given scene. + * Useful for displaying as the suspense fallback object. + */ +function buildFallbackScene(scene: Group, transform: Transform = new Transform()) { + return ( + + ); +} + +export const useFallbackScene = (scene: Group, setter?: (fallback: JSX.Element) => void) => { + const previousSceneRef = useRef(); + + useEffect(() => { + const newScene = scene.clone(); + + if (typeof setter === 'function') { + setter(buildFallbackScene(newScene)); + } + + if (previousSceneRef.current) { + disposeGltfScene(previousSceneRef.current); + } + + previousSceneRef.current = newScene; + + return () => { + if (previousSceneRef.current) { + disposeGltfScene(previousSceneRef.current); + } + }; + }, [scene, setter]); +}; + /** * Builds a fallback model for given nodes. * Useful for displaying as the suspense fallback object. @@ -374,8 +474,17 @@ export const expressions = { /** * Animates avatars facial expressions when morphTargets=ARKit,Eyes Extra is provided with the avatar. */ -export const useIdleExpression = (expression: keyof typeof expressions, nodes: Nodes) => { - const headMesh = (nodes.Wolf3D_Head || nodes.Wolf3D_Avatar || nodes.head) as unknown as SkinnedMeshProps; +export const useIdleExpression = (expression: keyof typeof expressions, nodes: Nodes | Group) => { + let headMesh: SkinnedMeshProps; + + if (nodes instanceof Group) { + headMesh = (nodes.getObjectByName('Wolf3D_Head') || + nodes.getObjectByName('Wolf3D_Avatar') || + nodes.getObjectByName('head')) as unknown as SkinnedMeshProps; + } else { + headMesh = (nodes.Wolf3D_Head || nodes.Wolf3D_Avatar || nodes.head) as unknown as SkinnedMeshProps; + } + const selectedExpression = expression in expressions ? expressions[expression] : undefined; const timeout = useRef(); const duration = useRef(Number.POSITIVE_INFINITY);