From ff45ebcf5e1c4c7e64d8abe6b535147a9e83a7c7 Mon Sep 17 00:00:00 2001 From: a-crowell Date: Tue, 6 Aug 2024 10:05:03 -0700 Subject: [PATCH 1/4] Hopeful fix (Works when OPFS on PC, but need Mac to test on) --- fission/src/mirabuf/MirabufLoader.ts | 146 ++++++++++++++++++++------- 1 file changed, 110 insertions(+), 36 deletions(-) diff --git a/fission/src/mirabuf/MirabufLoader.ts b/fission/src/mirabuf/MirabufLoader.ts index 00c0068733..505d684e1f 100644 --- a/fission/src/mirabuf/MirabufLoader.ts +++ b/fission/src/mirabuf/MirabufLoader.ts @@ -21,14 +21,39 @@ export interface MirabufRemoteInfo { src: string } -type MiraCache = { [id: string]: MirabufCacheInfo } +type MapCache = { [id: string]: MirabufCacheInfo } +type BackUpBufferCache = { [id: string]: ArrayBuffer } +// Local Storage +const robotMapDirName = "RobotMap" +const fieldMapDirName = "FieldMap" +const backUpRobotsDirName = "BackUpRobots" +const backUpFieldsDirName = "BackUpFields" + +// OPFS const robotsDirName = "Robots" const fieldsDirName = "Fields" const root = await navigator.storage.getDirectory() const robotFolderHandle = await root.getDirectoryHandle(robotsDirName, { create: true }) const fieldFolderHandle = await root.getDirectoryHandle(fieldsDirName, { create: true }) +const canOPFS = await (async() => { + try { + console.log(`trying OPFS`) + const fileHandle = await robotFolderHandle.getFileHandle( + "0", + { create: true } + ) + const writable = await fileHandle.createWritable() + await writable.close() + console.log(`yes OPFS`) + return true + } catch (e) { + console.log(`no OPFS`) + return false + } +})() + export function UnzipMira(buff: Uint8Array): Uint8Array { // Check if file is gzipped via magic gzip numbers 31 139 if (buff[0] == 31 && buff[1] == 139) { @@ -42,21 +67,47 @@ class MirabufCachingService { /** * Get the map of mirabuf keys and paired MirabufCacheInfo from local storage * - * @param {MiraType} miraType Type of Mirabuf Assembly. + * @param {MiraType} miraType Type of Mirabuf Assembly * - * @returns {MiraCache} Map of cached keys and paired MirabufCacheInfo + * @returns {MapCache} Map of cached keys and paired MirabufCacheInfo + */ + public static GetCacheMap(miraType: MiraType): MapCache { + if ( + (window.localStorage.getItem(MIRABUF_LOCALSTORAGE_GENERATION_KEY) ?? "") == MIRABUF_LOCALSTORAGE_GENERATION + ) { + window.localStorage.setItem(MIRABUF_LOCALSTORAGE_GENERATION_KEY, MIRABUF_LOCALSTORAGE_GENERATION) + window.localStorage.setItem(robotMapDirName, "{}") + window.localStorage.setItem(fieldMapDirName, "{}") + return {} + } + + const key = miraType == MiraType.ROBOT ? robotMapDirName : fieldMapDirName + const map = window.localStorage.getItem(key) + + if (map) { + return JSON.parse(map) + } else { + window.localStorage.setItem(key, "{}") + return {} + } + } + + /** + * Get back up cache from local storage. Used if no access to OPFS (Safari and iOS) + * @param miraType Type of Mirabuf Assembly + * @returns IDs and buffers */ - public static GetCacheMap(miraType: MiraType): MiraCache { + public static GetBackUpCache(miraType: MiraType): BackUpBufferCache { if ( (window.localStorage.getItem(MIRABUF_LOCALSTORAGE_GENERATION_KEY) ?? "") == MIRABUF_LOCALSTORAGE_GENERATION ) { window.localStorage.setItem(MIRABUF_LOCALSTORAGE_GENERATION_KEY, MIRABUF_LOCALSTORAGE_GENERATION) - window.localStorage.setItem(robotsDirName, "{}") - window.localStorage.setItem(fieldsDirName, "{}") + window.localStorage.setItem(robotMapDirName, "{}") + window.localStorage.setItem(fieldMapDirName, "{}") return {} } - const key = miraType == MiraType.ROBOT ? robotsDirName : fieldsDirName + const key = miraType == MiraType.ROBOT ? backUpRobotsDirName : backUpFieldsDirName const map = window.localStorage.getItem(key) if (map) { @@ -67,6 +118,7 @@ class MirabufCachingService { } } + /** * Cache remote Mirabuf file * @@ -159,19 +211,19 @@ class MirabufCachingService { thumbnailStorageID?: string ): Promise { try { - const map: MiraCache = this.GetCacheMap(miraType) + const map: MapCache = this.GetCacheMap(miraType) const id = map[key].id const _name = map[key].name const _thumbnailStorageID = map[key].thumbnailStorageID - const hi: MirabufCacheInfo = { + const info: MirabufCacheInfo = { id: id, cacheKey: key, miraType: miraType, name: name ?? _name, thumbnailStorageID: thumbnailStorageID ?? _thumbnailStorageID, } - map[key] = hi - window.localStorage.setItem(miraType == MiraType.ROBOT ? robotsDirName : fieldsDirName, JSON.stringify(map)) + map[key] = info + window.localStorage.setItem(miraType == MiraType.ROBOT ? robotMapDirName : fieldMapDirName, JSON.stringify(map)) return true } catch (e) { console.error(`Failed to cache info\n${e}`) @@ -213,16 +265,18 @@ class MirabufCachingService { */ public static async Get(id: MirabufCacheID, miraType: MiraType): Promise { try { - const fileHandle = await (miraType == MiraType.ROBOT ? robotFolderHandle : fieldFolderHandle).getFileHandle( + // If we have access to OPFS, get file handle from OPFS + const fileHandle = canOPFS ? await (miraType == MiraType.ROBOT ? robotFolderHandle : fieldFolderHandle).getFileHandle( id, { create: false, } - ) + ) : undefined + + // If we have OPFS file, get buffer from OPFS, otherwise get from local storage + const buff = fileHandle ? await fileHandle.getFile().then(x => x.arrayBuffer()) : this.GetBackUpCache(miraType)[id] - // Get assembly from file - if (fileHandle) { - const buff = await fileHandle.getFile().then(x => x.arrayBuffer()) + if (buff) { const assembly = this.AssemblyFromBuffer(buff) World.AnalyticsSystem?.Event("Cache Get", { key: id, @@ -232,11 +286,10 @@ class MirabufCachingService { }) return assembly } else { - console.error(`Failed to get file handle for ID: ${id}`) - return undefined + console.error(`Failed to find file for id: ${id}`) } } catch (e) { - console.error(`Failed to find file from OPFS\n${e}`) + console.error(`Failed to find file\n${e}`) return undefined } } @@ -256,13 +309,24 @@ class MirabufCachingService { if (map) { delete map[key] window.localStorage.setItem( - miraType == MiraType.ROBOT ? robotsDirName : fieldsDirName, + miraType == MiraType.ROBOT ? robotMapDirName : fieldMapDirName, JSON.stringify(map) ) } - const dir = miraType == MiraType.ROBOT ? robotFolderHandle : fieldFolderHandle - await dir.removeEntry(id) + if (canOPFS) { + const dir = miraType == MiraType.ROBOT ? robotFolderHandle : fieldFolderHandle + await dir.removeEntry(id) + } else { + const backUpCache = this.GetBackUpCache(miraType) + if (backUpCache) { + delete backUpCache[key] + window.localStorage.setItem( + miraType == MiraType.ROBOT ? backUpRobotsDirName : backUpFieldsDirName, + JSON.stringify(backUpCache) + ) + } + } World.AnalyticsSystem?.Event("Cache Remove", { key: key, @@ -287,8 +351,10 @@ class MirabufCachingService { fieldFolderHandle.removeEntry(key) } - window.localStorage.removeItem(robotsDirName) - window.localStorage.removeItem(fieldsDirName) + window.localStorage.removeItem(robotMapDirName) + window.localStorage.removeItem(fieldMapDirName) + window.localStorage.removeItem(backUpRobotsDirName) + window.localStorage.removeItem(backUpFieldsDirName) } // Optional name for when assembly is being decoded anyway like in CacheAndGetLocal() @@ -298,7 +364,6 @@ class MirabufCachingService { miraType?: MiraType, name?: string ): Promise { - // Store in OPFS const backupID = Date.now().toString() try { if (!miraType) { @@ -306,16 +371,8 @@ class MirabufCachingService { miraType = this.AssemblyFromBuffer(miraBuff).dynamic ? MiraType.ROBOT : MiraType.FIELD } - const fileHandle = await (miraType == MiraType.ROBOT ? robotFolderHandle : fieldFolderHandle).getFileHandle( - backupID, - { create: true } - ) - const writable = await fileHandle.createWritable() - await writable.write(miraBuff) - await writable.close() - // Local cache map - const map: MiraCache = this.GetCacheMap(miraType) + const map: MapCache = this.GetCacheMap(miraType) const info: MirabufCacheInfo = { id: backupID, cacheKey: key, @@ -323,7 +380,7 @@ class MirabufCachingService { name: name, } map[key] = info - window.localStorage.setItem(miraType == MiraType.ROBOT ? robotsDirName : fieldsDirName, JSON.stringify(map)) + window.localStorage.setItem(miraType == MiraType.ROBOT ? robotMapDirName : fieldMapDirName, JSON.stringify(map)) World.AnalyticsSystem?.Event("Cache Store", { name: name ?? "-", @@ -331,6 +388,23 @@ class MirabufCachingService { type: miraType == MiraType.ROBOT ? "robot" : "field", fileSize: miraBuff.byteLength, }) + + if (canOPFS) { + // Store in OPFS + const fileHandle = await (miraType == MiraType.ROBOT ? robotFolderHandle : fieldFolderHandle).getFileHandle( + backupID, + { create: true } + ) + const writable = await fileHandle.createWritable() + await writable.write(miraBuff) + await writable.close() + } else { + // Store in localStorage + const cache: BackUpBufferCache = this.GetBackUpCache(miraType) + cache[key] = miraBuff + window.localStorage.setItem(miraType == MiraType.ROBOT ? backUpRobotsDirName : backUpFieldsDirName, JSON.stringify(cache)) + } + return info } catch (e) { console.error("Failed to cache mira " + e) @@ -353,7 +427,7 @@ class MirabufCachingService { export enum MiraType { ROBOT = 1, - FIELD = 2, + FIELD, } export default MirabufCachingService From fa668d31a5057495218769a8e7535d6423e7a89d Mon Sep 17 00:00:00 2001 From: a-crowell Date: Tue, 6 Aug 2024 10:16:01 -0700 Subject: [PATCH 2/4] More checks --- fission/src/mirabuf/MirabufLoader.ts | 29 ++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/fission/src/mirabuf/MirabufLoader.ts b/fission/src/mirabuf/MirabufLoader.ts index 505d684e1f..4ce17696b4 100644 --- a/fission/src/mirabuf/MirabufLoader.ts +++ b/fission/src/mirabuf/MirabufLoader.ts @@ -40,14 +40,27 @@ const fieldFolderHandle = await root.getDirectoryHandle(fieldsDirName, { create: const canOPFS = await (async() => { try { console.log(`trying OPFS`) - const fileHandle = await robotFolderHandle.getFileHandle( - "0", - { create: true } - ) - const writable = await fileHandle.createWritable() - await writable.close() - console.log(`yes OPFS`) - return true + + if (robotFolderHandle.name == robotsDirName) { + robotFolderHandle.entries + robotFolderHandle.keys + + const fileHandle = await robotFolderHandle.getFileHandle( + "0", + { create: true } + ) + const writable = await fileHandle.createWritable() + await writable.close() + await fileHandle.getFile() + + robotFolderHandle.removeEntry(fileHandle.name) + + console.log(`yes OPFS`) + return true + } else { + console.log(`no OPFS`) + return false + } } catch (e) { console.log(`no OPFS`) return false From 1ed0fd2a5742c637e1e001599d8a06ecff2ddf08 Mon Sep 17 00:00:00 2001 From: a-crowell Date: Tue, 6 Aug 2024 12:24:02 -0700 Subject: [PATCH 3/4] Fixed --- fission/src/mirabuf/MirabufLoader.ts | 117 ++++++++++----------------- fission/src/mirabuf/MirabufParser.ts | 1 - fission/src/ui/panels/DebugPanel.tsx | 4 +- 3 files changed, 44 insertions(+), 78 deletions(-) diff --git a/fission/src/mirabuf/MirabufLoader.ts b/fission/src/mirabuf/MirabufLoader.ts index 4ce17696b4..cd51213ad7 100644 --- a/fission/src/mirabuf/MirabufLoader.ts +++ b/fission/src/mirabuf/MirabufLoader.ts @@ -21,26 +21,19 @@ export interface MirabufRemoteInfo { src: string } -type MapCache = { [id: string]: MirabufCacheInfo } -type BackUpBufferCache = { [id: string]: ArrayBuffer } +type MapCache = { [id: MirabufCacheID]: MirabufCacheInfo } -// Local Storage -const robotMapDirName = "RobotMap" -const fieldMapDirName = "FieldMap" -const backUpRobotsDirName = "BackUpRobots" -const backUpFieldsDirName = "BackUpFields" - -// OPFS const robotsDirName = "Robots" const fieldsDirName = "Fields" const root = await navigator.storage.getDirectory() const robotFolderHandle = await root.getDirectoryHandle(robotsDirName, { create: true }) const fieldFolderHandle = await root.getDirectoryHandle(fieldsDirName, { create: true }) +export const backUpRobots: Map = new Map() +export const backUpFields: Map = new Map() + const canOPFS = await (async() => { try { - console.log(`trying OPFS`) - if (robotFolderHandle.name == robotsDirName) { robotFolderHandle.entries robotFolderHandle.keys @@ -55,14 +48,13 @@ const canOPFS = await (async() => { robotFolderHandle.removeEntry(fileHandle.name) - console.log(`yes OPFS`) return true } else { - console.log(`no OPFS`) + console.log(`No access to OPFS`) return false } } catch (e) { - console.log(`no OPFS`) + console.log(`No access to OPFS`) return false } })() @@ -89,38 +81,12 @@ class MirabufCachingService { (window.localStorage.getItem(MIRABUF_LOCALSTORAGE_GENERATION_KEY) ?? "") == MIRABUF_LOCALSTORAGE_GENERATION ) { window.localStorage.setItem(MIRABUF_LOCALSTORAGE_GENERATION_KEY, MIRABUF_LOCALSTORAGE_GENERATION) - window.localStorage.setItem(robotMapDirName, "{}") - window.localStorage.setItem(fieldMapDirName, "{}") - return {} - } - - const key = miraType == MiraType.ROBOT ? robotMapDirName : fieldMapDirName - const map = window.localStorage.getItem(key) - - if (map) { - return JSON.parse(map) - } else { - window.localStorage.setItem(key, "{}") - return {} - } - } - - /** - * Get back up cache from local storage. Used if no access to OPFS (Safari and iOS) - * @param miraType Type of Mirabuf Assembly - * @returns IDs and buffers - */ - public static GetBackUpCache(miraType: MiraType): BackUpBufferCache { - if ( - (window.localStorage.getItem(MIRABUF_LOCALSTORAGE_GENERATION_KEY) ?? "") == MIRABUF_LOCALSTORAGE_GENERATION - ) { - window.localStorage.setItem(MIRABUF_LOCALSTORAGE_GENERATION_KEY, MIRABUF_LOCALSTORAGE_GENERATION) - window.localStorage.setItem(robotMapDirName, "{}") - window.localStorage.setItem(fieldMapDirName, "{}") + window.localStorage.setItem(robotsDirName, "{}") + window.localStorage.setItem(fieldsDirName, "{}") return {} } - const key = miraType == MiraType.ROBOT ? backUpRobotsDirName : backUpFieldsDirName + const key = miraType == MiraType.ROBOT ? robotsDirName : fieldsDirName const map = window.localStorage.getItem(key) if (map) { @@ -236,7 +202,7 @@ class MirabufCachingService { thumbnailStorageID: thumbnailStorageID ?? _thumbnailStorageID, } map[key] = info - window.localStorage.setItem(miraType == MiraType.ROBOT ? robotMapDirName : fieldMapDirName, JSON.stringify(map)) + window.localStorage.setItem(miraType == MiraType.ROBOT ? robotsDirName : fieldsDirName, JSON.stringify(map)) return true } catch (e) { console.error(`Failed to cache info\n${e}`) @@ -278,17 +244,19 @@ class MirabufCachingService { */ public static async Get(id: MirabufCacheID, miraType: MiraType): Promise { try { - // If we have access to OPFS, get file handle from OPFS - const fileHandle = canOPFS ? await (miraType == MiraType.ROBOT ? robotFolderHandle : fieldFolderHandle).getFileHandle( - id, - { - create: false, - } - ) : undefined - - // If we have OPFS file, get buffer from OPFS, otherwise get from local storage - const buff = fileHandle ? await fileHandle.getFile().then(x => x.arrayBuffer()) : this.GetBackUpCache(miraType)[id] - + // Get buffer from hashMap. If not in hashMap, check OPFS. Otherwise, buff is undefined + const cache = miraType == MiraType.ROBOT ? backUpRobots : backUpFields + const buff = cache.get(id) ?? await (async() => { + const fileHandle = canOPFS ? await (miraType == MiraType.ROBOT ? robotFolderHandle : fieldFolderHandle).getFileHandle( + id, + { + create: false, + } + ) : undefined + return fileHandle ? await fileHandle.getFile().then(x => x.arrayBuffer()) : undefined + })() + + // If we have buffer, get assembly if (buff) { const assembly = this.AssemblyFromBuffer(buff) World.AnalyticsSystem?.Event("Cache Get", { @@ -299,7 +267,7 @@ class MirabufCachingService { }) return assembly } else { - console.error(`Failed to find file for id: ${id}`) + console.error(`Failed to find arrayBuffer for id: ${id}`) } } catch (e) { console.error(`Failed to find file\n${e}`) @@ -322,7 +290,7 @@ class MirabufCachingService { if (map) { delete map[key] window.localStorage.setItem( - miraType == MiraType.ROBOT ? robotMapDirName : fieldMapDirName, + miraType == MiraType.ROBOT ? robotsDirName : fieldsDirName, JSON.stringify(map) ) } @@ -330,15 +298,11 @@ class MirabufCachingService { if (canOPFS) { const dir = miraType == MiraType.ROBOT ? robotFolderHandle : fieldFolderHandle await dir.removeEntry(id) - } else { - const backUpCache = this.GetBackUpCache(miraType) - if (backUpCache) { - delete backUpCache[key] - window.localStorage.setItem( - miraType == MiraType.ROBOT ? backUpRobotsDirName : backUpFieldsDirName, - JSON.stringify(backUpCache) - ) - } + } + + const backUpCache = miraType == MiraType.ROBOT ? backUpRobots : backUpFields + if (backUpCache) { + backUpCache.delete(id) } World.AnalyticsSystem?.Event("Cache Remove", { @@ -364,10 +328,11 @@ class MirabufCachingService { fieldFolderHandle.removeEntry(key) } - window.localStorage.removeItem(robotMapDirName) - window.localStorage.removeItem(fieldMapDirName) - window.localStorage.removeItem(backUpRobotsDirName) - window.localStorage.removeItem(backUpFieldsDirName) + window.localStorage.removeItem(robotsDirName) + window.localStorage.removeItem(fieldsDirName) + + backUpRobots.clear() + backUpFields.clear() } // Optional name for when assembly is being decoded anyway like in CacheAndGetLocal() @@ -393,7 +358,7 @@ class MirabufCachingService { name: name, } map[key] = info - window.localStorage.setItem(miraType == MiraType.ROBOT ? robotMapDirName : fieldMapDirName, JSON.stringify(map)) + window.localStorage.setItem(miraType == MiraType.ROBOT ? robotsDirName : fieldsDirName, JSON.stringify(map)) World.AnalyticsSystem?.Event("Cache Store", { name: name ?? "-", @@ -402,6 +367,7 @@ class MirabufCachingService { fileSize: miraBuff.byteLength, }) + // Store buffer if (canOPFS) { // Store in OPFS const fileHandle = await (miraType == MiraType.ROBOT ? robotFolderHandle : fieldFolderHandle).getFileHandle( @@ -411,13 +377,12 @@ class MirabufCachingService { const writable = await fileHandle.createWritable() await writable.write(miraBuff) await writable.close() - } else { - // Store in localStorage - const cache: BackUpBufferCache = this.GetBackUpCache(miraType) - cache[key] = miraBuff - window.localStorage.setItem(miraType == MiraType.ROBOT ? backUpRobotsDirName : backUpFieldsDirName, JSON.stringify(cache)) } + // Store in hash + const cache = miraType == MiraType.ROBOT ? backUpRobots : backUpFields + cache.set(backupID, miraBuff) + return info } catch (e) { console.error("Failed to cache mira " + e) diff --git a/fission/src/mirabuf/MirabufParser.ts b/fission/src/mirabuf/MirabufParser.ts index dd7df1e7cf..7dab72156b 100644 --- a/fission/src/mirabuf/MirabufParser.ts +++ b/fission/src/mirabuf/MirabufParser.ts @@ -240,7 +240,6 @@ class MirabufParser { console.log("Failed to get part definitions") return } - console.log(partDefinitions) } private NewRigidNode(suffix?: string): RigidNode { diff --git a/fission/src/ui/panels/DebugPanel.tsx b/fission/src/ui/panels/DebugPanel.tsx index 3daebda498..28b96397f4 100644 --- a/fission/src/ui/panels/DebugPanel.tsx +++ b/fission/src/ui/panels/DebugPanel.tsx @@ -6,7 +6,7 @@ import WPILibBrain from "@/systems/simulation/wpilib_brain/WPILibBrain" import { MainHUD_AddToast } from "../components/MainHUD" import { ToastType } from "../ToastContext" import { Random } from "@/util/Random" -import MirabufCachingService, { MiraType } from "@/mirabuf/MirabufLoader" +import MirabufCachingService, { backUpFields as hashedMiraFields, backUpRobots as hashedMiraRobots, MiraType } from "@/mirabuf/MirabufLoader" import { Box, styled } from "@mui/material" import { usePanelControlContext } from "../PanelContext" import APS from "@/aps/APS" @@ -113,6 +113,8 @@ const DebugPanel: React.FC = ({ panelId }) => { onClick={() => { console.log(MirabufCachingService.GetCacheMap(MiraType.ROBOT)) console.log(MirabufCachingService.GetCacheMap(MiraType.FIELD)) + console.log(hashedMiraRobots) + console.log(hashedMiraFields) }} />