From bce8758820a6baa002a7e3ca6af4e6b2bfc45c56 Mon Sep 17 00:00:00 2001 From: Sebastien DUMETZ Date: Thu, 25 Jan 2024 10:50:01 +0100 Subject: [PATCH] [WIP] rework pointers to dereference every indexable array property --- .../documents/01_simple.svx.json | 1 + source/server/utils/merge/apply.test.ts | 2 +- source/server/utils/merge/apply.ts | 73 +--- source/server/utils/merge/diff.test.ts | 2 +- source/server/utils/merge/diff.ts | 2 +- source/server/utils/merge/index.ts | 4 +- .../merge/{pointers.ts => pointers/index.ts} | 206 +++++----- source/server/utils/merge/pointers/meta.ts | 61 +++ source/server/utils/merge/pointers/model.ts | 45 +++ source/server/utils/merge/pointers/node.ts | 47 +++ .../merge/{ => pointers}/pointers.test.ts | 354 +++++++++--------- source/server/utils/merge/pointers/scene.ts | 31 ++ source/server/utils/merge/pointers/setup.ts | 44 +++ .../utils/merge/{ => pointers}/types.ts | 76 ++-- source/server/utils/schema/document.d.ts | 10 +- source/server/utils/schema/model.d.ts | 2 +- source/server/utils/schema/setup.d.ts | 2 +- source/server/utils/uid.ts | 11 +- 18 files changed, 606 insertions(+), 367 deletions(-) rename source/server/utils/merge/{pointers.ts => pointers/index.ts} (50%) create mode 100644 source/server/utils/merge/pointers/meta.ts create mode 100644 source/server/utils/merge/pointers/model.ts create mode 100644 source/server/utils/merge/pointers/node.ts rename source/server/utils/merge/{ => pointers}/pointers.test.ts (81%) create mode 100644 source/server/utils/merge/pointers/scene.ts create mode 100644 source/server/utils/merge/pointers/setup.ts rename source/server/utils/merge/{ => pointers}/types.ts (59%) diff --git a/source/server/__test_fixtures/documents/01_simple.svx.json b/source/server/__test_fixtures/documents/01_simple.svx.json index d9c41117..2272d189 100644 --- a/source/server/__test_fixtures/documents/01_simple.svx.json +++ b/source/server/__test_fixtures/documents/01_simple.svx.json @@ -12,6 +12,7 @@ "units": "cm" }], "nodes": [{ + "name": "Camera", "camera": 0 }, { "name": "Lights", diff --git a/source/server/utils/merge/apply.test.ts b/source/server/utils/merge/apply.test.ts index 7559bbca..4190ab51 100644 --- a/source/server/utils/merge/apply.test.ts +++ b/source/server/utils/merge/apply.test.ts @@ -1,5 +1,5 @@ -import {DELETE_KEY} from "./types.js"; +import {DELETE_KEY} from "./pointers/types.js"; import apply from "./apply.js"; describe("merge.apply()", function(){ diff --git a/source/server/utils/merge/apply.ts b/source/server/utils/merge/apply.ts index cfc353b3..c90c4c06 100644 --- a/source/server/utils/merge/apply.ts +++ b/source/server/utils/merge/apply.ts @@ -1,5 +1,5 @@ 'use strict'; -import {Diff, DELETE_KEY} from "./types.js"; +import {Diff, DELETE_KEY} from "./pointers/types.js"; /** * Deep assign two or more objects @@ -12,78 +12,39 @@ export default function apply>(into :T, ...diffs : for(const diff of diffs){ for(const key in diff){ const value = diff[key] as T[Extract]; - if(apply_core(into, key, value)){ - continue; - } - - //Then handle arrays to apply edge cases - if(Array.isArray(into[key])){ - apply_array(into[key], value); - continue; - } - //Default case : recurse. - into[key] ??= {} as any; - apply(into[key], value); - } - } - return into; -} -/** - * Handle special merge conditions for arrays - * In particular: - * - for arrays of native types, replace the whole array - * - for arrays of objects with a "name" or "id", tries to merge objects with a matching id/name, appends the rest - * - */ -function apply_array>(into :T, ...diffs :Diff[]):T{ - for (const diff of diffs){ - for(const idx in diff){ - const value = diff[idx]; - if(apply_core(into, idx, value)){ - continue; - } - if(typeof into[idx] !== undefined){ - //When an item already exist, check - let append = ["id", "name"].some(key =>{ - if(!value[key]) return false; - const ref_id = into[idx][key]; - if(!ref_id || ref_id == value[key]) return false; - return true; - }); - - if(append){ - into.push(value); - continue; + if(value === DELETE_KEY){ + delete into[key]; + }else if(typeof value !== "object" //primitive + || typeof into[key] === "undefined" //undefined target + ){ + into[key] = value; + }else if(Array.isArray(value)){ + if(value.some((v:any) => typeof v === "object")){ + console.warn("Applying array of non-native types : ", value); } + into[key] = value; + }else{ + //Default case : recurse. + into[key] ??= {} as any; + apply(into[key], value); } - //Default case : recurse as apply() would. - into[idx] ??= {} as any; - apply(into[idx], value); } } return into; } + /** * Return true if the value was applied, false if it needs further processing. * Handles the trivial cases: * - Applies DELETE_KEY * - Applies primitives (typeof value !== "object") + * - * - Applies properties that don't exist in the target */ function apply_core>(into :T, key:keyof T, value :any):boolean{ - if(value === DELETE_KEY){ - delete into[key]; - return true; - } - - if(typeof value !== "object" || typeof into[key] === "undefined"){ - //Handle primitives and undefined values - into[key] = value; - return true; - } return false; } \ No newline at end of file diff --git a/source/server/utils/merge/diff.test.ts b/source/server/utils/merge/diff.test.ts index 75592872..5bd2f898 100644 --- a/source/server/utils/merge/diff.test.ts +++ b/source/server/utils/merge/diff.test.ts @@ -1,5 +1,5 @@ -import {DELETE_KEY} from "./types.js"; +import {DELETE_KEY} from "./pointers/types.js"; import diff from "./diff.js"; diff --git a/source/server/utils/merge/diff.ts b/source/server/utils/merge/diff.ts index 34615be7..4ffdfcee 100644 --- a/source/server/utils/merge/diff.ts +++ b/source/server/utils/merge/diff.ts @@ -1,4 +1,4 @@ -import {Diff, DELETE_KEY} from "./types.js"; +import {Diff, DELETE_KEY} from "./pointers/types.js"; /** * Computes a diff between two objects. diff --git a/source/server/utils/merge/index.ts b/source/server/utils/merge/index.ts index 4176db1e..79f07d32 100644 --- a/source/server/utils/merge/index.ts +++ b/source/server/utils/merge/index.ts @@ -1,9 +1,9 @@ -import {Diff, DELETE_KEY, DerefDocument} from "./types.js"; +import {Diff, DELETE_KEY, DerefDocument} from "./pointers/types.js"; import apply from "./apply.js"; import diff from "./diff.js"; import { IDocument } from "../schema/document.js"; -import { fromPointers, toPointers } from "./pointers.js"; +import { fromPointers, toPointers } from "./pointers/index.js"; export { Diff, diff --git a/source/server/utils/merge/pointers.ts b/source/server/utils/merge/pointers/index.ts similarity index 50% rename from source/server/utils/merge/pointers.ts rename to source/server/utils/merge/pointers/index.ts index fdc1ec0f..90e0161e 100644 --- a/source/server/utils/merge/pointers.ts +++ b/source/server/utils/merge/pointers/index.ts @@ -1,51 +1,133 @@ -import { ICamera, IDocument, ILight, INode, IScene } from "../schema/document.js"; -import { IMeta } from "../schema/meta.js"; -import { IModel } from "../schema/model.js"; -import { ISetup } from "../schema/setup.js"; -import { DerefDocument, DerefNode, DerefScene } from "./types.js"; +import { ICamera, IDocument, ILight, INode, IScene } from "../../schema/document.js"; +import { IMeta } from "../../schema/meta.js"; +import { IModel } from "../../schema/model.js"; +import { ISetup } from "../../schema/setup.js"; +import uid from "../../uid.js"; +import { mapMeta } from "./meta.js"; +import { mapModel } from "./model.js"; +import { appendScene } from "./scene.js"; +import { mapSetup } from "./setup.js"; +import { DerefDocument, DerefMeta, DerefNode, DerefScene, DerefSetup } from "./types.js"; -type DocumentCollections = keyof Omit; + + + +/** + * Takes a dereferenced document and computes back the source svx.json document. + * It's akin to the `toDocument()`methods seen in DPO-voyager, in particular in `CVScene.ts`. + * @see toPointers for the inverse operation + */ +export function fromPointers({asset, scene} :DerefDocument): IDocument{ + + //The output document. + // All fields are initially created to later filter unused ones + let document :Required = { + asset, + scene: -1, + "scenes": [] as IScene[], + "nodes": [] as INode[], + "cameras": [] as ICamera[], + "lights": [] as ILight[], + "models": [] as IModel[], + "metas": [] as IMeta[], + "setups": [] as ISetup[], + }; + + + + document.scene = appendScene(document, scene); + + return cleanDocument(document); +} + +/** + * Clean up a document by removing empty collections + * The document object will get mutated in-place and is returned for good measure + * @param document + * @returns + */ +function cleanDocument(document :Required) :IDocument{ + //Remove empty collections + for(let key in document){ + let v = document[key as keyof typeof document]; + if(typeof v === "undefined" || (Array.isArray(v) && v.length == 0)) delete document[key as keyof typeof document]; + } + return document; +} /** * Dereference all indexed values into pointers in a SVX document * It unlinks all unreachable nodes (scenes, setups, etc...). * + * Where possible, arrays are converted to dictionaries + * indexed by any unique property their items might have (typ. `name`, `id` or `uri`). + * + * * It's something of an emulation of what DPO-voyager does when building a scene graph from a document. * See [CVScene.ts](https://github.com/Smithsonian/dpo-voyager/blob/master/source/client/components/CVScene.ts) * and [CVNode.ts](https://github.com/Smithsonian/dpo-voyager/blob/master/source/client/components/CVNode.ts) in voyager. * + * @fixme should be simplified by exporting most of its code into separate functions, like in `fromPointers()` */ export function toPointers(src :IDocument) :DerefDocument { - const nodes = src.nodes?.map(iNode=>{ + + // Dereference self-contained collections + const metas = src.metas?.map(mapMeta) ?? []; + const setups = src.setups?.map(mapSetup) ?? []; + const models = src.models?.map(mapModel)?? []; + + // nodes and scenes mapping couldn't be made side-effect-free because they map to other collections + + //Dereference every node's internal properties + const nodes = src.nodes?.map((iNode, nodeIndex)=>{ + if(!iNode.name) console.log("Node has no name : ", iNode); let node :DerefNode = { - name: iNode.name, + name: iNode.name ?? uid(), matrix: iNode.matrix, translation: iNode.translation, rotation: iNode.rotation, scale: iNode.scale, }; - //node.children will be assigned later if necessary - for(let key of ["camera", "light", "model", "meta"] as const){ + if(typeof iNode.meta !=="undefined"){ + node.meta = metas[iNode.meta]; + if(!node.meta) throw new Error(`Invalid meta index ${iNode.meta} in node #${nodeIndex}`); + } + if(typeof iNode.model !=="undefined"){ + node.model = models[iNode.model]; + if(!node.model) throw new Error(`Invalid model index ${iNode.model} in node #${nodeIndex}`); + } + + + for(let key of ["camera", "light"] as const){ const idx = iNode[key]; if(typeof idx == "number"){ const dict = src[key+"s" as `${typeof key}s`] let ptr = dict? dict[idx]: undefined; - if(!ptr) throw new Error(`Invalid ${key} index ${idx}`); + if(!ptr) throw new Error(`Invalid ${key} index ${idx} in node #${nodeIndex}`); (node[key] as any) = ptr; } } + + //node.children will be assigned later, because children might not have been dereferenced yet + return node; }); - + //Dereference node children. + //We don't need recursion because doc.nodes is a flat list nodes?.forEach((node, nodeIndex)=>{ const indices = (src.nodes as INode[])[nodeIndex].children; if(!indices) return; - node.children = indices.map(idx=>nodes[idx]); + node.children = indices.reduce((children, idx)=>{ + let node = nodes[idx]; + return {...children, [node.name]: node} + }, {}); }); + + if(!Array.isArray(src.scenes) || !src.scenes.length) throw new Error("Document has no valid scene"); if(typeof src.scene != "number"|| !src.scenes[src.scene]) throw new Error(`Document's scene #${src.scene} is invalid`); const iScene = src.scenes[src.scene]; @@ -53,109 +135,27 @@ export function toPointers(src :IDocument) :DerefDocument { const scene :DerefScene = { name: iScene.name, units: iScene.units, - nodes: [], + nodes: {}, } if(iScene.nodes?.length){ - scene.nodes = iScene.nodes.map(idx=>{ - if(!nodes || !nodes[idx]) throw new Error(`Invalid node index ${idx} in scene #${src.scene}`); - return nodes[idx]; - }); + scene.nodes = iScene.nodes.reduce((children, idx)=>{ + const node = nodes?.[idx] + if(!node) throw new Error(`Invalid node index ${idx} in scene #${src.scene}`); + return {...children, [node.name]: node}; + }, {}); } if(typeof iScene.setup == "number"){ if(!src.setups || !src.setups[iScene.setup]) throw new Error(`Invalid setup #${iScene.setup} in scene #${src.scene}`) - scene.setup = src.setups[iScene.setup]; + scene.setup = setups[iScene.setup]; } if(typeof iScene.meta == "number"){ if(!src.metas || !src.metas[iScene.meta]) throw new Error(`Invalid meta #${iScene.meta} in scene #${src.scene}`) - scene.meta = src.metas[iScene.meta]; + scene.meta = metas[iScene.meta]; } return { asset: src.asset, scene, }; -} - -/** - * Takes a dereferenced document and computes back the source svx.json document. - * It's akin to the `toDocument()`methods seen in DPO-voyager, in particular in `CVScene.ts`. - * @see toPointers for the inverse operation - */ -export function fromPointers({asset, scene} :DerefDocument): IDocument{ - - //The output document. - // All fields are initially created to later filter unused ones - let document :Required = { - asset, - scene: -1, - "scenes": [] as IScene[], - "nodes": [] as INode[], - "cameras": [] as ICamera[], - "lights": [] as ILight[], - "models": [] as IModel[], - "metas": [] as IMeta[], - "setups": [] as ISetup[], - }; - - - function appendScene(scene :DerefScene){ - let iScene :IScene = { - units: scene.units ?? "cm", //This is the default unit as per CVScene.ts - }; - if(scene.name) iScene.name = scene.name; - if(scene.nodes?.length){ - iScene.nodes = scene.nodes.map((node)=>appendNode(node)); - } - for(let key of ["setup", "meta"] as const){ - const value = scene[key]; - if(!value) continue; - let colKey = key+"s" as DocumentCollections; - const idx = document[colKey]!.length; - document[colKey]!.push(value as any); - iScene[key] = idx as any; - } - return document.scenes.push(iScene) - 1; - } - - function appendNode(node :DerefNode){ - - let iNode :INode = {}; - if(node.name) iNode.name = node.name; - - //Push to collection before appending children because that's a more "natural" order to have - const idx = document.nodes.push(iNode) - 1; - - if(node.children?.length){ - iNode.children = node.children.map((child)=>appendNode(child)); - } - - if(node.matrix) iNode.matrix = node.matrix; - if(node.translation) iNode.translation = node.translation; - if(node.rotation) iNode.rotation = node.rotation; - if(node.scale) iNode.scale = node.scale; - - - for(let key of ["camera", "light", "model", "meta"] as const){ - const value = node[key]; - if(!value) continue; - let colKey = key+"s" as DocumentCollections; - const idx = document[colKey].length; - document[colKey].push(value as any); - iNode[key] = idx as any; - } - return idx; - } - - - document.scene = appendScene(scene); - - - //Remove empty collections - for(let key in document){ - let v = document[key as keyof typeof document]; - if(typeof v === "undefined" || (Array.isArray(v) && v.length == 0)) delete document[key as keyof typeof document]; - } - - return document; } \ No newline at end of file diff --git a/source/server/utils/merge/pointers/meta.ts b/source/server/utils/merge/pointers/meta.ts new file mode 100644 index 00000000..cc96b360 --- /dev/null +++ b/source/server/utils/merge/pointers/meta.ts @@ -0,0 +1,61 @@ +import { IDocument } from "../../schema/document.js"; +import { IMeta } from "../../schema/meta.js"; +import { DerefMeta, toIdMap, toUriMap } from "./types.js"; + +export function appendMeta(document :Required, meta :DerefMeta) :number{ + let iMeta :IMeta = {}; + + if(meta.collection) iMeta.collection = meta.collection; + + if(meta.process) iMeta.process = meta.process; + + let images = Object.values(meta.images ?? {}); + if(images.length){ + iMeta.images = images; + } + + let articles = Object.values(meta.articles ?? {}); + if(articles.length){ + iMeta.articles = articles; + } + + let audio = Object.values(meta.audio ?? {}); + if(audio.length){ + iMeta.audio = audio; + } + + if(meta.leadArticle){ + const idx = iMeta.articles?.findIndex((article)=>article.id === meta.leadArticle); + if(idx != -1) iMeta.leadArticle = idx; + else throw new Error(`Lead article ${meta.leadArticle} not found in meta.articles`); + } + + const idx = document.metas.push(iMeta) - 1; + return idx; +} + + +export function mapMeta(iMeta :IMeta) :DerefMeta{ + let meta :DerefMeta = {}; + + if(iMeta.collection) meta.collection = iMeta.collection; + + if(iMeta.process) meta.process = iMeta.process; + + if(iMeta.images?.length){ + meta.images = toUriMap(iMeta.images); + } + if(iMeta.articles?.length){ + meta.articles = toIdMap(iMeta.articles); + } + + if(iMeta.audio?.length){ + meta.audio = toIdMap(iMeta.audio); + } + + if(typeof iMeta.leadArticle === "number"){ + meta.leadArticle = iMeta.articles![iMeta.leadArticle]?.id; + } + + return meta; +} diff --git a/source/server/utils/merge/pointers/model.ts b/source/server/utils/merge/pointers/model.ts new file mode 100644 index 00000000..2056ce89 --- /dev/null +++ b/source/server/utils/merge/pointers/model.ts @@ -0,0 +1,45 @@ +import { IDocument } from "../../schema/document.js"; + +import { IModel } from "../../schema/model.js"; + +import { DerefModel, toIdMap, toUriMap } from "./types.js"; + +export function appendModel(document :Required, {derivatives, annotations, ...model} :DerefModel) :number{ + let iModel :IModel = { + ...model, + derivatives: [], + }; + + for (let derivative in derivatives){ + iModel.derivatives.push({ + ...derivatives[derivative], + assets: Object.values(derivatives[derivative].assets), + }); + } + if(annotations){ + iModel.annotations = Object.values(annotations); + } + + const idx = document.models.push(iModel) - 1; + return idx; +} + + +export function mapModel({annotations, derivatives, ...iModel} :IModel) :DerefModel { + const model = { + ...iModel, + derivatives: {} as DerefModel["derivatives"], + annotations: annotations?toIdMap(annotations): undefined, + } + + for(let derivative of derivatives){ + const id = `${derivative.usage}/${derivative.quality}`; + + model.derivatives[id] = { + ...derivative, + assets: toUriMap(derivative.assets) + }; + } + + return model; +} diff --git a/source/server/utils/merge/pointers/node.ts b/source/server/utils/merge/pointers/node.ts new file mode 100644 index 00000000..7aef7f85 --- /dev/null +++ b/source/server/utils/merge/pointers/node.ts @@ -0,0 +1,47 @@ +import { IDocument, INode } from "../../schema/document.js"; +import { appendMeta } from "./meta.js"; +import { appendModel } from "./model.js"; +import { DerefNode } from "./types.js"; + + +type DocumentCollections = keyof Omit; + +export function appendNode(document :Required, node :DerefNode){ + + let iNode :INode = {}; + if(node.name) iNode.name = node.name; + + if(node.matrix) iNode.matrix = node.matrix; + if(node.translation) iNode.translation = node.translation; + if(node.rotation) iNode.rotation = node.rotation; + if(node.scale) iNode.scale = node.scale; + + if(node.meta){ + iNode.meta = appendMeta(document, node.meta); + } + + if(node.model){ + iNode.model = appendModel(document, node.model); + } + + if(node.camera){ + iNode["camera"] = document["cameras"].push(node.camera) -1; + } + if(node.light){ + iNode["light"] = document["lights"].push(node.light) -1; + } + + if(node.children){ + iNode.children = Object.values(node.children).map((child)=>appendNode(document, child)); + } + + + //Push to collection before appending children because that's a more "natural" order to have + const idx = document.nodes.push(iNode) - 1; + + if(node.children?.length){ + iNode.children = Object.values(node.children).map((child)=>appendNode(document, child)); + } + + return idx; +} diff --git a/source/server/utils/merge/pointers.test.ts b/source/server/utils/merge/pointers/pointers.test.ts similarity index 81% rename from source/server/utils/merge/pointers.test.ts rename to source/server/utils/merge/pointers/pointers.test.ts index c2149d7c..fd55d244 100644 --- a/source/server/utils/merge/pointers.test.ts +++ b/source/server/utils/merge/pointers/pointers.test.ts @@ -2,8 +2,9 @@ import fs from 'fs/promises'; import path from "path"; import { fileURLToPath } from 'url'; -import { fromPointers, toPointers } from "./pointers.js"; -import { IDocument } from '../schema/document.js'; +import { fromPointers, toPointers } from "./index.js"; +import { IDocument } from '../../schema/document.js'; +import { DerefNode } from './types.js'; const thisDir = path.dirname(fileURLToPath(import.meta.url)); @@ -25,145 +26,6 @@ describe("(de)reference pointers", function(){ * because we generally want to test reversibility */ - describe("toPointers()", function(){ - it("dereferences nodes with children", function(){ - const doc = { - ...baseDoc, - nodes: [ - {name: "Lights", children: [1]}, - {name: "L1", light: 0} - ], - scenes:[{nodes: [0]}], - lights: [{type: "ambient"}] - }; - - const deref:any = toPointers(doc as any); - expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); - expect(deref.asset).to.deep.equal(doc.asset); - //use JSON.parse(JSON.stringify()) to remove undefined values - expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({"nodes":[ - {"name":"Lights","children":[{"name":"L1","light":{"type":"ambient"}}]} - ]}); - }); - - it("dereferences scene setup", function(){ - const doc = { - ...baseDoc, - nodes: [], - scenes:[{setup: 0}], - setups: [{language: {language: "FR"}}] - }; - - const deref:any = toPointers(doc as any); - expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); - expect(deref.asset).to.deep.equal(doc.asset); - //use JSON.parse(JSON.stringify()) to remove undefined values - expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({ - "nodes": [], - "setup":{language: {language: "FR"}} - }); - }); - - it("dereferences scene meta", function(){ - const doc = { - ...baseDoc, - nodes: [], - scenes:[{meta: 0}], - metas: [{collection: {titles:{FR: "Hello World!"}}}] - }; - - const deref:any = toPointers(doc as any); - expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); - expect(deref.asset).to.deep.equal(doc.asset); - //use JSON.parse(JSON.stringify()) to remove undefined values - expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({ - "nodes": [], - "meta":{collection: {titles:{FR: "Hello World!"}}} - }); - - }); - - it("dereferences node's meta", function(){ - - const doc = { - ...baseDoc, - nodes: [{ - name: "Model", - model: 0, - meta: 0 - }], - scenes:[{ - nodes: [0] - }], - models: [{uri: "model.gltf"}], - metas: [{collection: {titles:{FR: "Hello World!"}}}] - }; - - const deref:any = toPointers(doc as any); - expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); - expect(deref.asset).to.deep.equal(doc.asset); - //use JSON.parse(JSON.stringify()) to remove undefined values - expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({ - "nodes": [{name: "Model", model: {uri: "model.gltf"}, meta: {collection: {titles:{FR: "Hello World!"}}}}], - }); - - }); - - it("keeps a scene's name and units", function(){ - - const doc = { - ...baseDoc, - scenes:[{ - name: "My Scene", - units: "km", - }], - }; - - const deref:any = toPointers(doc as any); - expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); - expect(deref.asset).to.deep.equal(doc.asset); - //use JSON.parse(JSON.stringify()) to remove undefined values - expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({ - "name": "My Scene", - "units": "km", - "nodes": [], - }); - }); - - describe("throws", function(){ - it("if doc has no scene", function(){ - const doc = {...baseDoc, scenes: []}; - expect(()=> toPointers(doc)).to.throw("Document has no valid scene"); - }); - - it("if doc has an invalid scene index", function(){ - const doc = {...baseDoc, scene: 1}; - expect(()=> toPointers(doc as any)).to.throw("Document's scene #1 is invalid"); - }); - - it("if scene has invalid node index", function(){ - const doc = {...baseDoc, scenes:[{nodes: [1]}], nodes:[{name: "Camera"}]}; - expect(()=> toPointers(doc as any)).to.throw(`Invalid node index 1 in scene #0`); - }); - - it("if node has invalid camera/light/model/meta index", function(){ - "camera/light/model/meta".split("/").forEach(key=>{ - const doc = {...baseDoc, scenes:[{nodes:[0]}], nodes:[{name: key.toUpperCase(), [key]: 0}]}; - expect(()=> toPointers(doc as any)).to.throw(`Invalid ${key} index 0 in node #0`); - }) - }); - - it("if scene has invalid setup index", function(){ - const doc = {...baseDoc, scenes:[{setup: 0}]}; - expect(()=> toPointers(doc as any)).to.throw(`Invalid setup #0 in scene #0`); - }); - - it("if scene has invalid meta index", function(){ - const doc = {...baseDoc, scenes:[{meta: 0}]}; - expect(()=> toPointers(doc as any)).to.throw(`Invalid meta #0 in scene #0`); - }); - }); - }); describe("fromPointers()", function(){ @@ -171,18 +33,23 @@ describe("(de)reference pointers", function(){ const deref = {asset: baseDoc.asset, scene:{ name: "My Scene", units: "km", + nodes: [], }}; - expect(fromPointers(deref as any).scene).to.deep.equal({ + const doc = fromPointers(deref as any); + expect(doc.scene).to.equal(0); + expect(doc.scenes).to.deep.equal([{ name: "My Scene", - units: "km" - }); + units: "km", + }]); }); it("auto-fills a scene's units", function(){ + //Units is supposedly required but might be missing const deref = {asset: baseDoc.asset, scene:{ name: "My Scene", + nodes: [], }}; - expect(fromPointers(deref as any).scene).to.have.property("units", "cm"); + expect((fromPointers(deref as any).scenes as any)[0]).to.have.property("units", "cm"); }); it("builds nodes reference arrays", function(){ @@ -214,15 +81,20 @@ describe("(de)reference pointers", function(){ }], cameras: [{type: "perspective"}], lights: [{type: "ambient"}], - models: [{uri: "model.gltf"}], + models: [{uri: "model.gltf", derivatives: []}], }); }); it("copies nodes matrix/translation/rotation/scale properties", function(){ - const model = { + const node :DerefNode = { name: "Model", - model: {uri: "model.gltf"}, - matrix:[0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0], + model: { + units: "cm", + derivatives:{ + "High/Web3D":{quality: "High", usage: "Web3D", assets:{"model.gltf": {uri: "model.gltf", type:"Model"}}} + } + }, + matrix: [0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0], translation:[0,0,0], rotation: [0, 0, 0, 0], scale: [1, 1, 1], @@ -232,20 +104,29 @@ describe("(de)reference pointers", function(){ asset: baseDoc.asset, scene: { units: "m", - nodes:[model], + nodes:{"Model": node}, } }; const doc:IDocument = fromPointers(deref as any); //use JSON.parse(JSON.stringify()) to remove undefined values - expect(doc).to.deep.equal({ - ...baseDoc, - nodes: [ - {name: "Model", model: 0}, - ], - models:[model], - scenes:[{ nodes: [0], units: "m" }], - }); + expect(doc).to.have.property("nodes").to.deep.equal([ + { + name: "Model", + model: 0, + matrix: node.matrix, + translation: node.translation, + rotation: node.rotation, + scale: node.scale, + }, + ]); + expect(doc).to.have.property("models").to.deep.equal([{ + units: "cm", + derivatives:[{ + quality: "High", usage: "Web3D", + assets:[{uri: "model.gltf", type:"Model"}] + }] + }]); }); @@ -255,6 +136,7 @@ describe("(de)reference pointers", function(){ scene: { units: "m", meta: {collection: {titles:{EN: "Meta Title"}}}, + nodes: [], } }; @@ -302,6 +184,7 @@ describe("(de)reference pointers", function(){ scene: { units: "m", setup: {language: {language: "FR"}}, + nodes: [], } }; @@ -320,17 +203,153 @@ describe("(de)reference pointers", function(){ }); + + describe("toPointers()", function(){ + it("dereferences nodes with children", function(){ + const doc = { + ...baseDoc, + nodes: [ + {name: "Lights", children: [1]}, + {name: "L1", light: 0} + ], + scenes:[{nodes: [0]}], + lights: [{type: "ambient"}] + }; + + const deref:any = toPointers(doc as any); + expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); + expect(deref.asset).to.deep.equal(doc.asset); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({"nodes":{ + "Lights": {"name":"Lights","children":{"L1": {"name":"L1","light":{"type":"ambient"}}}} + }}); + }); + + it("dereferences scene setup", function(){ + const doc = { + ...baseDoc, + nodes: [], + scenes:[{setup: 0}], + setups: [{language: {language: "FR"}}] + }; + + const deref:any = toPointers(doc as any); + expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); + expect(deref.asset).to.deep.equal(doc.asset); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({ + "nodes": {}, + "setup":{language: {language: "FR"}}, + }); + }); + + it("dereferences scene meta", function(){ + const doc = { + ...baseDoc, + nodes: [], + scenes:[{meta: 0}], + metas: [{collection: {titles:{FR: "Hello World!"}}}] + }; + + const deref:any = toPointers(doc as any); + expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); + expect(deref.asset).to.deep.equal(doc.asset); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({ + "nodes": {}, + "meta":{collection: {titles:{FR: "Hello World!"}}} + }); + + }); + + it("dereferences node's meta", function(){ + + const doc = { + ...baseDoc, + nodes: [{ + name: "Model", + meta: 0 + }], + scenes:[{ + nodes: [0] + }], + metas: [{collection: {titles:{FR: "Hello World!"}}}] + }; + + const deref:any = toPointers(doc as any); + expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); + expect(deref.asset).to.deep.equal(doc.asset); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({ + "nodes": {"Model": {name: "Model", meta: {collection: {titles:{FR: "Hello World!"}}}}}, + }); + + }); + + it("keeps a scene's name and units", function(){ + + const doc = { + ...baseDoc, + scenes:[{ + name: "My Scene", + units: "km", + }], + }; + + const deref:any = toPointers(doc as any); + expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); + expect(deref.asset).to.deep.equal(doc.asset); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({ + "name": "My Scene", + "units": "km", + "nodes": {}, + }); + }); + + describe("throws", function(){ + it("if doc has no scene", function(){ + const doc = {...baseDoc, scenes: []}; + expect(()=> toPointers(doc)).to.throw("Document has no valid scene"); + }); + + it("if doc has an invalid scene index", function(){ + const doc = {...baseDoc, scene: 1}; + expect(()=> toPointers(doc as any)).to.throw("Document's scene #1 is invalid"); + }); + + it("if scene has invalid node index", function(){ + const doc = {...baseDoc, scenes:[{nodes: [1]}], nodes:[{name: "Camera"}]}; + expect(()=> toPointers(doc as any)).to.throw(`Invalid node index 1 in scene #0`); + }); + + "camera/light/model/meta".split("/").forEach(key=>{ + it(`if node has invalid ${key} index`, function(){ + const doc = {...baseDoc, scenes:[{nodes:[0]}], nodes:[{name: key.toUpperCase(), [key]: 0}]}; + expect(()=> toPointers(doc as any)).to.throw(`Invalid ${key} index 0 in node #0`); + }); + }) + + + + it("if scene has invalid setup index", function(){ + const doc = {...baseDoc, scenes:[{setup: 0}]}; + expect(()=> toPointers(doc as any)).to.throw(`Invalid setup #0 in scene #0`); + }); + + it("if scene has invalid meta index", function(){ + const doc = {...baseDoc, scenes:[{meta: 0}]}; + expect(()=> toPointers(doc as any)).to.throw(`Invalid meta #0 in scene #0`); + }); + }); + }); + + describe("is reversible", function(){ const paths:Record = [ "01_simple.svx.json", - ].reduce((paths, file)=>({...paths, [file]: path.resolve(thisDir, "../../__test_fixtures/documents/"+file)}),{}); + ].reduce((paths, file)=>({...paths, [file]: path.resolve(thisDir, "../../../__test_fixtures/documents/"+file)}),{}); - it("on default.svx.json", async function(){ - const doc = JSON.parse(await fs.readFile(path.resolve(thisDir, "../schema/default.svx.json"), "utf8")); - const deref = toPointers(doc); - const redoc = fromPointers(deref); - expect(redoc).to.deep.equal(doc); - }); describe("01_simple.svx.json", function(){ let docString :string, doc :IDocument; @@ -347,13 +366,14 @@ describe("(de)reference pointers", function(){ expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); expect(deref.asset).to.deep.equal(doc.asset); expect(Object.keys(deref.scene)).to.deep.equal(["name", "units", "nodes", "setup"]); - expect(deref.scene.nodes).to.have.property("length", 3); - expect(deref.scene.nodes[2]).to.have.property("meta").to.deep.equal({collection: {titles:{EN: "Meta Title"}}}); + expect(Object.keys(deref.scene.nodes)).to.have.property("length", 3); + expect(deref.scene.nodes["Model Name"]).to.have.property("meta").to.deep.equal({collection: {titles:{EN: "Meta Title"}}}); }); it("dereferencing is indempotent", async function(){ const deref = toPointers(doc); const redoc = fromPointers(deref); + console.log(JSON.stringify(redoc, null, 2)); expect(redoc).to.deep.equal(doc); }); diff --git a/source/server/utils/merge/pointers/scene.ts b/source/server/utils/merge/pointers/scene.ts new file mode 100644 index 00000000..1620975f --- /dev/null +++ b/source/server/utils/merge/pointers/scene.ts @@ -0,0 +1,31 @@ +import { IDocument, IScene } from "../../schema/document.js"; +import { appendMeta } from "./meta.js"; +import { appendNode } from "./node.js"; +import { appendSetup } from "./setup.js"; + +import { DerefScene } from "./types.js"; + + + +export function appendScene(document :Required, scene :DerefScene){ + let iScene :IScene = { + units: scene.units ?? "cm", //This is the default unit as per CVScene.ts + }; + if(scene.name) iScene.name = scene.name; + + const nodes = Object.values(scene.nodes ?? {}); + if(nodes.length){ + iScene.nodes = nodes.map((node)=>appendNode(document, node)); + } + + + if(scene.setup){ + iScene.setup = appendSetup(document, scene.setup); + } + + if(scene.meta){ + iScene.meta = appendMeta(document, scene.meta); + } + + return document.scenes.push(iScene) - 1; +} diff --git a/source/server/utils/merge/pointers/setup.ts b/source/server/utils/merge/pointers/setup.ts new file mode 100644 index 00000000..62089b36 --- /dev/null +++ b/source/server/utils/merge/pointers/setup.ts @@ -0,0 +1,44 @@ +import { IDocument } from "../../schema/document.js"; +import { ISetup, ITour } from "../../schema/setup.js"; +import uid from "../../uid.js"; +import { DerefSetup, DerefSnapshots, IdMap } from "./types.js"; + +export function appendSetup(document :Required, {tours: toursMap, snapshots, ...setup} :DerefSetup) :number{ + let iSetup :ISetup = {...setup}; + + const tours = Object.values(toursMap ?? {}); + if(tours.length){ + iSetup.tours = tours; + } + if(snapshots){ + iSetup.snapshots = { + ...snapshots, + states: Object.values(snapshots.states ?? {}), + }; + } + + const idx = document.setups.push(iSetup) - 1; + return idx; +} + + +export function mapSetup({tours, snapshots, ...iSetup} :ISetup) :DerefSetup { + const setup = { + ...iSetup, + tours: (tours?.length ? {} : undefined) as IdMap, + snapshots: undefined as any as DerefSnapshots, + } + + for(let tour of tours??[]){ + tour.id ??= uid(); + setup.tours[tour.id] = tour as ITour & {id: string}; + } + setup.snapshots = ((snapshots)? { + ...snapshots, + states: {} as DerefSnapshots["states"], + } as DerefSnapshots : undefined as any); + for(let state of snapshots?.states??[]){ + setup.snapshots.states[state.id] = state; + } + return setup; +} diff --git a/source/server/utils/merge/types.ts b/source/server/utils/merge/pointers/types.ts similarity index 59% rename from source/server/utils/merge/types.ts rename to source/server/utils/merge/pointers/types.ts index 903694f5..e1b37576 100644 --- a/source/server/utils/merge/types.ts +++ b/source/server/utils/merge/pointers/types.ts @@ -1,9 +1,9 @@ -import { IDocumentAsset, IScene, INode, ICamera, ILight, Matrix4, Quaternion, Vector3 } from "../schema/document.js"; -import { IArticle, IAudioClip, IImage, IMeta } from "../schema/meta.js"; -import { IModel, TUnitType } from "../schema/model.js"; -import { IBackground, IEnvironment, IFloor, IGrid, IInterface, ILanguage, INavigation, IReader, ISetup, ISlicer, ISnapshots, ITape, ITour, ITourStep, ITours, IViewer } from "../schema/setup.js"; -import { Dictionary, Index } from "../schema/types.js"; +import { IDocumentAsset, ICamera, ILight, Matrix4, Quaternion, Vector3 } from "../../schema/document.js"; +import { IArticle, IAudioClip, IImage, IMeta } from "../../schema/meta.js"; +import { IAnnotation, IAsset, IDerivative, IModel, TUnitType } from "../../schema/model.js"; +import { ISetup, ITour, ITourStep } from "../../schema/setup.js"; +import { Dictionary, Index } from "../../schema/types.js"; /** * Special symbol to mark a field for deletion in a diff object @@ -20,7 +20,7 @@ export type Diff = { export interface DerefScene{ name?: string; nodes: NameMap; - setup?: ISetup; + setup?: DerefSetup; meta?: DerefMeta; units: TUnitType; } @@ -39,7 +39,7 @@ export interface DerefNode { camera?: ICamera; light?: ILight; - model?: IModel; + model?: DerefModel; meta?: DerefMeta; } @@ -70,46 +70,72 @@ export interface DerefSnapshots{ }>; } -export interface DerefMeta { - collection?: Dictionary; - process?: Dictionary; +export interface DerefMeta extends Omit { images?: UriMap; articles?: IdMap; audio?: IdMap; - leadArticle?: Index; + leadArticle?: string; } -export interface DerefSetup -{ - interface?: IInterface; - viewer?: IViewer; - reader?: IReader; - navigation?: INavigation; - background?: IBackground; - environment?: IEnvironment, - language?: ILanguage, - floor?: IFloor; - grid?: IGrid; - tape?: ITape; - slicer?: ISlicer; +export interface DerefSetup extends Omit{ tours?: IdMap; snapshots?: DerefSnapshots; } +export interface DerefModel extends Omit{ + derivatives: AbstractMap; + annotations?: IdMap; +} + +export interface DerefDerivative extends Omit{ + assets: UriMap; +} + //////// // Maps are replacing arrays of identified nodes with a map-by-id + +export type AbstractMap = { + [id: string]: T; +} + export type IdMap = { [id: string]: T; } +export function toIdMap(arr :T[]) :IdMap{ + const map = {} as IdMap; + for(let item of arr){ + map[item.id] = item; + } + return map; +} + + export type NameMap = { [name: string]: T; } +export function toNameMap(arr :T[]) :NameMap{ + const map = {} as NameMap; + for(let item of arr){ + map[item.name] = item; + } + return map; +} + + export type UriMap = { [uri: string]: T; } +export function toUriMap(arr :T[]) :UriMap{ + const map = {} as UriMap; + for(let item of arr){ + map[item.uri] = item; + } + return map; +} + /** * A document where all indexed references were replaced by pointers to the actual objects. * @see DerefNode for nodes @@ -119,4 +145,4 @@ export interface DerefDocument { asset: IDocumentAsset; scene: DerefScene; -} \ No newline at end of file +} diff --git a/source/server/utils/schema/document.d.ts b/source/server/utils/schema/document.d.ts index 34be4051..8d59f441 100644 --- a/source/server/utils/schema/document.d.ts +++ b/source/server/utils/schema/document.d.ts @@ -15,12 +15,12 @@ * limitations under the License. */ -import { Index } from "./types.ts"; +import { Index } from "./types.js"; -import { EUnitType, TUnitType, Vector3, Quaternion, Matrix4, ColorRGB } from "./common.ts"; -import { IMeta } from "./meta.ts"; -import { IModel } from "./model.ts"; -import { ISetup } from "./setup.ts"; +import { EUnitType, TUnitType, Vector3, Quaternion, Matrix4, ColorRGB } from "./common.js"; +import { IMeta } from "./meta.js"; +import { IModel } from "./model.js"; +import { ISetup } from "./setup.js"; //////////////////////////////////////////////////////////////////////////////// diff --git a/source/server/utils/schema/model.d.ts b/source/server/utils/schema/model.d.ts index 96e5d60a..dd6ddc23 100644 --- a/source/server/utils/schema/model.d.ts +++ b/source/server/utils/schema/model.d.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { Dictionary } from "./types.js"; +import { Dictionary } from "./types.ts"; import { ColorRGBA, EUnitType, TUnitType, Vector3, Vector4 } from "./common.js"; //////////////////////////////////////////////////////////////////////////////// diff --git a/source/server/utils/schema/setup.d.ts b/source/server/utils/schema/setup.d.ts index 13ef59fe..09fb5715 100644 --- a/source/server/utils/schema/setup.d.ts +++ b/source/server/utils/schema/setup.d.ts @@ -1,5 +1,5 @@ import { Dictionary } from "./types.ts"; -import { ELanguageType, TLanguageType } from "./common.ts"; +import { ELanguageType, TLanguageType } from "./common.js"; /** * 3D Foundation Project diff --git a/source/server/utils/uid.ts b/source/server/utils/uid.ts index 35bf543f..81bda489 100644 --- a/source/server/utils/uid.ts +++ b/source/server/utils/uid.ts @@ -12,13 +12,16 @@ for (let i = 0; i < alphabet.length; i++) { /** * Creates a, encoded globally unique identifier with a default length of 12 characters. * The identifier only uses letters and digits and can safely be used for file names. - * this is NOT a proper base 56 encoding. - * While the result should be crypto-secure + * this is NOT a proper base 56 encoding. + * It can be used in place of [@ff-core.uniqueId()](https://github.com/Smithsonian/blob/master/ff-core/source/uniqueId.ts) where necessary, + * though it does not use the same aplphabet. + * [source inspiration](https://codegolf.stackexchange.com/questions/1620/arbitrary-base-conversion/21672#21672) + * * @param length Number of characters in the identifier. * @returns Globally unique identifier - * @see https://codegolf.stackexchange.com/questions/1620/arbitrary-base-conversion/21672#21672 + * */ - export default function uid(size :number = 12){ + export default function uid(size :number = 12) :string{ let data = Array.from(crypto.randomBytes(size));