From f005f3d5b20ea198fa0547d0104c3d5180137415 Mon Sep 17 00:00:00 2001 From: Sebastien DUMETZ Date: Mon, 8 Jan 2024 16:28:02 +0100 Subject: [PATCH] [WIP] structured merge of scene files by pointer index dereferencing --- .../documents/01_simple.svx.json | 78 +++++ source/server/routes/scenes/put/document.ts | 2 +- source/server/utils/merge.test.ts | 246 --------------- .../server/utils/{merge.ts => merge/apply.ts} | 63 +--- source/server/utils/merge/diff.ts | 48 +++ source/server/utils/merge/index.ts | 26 ++ source/server/utils/merge/merge.test.ts | 186 +++++++++++ source/server/utils/merge/pointers.test.ts | 289 ++++++++++++++++++ source/server/utils/merge/pointers.ts | 152 +++++++++ source/server/utils/merge/types.ts | 55 ++++ source/server/utils/schema/default.svx.json | 3 +- 11 files changed, 841 insertions(+), 307 deletions(-) create mode 100644 source/server/__test_fixtures/documents/01_simple.svx.json delete mode 100644 source/server/utils/merge.test.ts rename source/server/utils/{merge.ts => merge/apply.ts} (52%) create mode 100644 source/server/utils/merge/diff.ts create mode 100644 source/server/utils/merge/index.ts create mode 100644 source/server/utils/merge/merge.test.ts create mode 100644 source/server/utils/merge/pointers.test.ts create mode 100644 source/server/utils/merge/pointers.ts create mode 100644 source/server/utils/merge/types.ts diff --git a/source/server/__test_fixtures/documents/01_simple.svx.json b/source/server/__test_fixtures/documents/01_simple.svx.json new file mode 100644 index 00000000..d9c41117 --- /dev/null +++ b/source/server/__test_fixtures/documents/01_simple.svx.json @@ -0,0 +1,78 @@ +{ + "asset": { + "type": "application/si-dpo-3d.document+json", + "version": "1.0", + "copyright": "(c) Holusion SAS, all rights reserved", + "generator": "Voyager" + }, + "scene": 0, + "scenes": [{ + "nodes": [0, 1, 3], + "setup": 0, + "units": "cm" + }], + "nodes": [{ + "camera": 0 + }, { + "name": "Lights", + "children": [2] + }, { + "translation": [0, 0, 2], + "rotation": [0.1, 0.2, 0.2, 0.8], + "scale": [1, 1, 1], + "name": "Key", + "light": 0 + }, { + "name": "Model Name", + "model": 0, + "meta": 0 + }], + "cameras": [{ + "type": "perspective", + "perspective": { + "yfov": 52, + "znear": 0.1, + "zfar": 100000 + } + }], + "lights": [{ + "color": [1, 0.95, 0.9], + "intensity": 1, + "type": "directional", + "shadowEnabled": true + }], + "models": [{ + "units": "mm", + "boundingBox": { + "min": [10, 10, 10], + "max": [10, 10, 10] + }, + "derivatives": [ + { + "usage": "Web3D", + "quality": "High", + "assets": [ + { + "uri": "models/foo.glb", + "type": "Model", + "byteSize": 150, + "numFaces": 25, + "imageSize": 1 + } + ] + } + ] + }], + "metas":[ + { + "collection": { + "titles": { + "EN": "Meta Title" + } + } + } + ], + "setups": [{ + "units": "cm" + }] +} diff --git a/source/server/routes/scenes/put/document.ts b/source/server/routes/scenes/put/document.ts index 4f3a2a3e..fab5648e 100644 --- a/source/server/routes/scenes/put/document.ts +++ b/source/server/routes/scenes/put/document.ts @@ -5,7 +5,7 @@ import { Request, Response } from "express"; import { getUserId, getVfs } from "../../../utils/locals.js"; import { BadRequestError } from "../../../utils/errors.js"; -import * as merge from "../../../utils/merge.js"; +import * as merge from "../../../utils/merge/index.js"; /** * Simple case of PUT /scenes/:scene/scene.svx.json diff --git a/source/server/utils/merge.test.ts b/source/server/utils/merge.test.ts deleted file mode 100644 index 42185d84..00000000 --- a/source/server/utils/merge.test.ts +++ /dev/null @@ -1,246 +0,0 @@ -import {DELETE_KEY, apply, diff} from "./merge.js"; - - - -describe("merge.diff()", function(){ - describe("handle native data types ", function(){ - [ //All JSON-serializable data types with possible bounday values - 1, - 0, - "1", - "", - true, - false, - null, - {}, - {a:1}, - [], - ].forEach(v=>{ - const v_clone :typeof v = JSON.parse(JSON.stringify(v)); - it(`${JSON.stringify(v)} deep equals itself`, function(){ - expect(diff({v}, {v:v_clone})).to.deep.equal({}); - }); - it(`${JSON.stringify({v})} compares from {}`, function(){ - expect(diff({}, {v})).to.deep.equal({v:v_clone}); - }); - it(`${JSON.stringify({v})} compares from {}`, function(){ - expect(diff({v}, {})).to.deep.equal({v:DELETE_KEY}); - }); - }); - }); - - describe("arrays", function(){ - //Arrays behave like objects with numeric keys, with some specific handling - - it("handles an array equality", function(){ - expect(diff({a:["foo"]}, {a:["foo"]})).to.deep.equal({}); - }); - - it("handles nested objects equality", function(){ - expect(diff({a:[{v: "foo"}]}, {a:[{v:"foo"}]})).to.deep.equal({}); - }); - - it("handle array.push", function(){ - expect(diff({a:[]}, {a:[1]})).to.deep.equal({a:{0:1}}); - expect(diff({a:[1]}, {a:[1, 2]})).to.deep.equal({a:{1:2}}); - }); - - it("handle Array.pop", function(){ - expect(diff({a:[1]}, {a:[]})).to.deep.equal({a:{0: DELETE_KEY}}); - expect(diff({a:[1, 2]}, {a:[1]})).to.deep.equal({a:{1: DELETE_KEY}}); - }); - - it("handle Array.replace", function(){ - expect(diff({a:["foo", "bar"]}, {a:["foo", "baz"]})).to.deep.equal({a:{1: "baz"}}); - }); - - it("handle Array.splice", function(){ - // Might be improved to detect a splice and use special syntax to represent it - expect(diff({a:["foo", "bar", "baz"]}, {a:["foo", "baz"]})).to.deep.equal({a:{1: "baz", 2: DELETE_KEY}}); - }); - - it("handle deep changes", function(){ - expect(diff({a:[{v:"foo"}, {v:"bar"}]}, {a:[{v:"foo"}, {v:"baz"}]})).to.deep.equal({a:{1:{v:"baz"}}}); - }); - - it("throws when diffing an array with an object", function(){ - expect(()=>diff({a:[]}, {a:{}})).to.throw("Can't diff an array with an object"); - }); - - - }); -}); - - -describe("merge.apply()", function(){ - it("merges a no-op", function(){ - const a = {a:1}; - expect(apply(a, {})).to.deep.equal(a); - }); - - it("merges a deleted key", function(){ - expect(apply({a:1, b:2}, {b:DELETE_KEY})).to.deep.equal({a:1}); - }); - - it("merges an updated key", function(){ - expect(apply({a:1, b:2}, {b:3})).to.deep.equal({a:1, b:3}); - }); - it("merges nested objects", function(){ - expect(apply({a:{b:1}, c:3}, {a:{b:2}})).to.deep.equal({a:{b:2}, c:3}); - }); - it("merges updated arrays", function(){ - expect(apply({a:[1,2]}, {a:{0:1, 1:3}})).to.deep.equal({a:[1,3]}); - }); - - -}); - - -describe("three-way merge", function(){ - /* This is the real point of this module - * For consistency, tests should follow the following naming scheme: - * a *ref* as the common source, - * a *current* as the document that have been saved in-between - * a *next* as the document that is being saved now - */ - - it("string update", function(){ - /** @TODO could try to do string splicing for finer results */ - const ref = {greet:"Hello", name:"World"}; - const current = {greet:"Hi", name:"World"}; - const next = {greet:"Hello", name:"Universe"}; - expect(apply(current, diff(ref, next))).to.deep.equal({greet:"Hi", name:"Universe"}); - }); - - it.skip("ignores camera translation/rotation changes", function(){ - //This is a potential optimization - //The camera node's position can safely be ignored as it will get reset on load, in favor of `scene.setups[].navigation` - }); - - describe("fields ignore optimization", function(){ - //This is a potential optimization - //The camera node's position can safely be ignored as it will get reset on load, in favor of `scene.setups[].navigation` - it("ignores camera translation changes", function(){ - const ref = {nodes:[{name:"camera", translation:[1,0,0]}]}; - const next = {nodes:[{name:"camera", translation:[0,1,0]}]}; - expect(apply(ref, diff(ref, next))).to.deep.equal({nodes:[{name:"camera", translation:[1,0,0]}]}); - }); - }) - - describe("conflict resolution", function(){ - //Test only cases where "smart" conflict resolution is possible - describe("by ID", function(){ - //Here it would be possible to differentiate annotations or articles by their id - //thus detecting a double-push and resolving it - let A1: any, A2: any, A3 : any; - beforeEach(function(){ - A1 = { "id": "mMzG2tLjmIst", "titles": { "EN": "A1"} } - A2 = { "id": "FiIaONzRIbL4", "titles": { "EN": "A2"} } - A3 = { "id": "rJpCltJjxyyL", "titles": { "EN": "A3"} } - }); - - it("handle double array push (annotations)", function(){ - const ref = {annotations: [A1]}; - const current = {annotations: [A1, A2]}; - const next = {annotations: [A1, A3]}; - - const d = diff(ref, next); - expect(d).to.deep.equal({annotations: {1: A3}}); - expect(apply(current, d)).to.deep.equal({annotations: [A1, A2, A3]}); - }); - - it("can still modify annotations in place", function(){ - const A1b = {"id": "mMzG2tLjmIst", "titles": { "EN": "A1b", "FR": "A1b"} } - const ref = {annotations: [A1]}; - const current = {annotations: [A1, A2]}; - const next = {annotations: [A1b]}; - expect(apply(current, diff(ref, next))).to.deep.equal({annotations: [A1b, A2]}); - - }) - - it("handle double array push (articles)", function(){ - const ref = {articles: [A1]}; - const current = {articles: [A1, A2]}; - const next = {articles: [A1, A3]}; - const d = diff(ref, next); - expect(d).to.deep.equal({articles: {1: A3}}); - const patch = apply(current, d) - expect(patch, JSON.stringify(patch)).to.deep.equal({articles: [A1, A2, A3]}); - }); - }) - - describe("by name", function(){ - - it("reassign scene nodes appropriately", function(){ - //Differentiate nodes by their names - //However, we also need to handle possible conflict in `scenes[x].nodes` if we reorder nodes. - const N1 = { "name": "n1"}; - const N2 = { "name": "n2"}; - const N3 = { "name": "n3"}; - const ref = {nodes: [N1], scenes:[{nodes:[0]}] }; - const current = {nodes: [N1,N2], scenes:[{nodes:[0, 1]}]}; - const next = {nodes: [N1,N3], scenes:[{nodes:[0, 1]}]}; - - expect(apply(current, diff(ref, next))).to.deep.equal({nodes: [N1,N2,N3], scenes:[{nodes:[0, 1, 2]}]}); - }); - - it("reassign nodes children appropriately", function(){ - const common = [ - {name: "Parent", children: [1,2]}, - {name: "Child 1"}, - {name: "Child 2"}, - ] - const parent2 = { "name": "Parent 2"}; - const child3 = { "name": "Child 3"}; - const ref = {nodes: common, scenes:[{nodes:[0]}] }; - const current = {nodes: [...common, parent2], scenes:[{nodes:[0, 3]}]}; - const next = {nodes: [{name:"Parent", children:[1,2,3] }, ...common.slice(1), child3], scenes:[{nodes:[0]}]}; - - expect(apply(current, diff(ref, next))).to.deep.equal({nodes: [ - {name:"Parent", children:[1,2,4] }, - common[1], - common[2], - parent2, - child3, - ], scenes:[{nodes:[0, 3]}]}); - }); - - ["lights", "cameras", "models"].forEach(type=>{ - it(`reassign ${type} appropriately`, function(){ - const stype = type.slice(0, -1); - function make(nodes: object[]=[], types: object[]=[]){ - return { - nodes: [ - {name: `${stype}.0`, [stype]: 0}, - ...nodes, - ], [type]:[ - {reference: "default item"}, - [...types] - ] - }; - } - //Lights, cameras and models have no property that could be used to differentiate them - //We can only suppose that if someone adds a node, the light attached to this node is unique to this one and should stay with it. - const ref = make(); - const current = make([{name: `${stype}.current`, [stype]: 1}], [{reference: "current_item"}]); - const next = make([{name: `${stype}.next`, [stype]: 1}], [{reference: "next_item"}]); - - expect(apply(current, diff(ref, next))).to.deep.equal({ - nodes: [ - {name: `${stype}.0`, [stype]: 0}, - {name: `${stype}.current`, [stype]: 1}, - {name: `${stype}.next`, [stype]: 2}, - ], [type]:[ - {reference: "default item"}, - {reference: "current_item"}, - {reference: "next_item"}, - ] - }); - - }); - }); - }); - - - }); -}); diff --git a/source/server/utils/merge.ts b/source/server/utils/merge/apply.ts similarity index 52% rename from source/server/utils/merge.ts rename to source/server/utils/merge/apply.ts index cbe9888e..8cab1f85 100644 --- a/source/server/utils/merge.ts +++ b/source/server/utils/merge/apply.ts @@ -1,13 +1,5 @@ - -/** - * Special symbol to mark a field for deletion in a diff object - */ -export const DELETE_KEY = Symbol("_DELETE_KEY"); - -type Diff = { - [K in keyof T]?: typeof DELETE_KEY|Diff|T[K]|Record; -} - +'use strict'; +import {Diff, DELETE_KEY} from "./types.js"; /** * Deep assign two or more objects @@ -16,7 +8,7 @@ type Diff = { * @param into Object to merge into (will be mutated in-place) * @returns into, merged with source(s) */ -export function apply>(into :T, ...diffs :Diff[]):T{ +export default function apply>(into :T, ...diffs :Diff[]):T{ for(const diff of diffs){ for(const key in diff){ const value = diff[key] as T[Extract]; @@ -81,51 +73,4 @@ function apply_core>(into :T, key:keyof T, value : } return false; -} - -/** - * Computes a diff between two objects. - * Applying deepMerge(a, diff(a,b)) should yield b - * deleted keys are represented using the DELETE_KEY symbol - * @param from origin document - * @param to target document - * @returns - */ -export function diff>(from :T, to :T) :Diff{ - const is_array = Array.isArray(from); - if(is_array && !Array.isArray(to)) throw new Error("Can't diff an array with an object"); - - let r :Diff = {} as any; - const keys :Set= new Set([...Object.keys(from), ...Object.keys(to)]); - for(const key of keys.values()){ - - if(typeof to[key] == "undefined"){ - r[key] = DELETE_KEY; - continue; - } - - if(typeof from[key] != "object"){ - if(from[key] === to[key]) continue; - //Simple case with primitive values - //console.log("Assigning ", key, to[key]); - r[key] = to[key]; - continue; - } - - //Handle cases where `typeof from[key] === "object"` - if(to[key] == null){ - //Null is special because it can't be fed back to diff() recursively - if(from[key] != null ) r[key] = null as T[Extract]; - continue; - } - - const d = diff(from[key] as any, to[key] as any); - if(Object.keys(d).length){ - //console.log("Diffing", key, from[key], to[key]); - r[key] = d; - } - - - } - return r; -} +} \ No newline at end of file diff --git a/source/server/utils/merge/diff.ts b/source/server/utils/merge/diff.ts new file mode 100644 index 00000000..25983046 --- /dev/null +++ b/source/server/utils/merge/diff.ts @@ -0,0 +1,48 @@ +import {Diff, DELETE_KEY} from "./types.js"; + +/** + * Computes a diff between two objects. + * Applying deepMerge(a, diff(a,b)) should yield b + * deleted keys are represented using the DELETE_KEY symbol + * @param from origin document + * @param to target document + * @returns + */ +export default function diff>(from :T, to :T) :Diff{ + const is_array = Array.isArray(from); + if(is_array && !Array.isArray(to)) throw new Error("Can't diff an array with an object"); + + let r :Diff = {} as any; + const keys :Set= new Set([...Object.keys(from), ...Object.keys(to)]); + for(const key of keys.values()){ + + if(typeof to[key] == "undefined"){ + r[key] = DELETE_KEY; + continue; + } + + if(typeof from[key] != "object"){ + if(from[key] === to[key]) continue; + //Simple case with primitive values + //console.log("Assigning ", key, to[key]); + r[key] = to[key]; + continue; + } + + //Handle cases where `typeof from[key] === "object"` + if(to[key] == null){ + //Null is special because it can't be fed back to diff() recursively + if(from[key] != null ) r[key] = null as T[Extract]; + continue; + } + + const d = diff(from[key] as any, to[key] as any); + if(Object.keys(d).length){ + //console.log("Diffing", key, from[key], to[key]); + r[key] = d; + } + + + } + return r; +} diff --git a/source/server/utils/merge/index.ts b/source/server/utils/merge/index.ts new file mode 100644 index 00000000..9c8efadc --- /dev/null +++ b/source/server/utils/merge/index.ts @@ -0,0 +1,26 @@ + +import {Diff, DELETE_KEY, DerefDocument} from "./types.js"; +import apply from "./apply.js"; +import diff from "./diff.js"; +import { IDocument } from "../schema/document.js"; +import { fromPointers, toPointers } from "./pointers.js"; + +export { + Diff, + DELETE_KEY, + apply, + diff +}; + +/** + * like `diff()` but dereferences a document's pointers first to make a cleaner diff + * @returns + */ +export function diffDoc(from:IDocument,to:IDocument){ + return diff( toPointers(from), toPointers(to)); +} + + +export function applyDoc(from:IDocument, diff:Diff){ + return fromPointers(apply(toPointers(from), diff)); +} diff --git a/source/server/utils/merge/merge.test.ts b/source/server/utils/merge/merge.test.ts new file mode 100644 index 00000000..b0741c70 --- /dev/null +++ b/source/server/utils/merge/merge.test.ts @@ -0,0 +1,186 @@ +import fs from "fs/promises"; +import path from "path"; +import { fileURLToPath } from 'url'; + +const thisDir = path.dirname(fileURLToPath(import.meta.url)); + + + +import { IDocument, INode } from "../schema/document.js"; +import {DELETE_KEY, apply, applyDoc, diff} from "./index.js"; + + + +describe("merge.diff()", function(){ + describe("handle native data types ", function(){ + [ //All JSON-serializable data types with possible bounday values + 1, + 0, + "1", + "", + true, + false, + null, + {}, + {a:1}, + [], + ].forEach(v=>{ + const v_clone :typeof v = JSON.parse(JSON.stringify(v)); + it(`${JSON.stringify(v)} deep equals itself`, function(){ + expect(diff({v}, {v:v_clone})).to.deep.equal({}); + }); + it(`${JSON.stringify({v})} compares from {}`, function(){ + expect(diff({}, {v})).to.deep.equal({v:v_clone}); + }); + it(`${JSON.stringify({v})} compares from {}`, function(){ + expect(diff({v}, {})).to.deep.equal({v:DELETE_KEY}); + }); + }); + }); + + describe("arrays", function(){ + //Arrays behave like objects with numeric keys, with some specific handling + + it("handles an array equality", function(){ + expect(diff({a:["foo"]}, {a:["foo"]})).to.deep.equal({}); + }); + + it("handles nested objects equality", function(){ + expect(diff({a:[{v: "foo"}]}, {a:[{v:"foo"}]})).to.deep.equal({}); + }); + + it("handle array.push", function(){ + expect(diff({a:[]}, {a:[1]})).to.deep.equal({a:{0:1}}); + expect(diff({a:[1]}, {a:[1, 2]})).to.deep.equal({a:{1:2}}); + }); + + it("handle Array.pop", function(){ + expect(diff({a:[1]}, {a:[]})).to.deep.equal({a:{0: DELETE_KEY}}); + expect(diff({a:[1, 2]}, {a:[1]})).to.deep.equal({a:{1: DELETE_KEY}}); + }); + + it("handle Array.replace", function(){ + expect(diff({a:["foo", "bar"]}, {a:["foo", "baz"]})).to.deep.equal({a:{1: "baz"}}); + }); + + it("handle Array.splice", function(){ + // Might be improved to detect a splice and use special syntax to represent it + expect(diff({a:["foo", "bar", "baz"]}, {a:["foo", "baz"]})).to.deep.equal({a:{1: "baz", 2: DELETE_KEY}}); + }); + + it("handle deep changes", function(){ + expect(diff({a:[{v:"foo"}, {v:"bar"}]}, {a:[{v:"foo"}, {v:"baz"}]})).to.deep.equal({a:{1:{v:"baz"}}}); + }); + + it("throws when diffing an array with an object", function(){ + expect(()=>diff({a:[]}, {a:{}})).to.throw("Can't diff an array with an object"); + }); + + + }); +}); + + +describe("merge.apply()", function(){ + it("merges a no-op", function(){ + const a = {a:1}; + expect(apply(a, {})).to.deep.equal(a); + }); + + it("merges a deleted key", function(){ + expect(apply({a:1, b:2}, {b:DELETE_KEY})).to.deep.equal({a:1}); + }); + + it("merges an updated key", function(){ + expect(apply({a:1, b:2}, {b:3})).to.deep.equal({a:1, b:3}); + }); + it("merges nested objects", function(){ + expect(apply({a:{b:1}, c:3}, {a:{b:2}})).to.deep.equal({a:{b:2}, c:3}); + }); + it("merges updated arrays", function(){ + expect(apply({a:[1,2]}, {a:{0:1, 1:3}})).to.deep.equal({a:[1,3]}); + }); + +}); + +describe("merge.diffDoc()", function(){ + +}) + + +describe("three-way merge", function(){ + /* + * For consistency, tests should follow the following naming scheme: + * a *ref* as the common source, + * a *current* as the document that have been saved in-between + * a *next* as the document that is being saved now + */ + + it("string update", function(){ + /** @TODO could try to do string splicing for finer results */ + const ref = {greet:"Hello", name:"World"}; + const current = {greet:"Hi", name:"World"}; + const next = {greet:"Hello", name:"Universe"}; + expect(apply(current, diff(ref, next))).to.deep.equal({greet:"Hi", name:"Universe"}); + }); + + it.skip("ignores camera translation/rotation changes", function(){ + //This is a potential optimization + //The camera node's position can safely be ignored as it will get reset on load, in favor of `scene.setups[].navigation` + }); + + describe("fields ignore optimization", function(){ + //This is a potential optimization + //The camera node's position can safely be ignored as it will get reset on load, in favor of `scene.setups[].navigation` + it("ignores camera translation changes", function(){ + const ref = {nodes:[{name:"camera", translation:[1,0,0]}]}; + const next = {nodes:[{name:"camera", translation:[0,1,0]}]}; + expect(apply(ref, diff(ref, next))).to.deep.equal({nodes:[{name:"camera", translation:[1,0,0]}]}); + }); + }); +}); + + +describe("merge documents", function(){ + let docString :string, doc :IDocument; + this.beforeAll(async function(){ + docString = await fs.readFile(path.resolve(thisDir, "../../__test_fixtures/documents/01_simple.svx.json"), "utf8"); + }); + this.beforeEach(function(){ + doc = JSON.parse(docString); + }); + it("merge simple document changes", function(){ + const current = JSON.parse(docString); + current.lights.push({type:"ambient"}); + + current.nodes.push({ + "name": "Ambiant Light", + "light": 1, + }); + current.nodes.find( (n:INode) =>n.name=="Lights").children.push(current.nodes.length-1); + + + const next = JSON.parse(docString); + next.lights.push({type:"directional"}); + + next.nodes.push({ + "name": "Directional Light", + "light": 1, + }); + next.nodes.find( (n:INode) =>n.name=="Lights").children.push(next.nodes.length-1); + const d = diff(doc, next); + console.log("Diff : ", d); + const result = applyDoc(current, d); + expect(result.nodes).to.deep.equal([ + ...(doc as any).nodes, + { + "name": "Ambiant Light", + "light": 1, + }, + { + "name": "Directional Light", + "light": 1, + } + ]); + }) +}) \ No newline at end of file diff --git a/source/server/utils/merge/pointers.test.ts b/source/server/utils/merge/pointers.test.ts new file mode 100644 index 00000000..8ee0b830 --- /dev/null +++ b/source/server/utils/merge/pointers.test.ts @@ -0,0 +1,289 @@ +import fs from 'fs/promises'; +import path from "path"; +import { fileURLToPath } from 'url'; + +import { fromPointers, toPointers } from "./pointers.js"; +import { IDocument } from '../schema/document.js'; + + +const thisDir = path.dirname(fileURLToPath(import.meta.url)); + +const baseDoc = { + asset: { + type: "application/si-dpo-3d.document+json", + version: "1" + }, + scene: 0, + scenes:[{}] +} as const; + + +describe("(de)reference pointers", function(){ + /** + * Most Tests are grouped together (see below) + * as a fromPointer => toPointer comparison + * because we generally want to test reversibility + */ + + describe("toPointers()", function(){ + it("dereferences nodes with children", function(){ + const doc = { + ...baseDoc, + nodes: [ + {name: "Lights", children: [1]}, + {name: "L1", light: 0} + ], + scenes:[{nodes: [0]}], + lights: [{type: "ambient"}] + }; + + const deref:any = toPointers(doc as any); + expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); + expect(deref.asset).to.deep.equal(doc.asset); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({"nodes":[ + {"name":"Lights","children":[{"name":"L1","light":{"type":"ambient"}}]} + ]}); + }); + + it("dereferences scene setup", function(){ + const doc = { + ...baseDoc, + nodes: [], + scenes:[{setup: 0}], + setups: [{language: {language: "FR"}}] + }; + + const deref:any = toPointers(doc as any); + expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); + expect(deref.asset).to.deep.equal(doc.asset); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({ + "nodes": [], + "setup":{language: {language: "FR"}} + }); + }); + + it("dereferences scene meta", function(){ + const doc = { + ...baseDoc, + nodes: [], + scenes:[{meta: 0}], + metas: [{collection: {titles:{FR: "Hello World!"}}}] + }; + + const deref:any = toPointers(doc as any); + expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); + expect(deref.asset).to.deep.equal(doc.asset); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({ + "nodes": [], + "meta":{collection: {titles:{FR: "Hello World!"}}} + }); + + }); + + it("dereferences node's meta", function(){ + + const doc = { + ...baseDoc, + nodes: [{ + name: "Model", + model: 0, + meta: 0 + }], + scenes:[{ + nodes: [0] + }], + models: [{uri: "model.gltf"}], + metas: [{collection: {titles:{FR: "Hello World!"}}}] + }; + + const deref:any = toPointers(doc as any); + expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); + expect(deref.asset).to.deep.equal(doc.asset); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({ + "nodes": [{name: "Model", model: {uri: "model.gltf"}, meta: {collection: {titles:{FR: "Hello World!"}}}}], + }); + + }); + + it("keeps a scene's name and units", function(){ + + const doc = { + ...baseDoc, + scenes:[{ + name: "My Scene", + units: "km", + }], + }; + + const deref:any = toPointers(doc as any); + expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); + expect(deref.asset).to.deep.equal(doc.asset); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(JSON.parse(JSON.stringify(deref.scene))).to.deep.equal({ + "name": "My Scene", + "units": "km", + "nodes": [], + }); + }); + }); + + describe("fromPointers()", function(){ + + it("builds nodes reference arrays", function(){ + const deref = { + asset: baseDoc.asset, + scene: { + units: "m", + nodes:[ + {name: "Camera", camera: {type: "perspective"}}, + {name: "Lights", children: [{name:"L1",light:{"type":"ambient"}}]}, + {name: "Model", model: {uri: "model.gltf"}}, + ], + } + }; + + const doc:IDocument = fromPointers(deref as any); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(doc).to.deep.equal({ + ...baseDoc, + nodes: [ + {name: "Camera", camera: 0}, + {name: "Lights", children: [2]}, + {name: "L1", light: 0}, + {name: "Model", model: 0}, + ], + scenes:[{ + nodes: [0, 1, 3], + units: "m", + }], + cameras: [{type: "perspective"}], + lights: [{type: "ambient"}], + models: [{uri: "model.gltf"}], + }); + }); + + it("builds metas from scene", function(){ + const deref = { + asset: baseDoc.asset, + scene: { + units: "m", + meta: {collection: {titles:{EN: "Meta Title"}}}, + } + }; + + const doc:IDocument = fromPointers(deref as any); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(doc).to.deep.equal({ + ...baseDoc, + scenes:[{ + units: "m", + meta: 0, + }], + metas: [{collection: {titles:{EN: "Meta Title"}}}], + }); + }); + + it("builds metas from nodes", function(){ + const deref = { + asset: baseDoc.asset, + scene: { + units: "m", + nodes:[ + {name: "Model", meta: {collection: {titles:{EN: "Meta Title"}}}}, + ], + } + }; + + const doc:IDocument = fromPointers(deref as any); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(doc).to.deep.equal({ + ...baseDoc, + nodes: [ + {name: "Model", meta: 0}, + ], + scenes:[{ + nodes: [0], + units: "m", + }], + metas: [{collection: {titles:{EN: "Meta Title"}}}] + }); + }) + + it("builds setups from scene", function(){ + const deref = { + asset: baseDoc.asset, + scene: { + units: "m", + setup: {language: {language: "FR"}}, + } + }; + + const doc:IDocument = fromPointers(deref as any); + //use JSON.parse(JSON.stringify()) to remove undefined values + expect(doc).to.deep.equal({ + ...baseDoc, + + scenes:[{ + setup: 0, + units: "m", + }], + setups: [{language: {language: "FR"}}] + }); + }); + + }); + + describe("is reversible", function(){ + const paths:Record = [ + "01_simple.svx.json", + ].reduce((paths, file)=>({...paths, [file]: path.resolve(thisDir, "../../__test_fixtures/documents/"+file)}),{}); + + it("on default.svx.json", async function(){ + const doc = JSON.parse(await fs.readFile(path.resolve(thisDir, "../schema/default.svx.json"), "utf8")); + const deref = toPointers(doc); + const redoc = fromPointers(deref); + expect(redoc).to.deep.equal(doc); + }); + + describe("01_simple.svx.json", function(){ + let docString :string, doc :IDocument; + this.beforeAll(async function(){ + docString = await fs.readFile(paths["01_simple.svx.json"], "utf8"); + }); + this.beforeEach(function(){ + doc = JSON.parse(docString); + }); + + it("can be dereferenced", async function(){ + const deref = toPointers(doc); + + expect(Object.keys(deref)).to.deep.equal(["asset", "scene"]); + expect(deref.asset).to.deep.equal(doc.asset); + expect(Object.keys(deref.scene)).to.deep.equal(["name", "units", "nodes", "setup"]); + expect(deref.scene.nodes).to.have.property("length", 3); + expect(deref.scene.nodes[2]).to.have.property("meta").to.deep.equal({collection: {titles:{EN: "Meta Title"}}}); + }); + + it("dereferencing is indempotent", async function(){ + const deref = toPointers(doc); + const redoc = fromPointers(deref); + expect(redoc).to.deep.equal(doc); + }); + + it("is JSON - stable", async function(){ + let previous_string = null; + for(let i=0; i <2; i++){ + const deref = toPointers(doc); + const redoc = fromPointers(deref); + const redoc_string = JSON.stringify(redoc, null, 2); + if(previous_string) expect(redoc_string).to.equal(previous_string); + previous_string = redoc_string; + } + }) + }) + }); +}) \ No newline at end of file diff --git a/source/server/utils/merge/pointers.ts b/source/server/utils/merge/pointers.ts new file mode 100644 index 00000000..e7a055bc --- /dev/null +++ b/source/server/utils/merge/pointers.ts @@ -0,0 +1,152 @@ +import { ICamera, IDocument, ILight, INode, IScene } from "../schema/document.js"; +import { IMeta } from "../schema/meta.js"; +import { IModel } from "../schema/model.js"; +import { ISetup } from "../schema/setup.js"; +import { DerefDocument, DerefNode, DerefScene } from "./types.js"; + + + +/** + * Dereference all indexed values into pointers in a SVX document + * It effectively forgets about all unreachable nodes (scenes, setups, etc...). + */ +export function toPointers(src :IDocument) :DerefDocument { + const nodes = src.nodes?.map(iNode=>{ + let node :DerefNode = { + name: iNode.name, + matrix: iNode.matrix, + translation: iNode.translation, + rotation: iNode.rotation, + scale: iNode.scale, + }; + //node.children will be assigned later if necessary + + for(let key of ["camera", "light", "model", "meta"] as const){ + const idx = iNode[key]; + if(typeof idx == "number"){ + const dict = src[key+"s" as `${typeof key}s`] + let ptr = dict? dict[idx]: undefined; + if(!ptr) throw new Error(`Invalid ${key} index ${idx}`); + (node[key] as any) = ptr; + } + } + return node; + }); + + nodes?.forEach((node, nodeIndex)=>{ + const indices = (src.nodes as INode[])[nodeIndex].children; + if(!indices) return; + node.children = indices.map(idx=>nodes[idx]); + }); + + const scenes = src.scenes?.map((iScene, sceneIndex)=>{ + let scene :DerefScene = { + name: iScene.name, + units: iScene.units, + nodes: [], + } + if(iScene.nodes?.length){ + scene.nodes = iScene.nodes.map(idx=>{ + if(!nodes || !nodes[idx]) throw new Error(`Invalid node index ${idx} in scene ${iScene.name}(${sceneIndex})`); + return nodes[idx]; + }); + } + if(typeof iScene.setup == "number"){ + if(!src.setups || !src.setups[iScene.setup]) throw new Error(`Invalid setup index ${iScene.setup} in scene ${iScene.name}(${sceneIndex})`) + scene.setup = src.setups[iScene.setup]; + } + if(typeof iScene.meta == "number"){ + if(!src.metas || !src.metas[iScene.meta]) throw new Error(`Invalid meta index ${iScene.meta} in scene ${iScene.name}(${sceneIndex})`) + scene.meta = src.metas[iScene.meta]; + } + + return scene; + }); + if(!scenes || !scenes[src.scene!]) throw new Error("Document has no valid scene"); + return { + asset: src.asset, + scene: scenes[src.scene!], + }; +} + + +export function fromPointers({asset, scene} :DerefDocument): IDocument{ + + + let collections = { + "scenes": [] as IScene[], + "nodes": [] as INode[], + "cameras": [] as ICamera[], + "lights": [] as ILight[], + "models": [] as IModel[], + "metas": [] as IMeta[], + "setups": [] as ISetup[], + + } as const; + + + function appendScene(scene :DerefScene){ + let iScene :IScene = { + units: scene.units ?? "cm", + }; + if(scene.name) iScene.name = scene.name; + if(scene.nodes?.length){ + iScene.nodes = scene.nodes.map((node)=>appendNode(node)); + } + for(let key of ["setup", "meta"] as const){ + const value = scene[key]; + if(!value) continue; + let colKey = key+"s" as keyof typeof collections; + const idx = collections[colKey].length; + collections[colKey].push(value as any); + iScene[key] = idx as any; + } + return collections.scenes.push(iScene) - 1; + } + + function appendNode(node :DerefNode){ + + let iNode :INode = {}; + if(node.name) iNode.name = node.name; + + //Push to collection before appending children because that's a more "natural" order to have + const idx = collections.nodes.push(iNode) - 1; + + if(node.children?.length){ + iNode.children = node.children.map((child)=>appendNode(child)); + } + + if(node.matrix) iNode.matrix = node.matrix; + if(node.translation) iNode.translation = node.translation; + if(node.rotation) iNode.rotation = node.rotation; + if(node.scale) iNode.scale = node.scale; + + + for(let key of ["camera", "light", "model", "meta"] as const){ + const value = node[key]; + if(!value) continue; + let colKey = key+"s" as keyof typeof collections; + const idx = collections[colKey].length; + collections[colKey].push(value as any); + iNode[key] = idx as any; + } + return idx; + } + + + let sceneIndex = appendScene(scene); + + + let out :IDocument = { + asset, + scene: sceneIndex, + ...collections, + }; + + for(let key in out){ + let v = out[key as keyof typeof out]; + if(typeof v === "undefined" || (Array.isArray(v) && v.length == 0)) delete out[key as keyof typeof out]; + } + + return out; +} \ No newline at end of file diff --git a/source/server/utils/merge/types.ts b/source/server/utils/merge/types.ts new file mode 100644 index 00000000..efb65d2c --- /dev/null +++ b/source/server/utils/merge/types.ts @@ -0,0 +1,55 @@ + +import { IDocumentAsset, IScene, INode, ICamera, ILight, Matrix4, Quaternion, Vector3 } from "../schema/document.js"; +import { IMeta } from "../schema/meta.js"; +import { IModel, TUnitType } from "../schema/model.js"; +import { ISetup } from "../schema/setup.js"; +import { Index } from "../schema/types.js"; + +/** + * Special symbol to mark a field for deletion in a diff object + */ +export const DELETE_KEY = Symbol("_DELETE_KEY"); + +export type Diff = { + [K in keyof T]?: typeof DELETE_KEY|Diff|T[K]|Record; +} + +/** + * IScene where all indexed references were replaced by pointers to the actual objects. + */ +export interface DerefScene{ + name?: string; + nodes: DerefNode[]; + setup?: ISetup; + meta?: IMeta; + units: TUnitType; +} + +/** + * INode where all indexed references were replaced by pointers to the actual objects. + */ +export interface DerefNode { + name?: string; + children?: DerefNode[]; + + matrix?: Matrix4; + translation?: Vector3; + rotation?: Quaternion; + scale?: Vector3; + + camera?: ICamera; + light?: ILight; + model?: IModel; + meta?: IMeta; +} + +/** + * A document where all indexed references were replaced by pointers to the actual objects. + * @see DerefNode for nodes + * @see DerefScene for scenes + */ +export interface DerefDocument +{ + asset: IDocumentAsset; + scene: DerefScene; +} \ No newline at end of file diff --git a/source/server/utils/schema/default.svx.json b/source/server/utils/schema/default.svx.json index f544a883..ffbcd235 100644 --- a/source/server/utils/schema/default.svx.json +++ b/source/server/utils/schema/default.svx.json @@ -8,7 +8,8 @@ "scene": 0, "scenes": [{ "nodes": [0, 1], - "setup": 0 + "setup": 0, + "units": "m" }], "nodes": [{ "translation": [0, 0, 15],