Support camera poses

package.json

@@ -13,32 +13,6 @@
     ],
     "license": "MIT",
     "main": "index.js",
-    "eslintConfig": {
-        "root": true,
-        "parser": "@typescript-eslint/parser",
-        "plugins": [
-            "@typescript-eslint"
-        ],
-        "extends": [
-            "@playcanvas/eslint-config",
-            "plugin:@typescript-eslint/recommended",
-            "plugin:import/typescript"
-        ],
-        "rules": {
-            "@typescript-eslint/ban-ts-comment": 0,
-            "@typescript-eslint/no-explicit-any": 0,
-            "@typescript-eslint/no-empty-function": 0,
-            "@typescript-eslint/explicit-module-boundary-types": 0,
-            "lines-between-class-members": [
-                "error",
-                "always",
-                {
-                    "exceptAfterSingleLine": true
-                }
-            ],
-            "import/no-unresolved": "off"
-        }
-    },
     "scripts": {
         "build": "rollup -c",
         "watch": "rollup -c -w",
@@ -52,31 +26,32 @@
     },
     "devDependencies": {
         "@playcanvas/eslint-config": "^1.7.4",
-        "@playcanvas/pcui": "^4.4.0",
-        "@rollup/plugin-alias": "^5.1.0",
+        "@playcanvas/pcui": "^4.5.0",
+        "@rollup/plugin-alias": "^5.1.1",
         "@rollup/plugin-image": "^3.0.3",
         "@rollup/plugin-json": "^6.1.0",
-        "@rollup/plugin-node-resolve": "^15.2.3",
+        "@rollup/plugin-node-resolve": "^15.3.0",
         "@rollup/plugin-strip": "^3.0.4",
         "@rollup/plugin-terser": "^0.4.4",
-        "@rollup/plugin-typescript": "^11.1.6",
+        "@rollup/plugin-typescript": "^12.1.0",
         "@types/wicg-file-system-access": "^2023.10.5",
         "@typescript-eslint/eslint-plugin": "^7.15.0",
         "@typescript-eslint/parser": "^7.15.0",
         "autoprefixer": "^10.4.20",
-        "concurrently": "^8.2.2",
+        "concurrently": "^9.0.1",
         "cors": "^2.8.5",
         "cross-env": "^7.0.3",
         "eslint": "^8.56.0",
         "i18next": "^23.15.1",
         "i18next-browser-languagedetector": "^8.0.0",
         "jest": "^29.7.0",
         "playcanvas": "^1.74.0",
-        "postcss": "^8.4.41",
-        "rollup": "^4.21.0",
-        "rollup-plugin-sass": "^1.13.2",
+        "postcss": "^8.4.47",
+        "rollup": "^4.23.0",
+        "rollup-plugin-postcss": "^4.0.2",
         "rollup-plugin-visualizer": "^5.12.0",
+        "sass": "^1.79.4",
         "serve": "^14.2.3",
-        "tslib": "^2.6.3"
+        "tslib": "^2.7.0"
     }
 }

rollup.config.mjs

@@ -10,8 +10,7 @@ import json from '@rollup/plugin-json';
 // import { visualizer } from 'rollup-plugin-visualizer';
 
 import autoprefixer from 'autoprefixer';
-import postcss from 'postcss';
-import sass from 'rollup-plugin-sass';
+import postcss from 'rollup-plugin-postcss';
 
 // prod is release build
 if (process.env.BUILD_TYPE === 'prod') {
@@ -29,11 +28,28 @@ const ENGINE_PATH = path.resolve(ENGINE_DIR, 'build', ENGINE_NAME);
 
 const PCUI_DIR = path.resolve(process.env.PCUI_PATH || 'node_modules/@playcanvas/pcui');
 
-const aliasEntries = {
-    playcanvas: ENGINE_PATH,
-    pcui: PCUI_DIR
+const outputHeader = () => {
+    const BLUE_OUT = '\x1b[34m';
+    const BOLD_OUT = `\x1b[1m`;
+    const REGULAR_OUT = `\x1b[22m`;
+    const RESET_OUT = `\x1b[0m`;
+
+    const title = [
+        `Building SuperSplat`,
+        `type ${BOLD_OUT}${BUILD_TYPE}${REGULAR_OUT}`,
+        `engine ${BOLD_OUT}${ENGINE_DIR}${REGULAR_OUT}`,
+        `pcui ${BOLD_OUT}${PCUI_DIR}${REGULAR_OUT}`
+    ].map(l => `${BLUE_OUT}${l}`).join(`\n`);
+    console.log(`${BLUE_OUT}${title}${RESET_OUT}\n`);
 };
 
+outputHeader();
+
+const aliasEntries = [
+    { find: 'playcanvas', replacement: ENGINE_PATH },
+    { find: 'pcui', replacement: PCUI_DIR }
+];
+
 const tsCompilerOptions = {
     baseUrl: '.',
     paths: {
@@ -58,22 +74,26 @@ const application = {
                         return contents.toString().replace('__BASE_HREF__', HREF);
                     }
                 },
-                {src: 'src/manifest.json'},
-                {src: 'static/images', dest: 'static'},
-                {src: 'static/icons', dest: 'static'},
-                {src: 'static/env/VertebraeHDRI_v1_512.png', dest: 'static/env'}
+                { src: 'src/manifest.json' },
+                { src: 'static/images', dest: 'static' },
+                { src: 'static/icons', dest: 'static' },
+                { src: 'static/env/VertebraeHDRI_v1_512.png', dest: 'static/env' }
             ]
         }),
-        alias({entries: aliasEntries}),
+        alias({ entries: aliasEntries }),
         resolve(),
-        image({dom: false}),
-        sass({
-            output: 'dist/index.css',
-            processor: (css) => {
-                return postcss([autoprefixer])
-                        .process(css)
-                        .then(result => result.css);
-            }
+        image({ dom: false }),
+        postcss({
+            extract: 'index.css',
+            extensions: ['.scss', '.sass', '.css'],
+            use: [
+                ['sass', {
+                    includePaths: [ path.resolve(PCUI_DIR, 'dist') ]
+                }]
+            ],
+            plugins: [
+                autoprefixer
+            ]
         }),
         json(),
         typescript({

src/camera.ts

@@ -163,13 +163,14 @@ class Camera extends Element {
         t.goto({ distance }, dampingFactorFactor * controls.dampingFactor);
     }
 
-    // convert point (relative to camera focus point) to azimuth, elevation, distance
-    setOrientation(point: Vec3, dampingFactorFactor: number = 1.0) {
-        const distance = point.length();
-        const azim = Math.atan2(-point.x / distance, -point.z / distance) * math.RAD_TO_DEG;
-        const elev = Math.asin(point.y / distance) * math.RAD_TO_DEG;
+    setPose(position: Vec3, target: Vec3, dampingFactorFactor: number = 1) {
+        vec.sub2(target, position);
+        const l = vec.length();
+        const azim = Math.atan2(-vec.x / l, -vec.z / l) * math.RAD_TO_DEG;
+        const elev = Math.asin(vec.y / l) * math.RAD_TO_DEG;
+        this.setFocalPoint(target, dampingFactorFactor);
         this.setAzimElev(azim, elev, dampingFactorFactor);
-        this.setDistance(distance, dampingFactorFactor);
+        this.setDistance(l / this.sceneRadius * this.fovFactor, dampingFactorFactor);
     }
 
     // convert world to screen coordinate

src/controllers.ts

@@ -135,7 +135,6 @@ class PointerController {
         };
 
         const wheel = (event: WheelEvent) => {
-            event.preventDefault();
             const sign = (v: number) => v > 0 ? 1 : v < 0 ? -1 : 0;
             zoom(sign(event.deltaY) * -0.2);
             orbit(sign(event.deltaX) * 2.0, 0);
@@ -161,16 +160,12 @@ class PointerController {
         const keydown = (event: KeyboardEvent) => {
             if (keys.hasOwnProperty(event.key) && event.target === document.body) {
                 keys[event.key] = event.shiftKey ? 10 : (event.ctrlKey || event.metaKey || event.altKey ? 0.1 : 1);
-                event.preventDefault();
-                event.stopPropagation();
             }
         };
 
         const keyup = (event: KeyboardEvent) => {
             if (keys.hasOwnProperty(event.key)) {
                 keys[event.key] = 0;
-                event.preventDefault();
-                event.stopPropagation();
             }
         };
 
@@ -188,23 +183,29 @@ class PointerController {
             }
         };
 
-        target.addEventListener('pointerdown', pointerdown);
-        target.addEventListener('pointerup', pointerup);
-        target.addEventListener('pointermove', pointermove);
-        target.addEventListener('wheel', wheel);
-        target.addEventListener('dblclick', dblclick);
-        document.addEventListener('keydown', keydown);
-        document.addEventListener('keyup', keyup);
-
-        this.destroy = () => {
-            target.removeEventListener('pointerdown', pointerdown);
-            target.removeEventListener('pointerup', pointerup);
-            target.removeEventListener('pointermove', pointermove);
-            target.removeEventListener('wheel', wheel);
-            target.removeEventListener('dblclick', dblclick);
-            document.removeEventListener('keydown', keydown);
-            document.removeEventListener('keyup', keyup);
+        let destroy: () => void = null;
+
+        const wrap = (target: any, name: string, fn: any) => {
+            const callback = (event: any) => {
+                camera.scene.events.fire('camera.controller', name);
+                fn(event);
+            };
+            target.addEventListener(name, callback);
+            destroy = () => {
+                destroy?.();
+                target.removeEventListener(name, callback);
+            };
         };
+
+        wrap(target, 'pointerdown', pointerdown);
+        wrap(target, 'pointerup', pointerup);
+        wrap(target, 'pointermove', pointermove);
+        wrap(target, 'wheel', wheel);
+        wrap(target, 'dblclick', dblclick);
+        wrap(document, 'keydown', keydown);
+        wrap(document, 'keyup', keyup);
+
+        this.destroy = destroy;
     }
 }
 

src/editor.ts

@@ -5,7 +5,6 @@ import {
     Vec4,
 } from 'playcanvas';
 import { Scene } from './scene';
-import { EditorUI } from './ui/editor';
 import { EditHistory } from './edit-history';
 import { Splat } from './splat';
 import { State } from './splat-state';
@@ -572,6 +571,20 @@ const registerEditorEvents = (events: Events, editHistory: EditHistory, scene: S
         setViewBands(value);
     });
 
+    events.function('camera.getPose', () => {
+        const camera = scene.camera;
+        const position = camera.entity.getPosition();
+        const focalPoint = camera.focalPoint;
+        return {
+            position: { x: position.x, y: position.y, z: position.z },
+            target: { x: focalPoint.x, y: focalPoint.y, z: focalPoint.z }
+        };
+    });
+
+    events.on('camera.setPose', (pose: { position: Vec3, target: Vec3 }, speed = 1) => {
+        scene.camera.setPose(pose.position, pose.target, speed);
+    });
+
     // hack: fire events to initialize UI
     events.fire('camera.fov', scene.camera.fov);
     events.fire('camera.debug', cameraDebug);

src/file-handler.ts

@@ -1,4 +1,4 @@
-import { path } from 'playcanvas';
+import { path, Mat3, Mat4, Vec3 } from 'playcanvas';
 import { Scene } from './scene';
 import { Events } from './events';
 import { CreateDropHandler } from './drop-handler';
@@ -43,6 +43,8 @@ const filePickerTypes = {
 
 let fileHandle: FileSystemFileHandle = null;
 
+const vec = new Vec3();
+
 // download the data to the given filename
 const download = (filename: string, data: ArrayBuffer) => {
     const blob = new Blob([data], { type: "octet/stream" });
@@ -86,9 +88,53 @@ const writeToFile = async (stream: FileSystemWritableFileStream, data: ArrayBuff
     await stream.close();
 };
 
+const loadCameraPoses = async (url: string, filename: string, events: Events) => {
+    const response = await fetch(url);
+    const json = await response.json();
+    if (json.length > 0) {
+        // calculate the average position of the camera poses
+        const ave = new Vec3(0, 0, 0);
+        json.forEach((pose: any) => {
+            vec.set(pose.position[0], pose.position[1], pose.position[2]);
+            ave.add(vec);
+        });
+        ave.mulScalar(1 / json.length);
+
+        json.forEach((pose: any, i: number) => {
+            if (pose.hasOwnProperty('position') && pose.hasOwnProperty('rotation')) {
+                const p = new Vec3(pose.position);
+                const z = new Vec3(pose.rotation[0][2], pose.rotation[1][2], pose.rotation[2][2]);
+
+                const dot = vec.sub2(ave, p).dot(z);
+                vec.copy(z).mulScalar(dot).add(p);
+
+                events.fire('camera.addPose', {
+                    name: pose.img_name ?? `${filename}_${i}`,
+                    position: new Vec3(-p.x, -p.y, p.z),
+                    target: new Vec3(-vec.x, -vec.y, vec.z)
+                });
+            }
+        });
+    }
+};
+
 // initialize file handler events
 const initFileHandler = async (scene: Scene, events: Events, dropTarget: HTMLElement, remoteStorageDetails: RemoteStorageDetails) => {
 
+    const handleLoad = (url: string, filename: string) => {
+        if (filename.toLowerCase().endsWith('.json')) {
+            return loadCameraPoses(url, filename, events);
+        } else if (filename.toLowerCase().endsWith('.ply')) {
+            return scene.loadModel(url, filename);
+        } else {
+            return null;
+        }
+    };
+
+    events.function('load', (url: string, filename: string) => {
+        return handleLoad(url, filename);
+    });
+
     // create a file selector element as fallback when showOpenFilePicker isn't available
     let fileSelector: HTMLInputElement;
     if (!window.showOpenFilePicker) {
@@ -103,7 +149,7 @@ const initFileHandler = async (scene: Scene, events: Events, dropTarget: HTMLEle
             for (let i = 0; i < files.length; i++) {
                 const file = fileSelector.files[i];
                 const url = URL.createObjectURL(file);
-                await scene.loadModel(url, file.name);
+                await handleLoad(url, file.name);
                 URL.revokeObjectURL(url);
             }
         };
@@ -112,12 +158,9 @@ const initFileHandler = async (scene: Scene, events: Events, dropTarget: HTMLEle
 
     // create the file drag & drop handler
     CreateDropHandler(dropTarget, async (entries) => {
-        const modelExtensions = ['.ply'];
         for (let i = 0; i < entries.length; i++) {
             const entry = entries[i];
-            if (modelExtensions.some(extension => entry.filename.endsWith(extension))) {
-                await scene.loadModel(entry.url, entry.filename);
-            }
+            await handleLoad(entry.url, entry.filename);
         }
     });
 
@@ -162,7 +205,7 @@ const initFileHandler = async (scene: Scene, events: Events, dropTarget: HTMLEle
                     const handle = handles[i];
                     const file = await handle.getFile();
                     const url = URL.createObjectURL(file);
-                    await scene.loadModel(url, file.name);
+                    await handleLoad(url, file.name);
                     URL.revokeObjectURL(url);
 
                     if (i === 0) {

src/index.ts

@@ -1,11 +1,11 @@
-import style from './ui/style.scss';
+import './ui/style.scss';
 import { main } from './main';
 import { version as appVersion } from '../package.json';
 import { version as pcuiVersion, revision as pcuiRevision } from 'pcui';
 import { version as engineVersion, revision as engineRevision } from 'playcanvas';
 
 // print out versions of dependent packages
 // NOTE: add dummy style reference to prevent tree shaking
-console.log(`SuperSplat v${appVersion} | PCUI v${pcuiVersion} (${pcuiRevision}) | Engine v${engineVersion} (${engineRevision})${style.slice(0, 0)}`);
+console.log(`SuperSplat v${appVersion} | PCUI v${pcuiVersion} (${pcuiRevision}) | Engine v${engineVersion} (${engineRevision})`);
 
 main();