add a diff route. Some adjustments to vfs queries

source/server/routes/history/diff/get.test.ts

@@ -0,0 +1,51 @@
+
+import request from "supertest";
+import Vfs from "../../../vfs/index.js";
+import User from "../../../auth/User.js";
+import UserManager from "../../../auth/UserManager.js";
+
+
+
+/**
+ * Minimal tests as most
+ */
+
+describe("GET /history/:scene/:id/diff", function(){
+  let vfs :Vfs, userManager :UserManager, user :User, admin :User, opponent :User;
+  let scene_id :number;
+  this.beforeAll(async function(){
+    let locals = await createIntegrationContext(this);
+    vfs = locals.vfs;
+    userManager = locals.userManager;
+    user = await userManager.addUser("bob", "12345678");
+    admin = await userManager.addUser("alice", "12345678", true);
+    opponent = await userManager.addUser("oscar", "12345678");
+    scene_id = await vfs.createScene("foo", user.uid);
+  });
+
+  this.afterAll(async function(){
+    await cleanIntegrationContext(this);
+  });
+
+  it("get a text file's diff from previous version", async function(){
+    await vfs.writeFile(dataStream(["Hello\n"]), {scene:scene_id, name:"hello.txt", user_id: 0, mime: "text/plain"});
+    let ref = await vfs.writeFile(dataStream(["Hello World\n"]), {scene:scene_id, name:"hello.txt", user_id: 0, mime: "text/plain"});
+    let res = await request(this.server).get(`/history/foo/${ref.id}/diff`)
+    .set("Accept", "text/plain")
+    .expect(200)
+    .expect("Content-Type", "text/plain; charset=utf-8");
+    expect(res.text).to.equal(`1c1\n< Hello\n---\n> Hello World\n`);
+  });
+
+  it("get diff summary for documents", async function(){
+    await vfs.writeDoc(`{"label":"foo"}`, {scene: scene_id, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json", user_id: 0});
+    let ref = await vfs.writeDoc(`{"label":"bar"}`, {scene: scene_id, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json", user_id: 0});
+    let res = await request(this.server).get(`/history/foo/${ref.id}/diff`)
+    .set("Accept", "application/json")
+    .expect(200)
+    .expect("Content-Type", "application/json; charset=utf-8");
+    expect(res.body).to.have.property("diff", 'Diff of JSON data not yet supported');
+    expect(res.body).to.have.property("src");
+    expect(res.body).to.have.property("dst");
+  })
+});

source/server/routes/history/diff/get.ts

@@ -1,6 +1,106 @@
+import {execFile} from "node:child_process";
 
 import {Request, Response} from "express";
 
+import { getVfs } from "../../../utils/locals.js";
+import { BadRequestError } from "../../../utils/errors.js";
+import { FileProps } from "../../../vfs/types.js";
+import { diffDoc } from "../../../utils/merge/index.js";
+import { fromPointers } from "../../../utils/merge/pointers/index.js";
+
+const sizeMax = 400*1000;
+
+/**
+ * Computes a somewhat arbitrary text representation of the difference between 
+ */
 export default async function handleGetDiff(req :Request, res :Response){
-
+  const vfs = getVfs(req);
+  const {scene, id:idString, from:fromString="-1"} = req.params;
+  const id = parseInt(idString);
+  const fromIdOrGen = parseInt(fromString);
+  if(!Number.isInteger(id)) throw new BadRequestError(`Requested fromId ${idString} is not a valid ID`);
+  if(!Number.isInteger(fromIdOrGen)) throw new BadRequestError(`Requested toId ${fromString} is not a valid ID`);
+
+  const dstFile = await vfs.getFileById(id);
+  let fromFile :FileProps;
+  if(0 < fromIdOrGen){
+    fromFile = await vfs.getFileById(fromIdOrGen);
+  }else if(0 < dstFile.generation + fromIdOrGen){
+    fromFile = await vfs.getFileProps({
+      scene: scene,
+      name: dstFile.name,
+      generation: dstFile.generation + fromIdOrGen,
+      archive: true,
+    }, true);
+  }else{
+    fromFile = {
+      id: -1, //Ensure it _can't_ exist
+      name: dstFile.name,
+      mime: dstFile.mime,
+      hash: null,
+      data: undefined,
+      size: 0,
+      generation: 0,
+      ctime: new Date(0),
+      mtime: new Date(0),
+      author_id: 0,
+      author: "default",
+    }
+  }
+  let diff:string;
+  if(fromFile.data && dstFile.data && dstFile.mime == fromFile.mime && fromFile.mime === "application/si-dpo-3d.document+json"){
+    try{
+      let fromDoc = JSON.parse(fromFile.data);
+      let toDoc = JSON.parse(dstFile.data);
+      diff = `STRUCTURED CHANGES SUMMARY\n`+JSON.stringify(fromPointers(diffDoc(fromDoc, toDoc) as any), null, 2);
+    }catch(e:any){
+      console.error("Failed to compile scene diff : ", e);
+      diff = `Couldn't compile diff from #${fromFile.id} to #${dstFile.id}: ${e.message}`;
+    }
+
+  }else if(sizeMax < dstFile.size || sizeMax < fromFile.size){
+    diff = "Diff not computed for large files"
+  }else if(fromFile.hash === dstFile.hash){
+    diff = "No differences";
+  }else if(fromFile.hash === "directory" || dstFile.hash === "directory"){
+    diff = "No diff for folders";
+  }else{
+    let srcPath = (vfs.exists(fromFile))? vfs.getPath(fromFile): "/dev/null";
+    let dstPath = (vfs.exists(dstFile))? vfs.getPath(dstFile): "/dev/null";
+    let stdout = await new Promise<string>((resolve, reject)=>{
+        execFile("diff", [
+          "--unified",
+          "--tabsize=2",
+          "--ignore-space-change",
+          "--label", fromFile.name,
+          "--label", dstFile.name,
+          srcPath,
+          dstPath
+        ], {
+        encoding: "utf8",
+        timeout: 500,
+      }, (error, stdout, stderr)=>{
+        if(error && error.code != 0 && error.code != 1){
+          return reject(error);
+        }
+        resolve(stdout);
+      });
+    });
+    if(10000000 < stdout.length){
+      diff = "Large diff not shown";
+    }else{
+      diff = stdout;
+    }
+  }
+
+  res.format({
+    "text/plain": ()=>{
+      res.status(200).send(diff);
+    },
+    "application/json": ()=>{
+      let src = {...fromFile, data: null};
+      const dst = {...dstFile, data: null};
+      res.status(200).send({src, dst, diff});
+    }
+  });
 }

source/server/routes/history/index.ts

@@ -8,7 +8,6 @@ import bodyParser from "body-parser";
 
 import { postSceneHistory } from "./post.js";
 import getSceneHistory from "./get.js";
-import handleGetFileRef from "./refs/get.js";
 import handleGetDiff from "./diff/get.js";
 
 const router = Router();
@@ -27,7 +26,7 @@ router.use("/:scene", canRead);
 router.get("/:scene", wrap(getSceneHistory));
 router.post("/:scene", canAdmin, bodyParser.json(), wrap(postSceneHistory));
 
-router.get("/:scene/diff/:fromRef/:toRef", handleGetDiff);
-router.get("/:scene/:ref(\d+)/:name(*)", handleGetFileRef);
+router.get("/:scene/:id/diff", wrap(handleGetDiff));
+router.get("/:scene/:id/diff/:from", wrap(handleGetDiff));
 
 export default router;

source/server/vfs/Files.ts

@@ -212,9 +212,9 @@ export default abstract class FilesVfs extends BaseVfs{
    * This function is growing out of control, having to manage disk vs doc stored files, mtime aggregation, etc...
    * The whole thing might become a performance bottleneck one day.
    */
-  async getFileProps({scene, name, archive} :GetFileParams, withData?:false) :Promise<Omit<FileProps, "data">>
-  async getFileProps({scene, name, archive} :GetFileParams, withData :true) :Promise<FileProps>
-  async getFileProps({scene, name, archive = false} :GetFileParams, withData = false) :Promise<FileProps>{
+  async getFileProps({scene, name, archive, generation} :GetFileParams, withData?:false) :Promise<Omit<FileProps, "data">>
+  async getFileProps({scene, name, archive, generation} :GetFileParams, withData :true) :Promise<FileProps>
+  async getFileProps({scene, name, archive = false, generation} :GetFileParams, withData = false) :Promise<FileProps>{
     let is_string = typeof scene === "string";
     let r = await this.db.get(`
       WITH scene AS (SELECT scene_id FROM scenes WHERE ${(is_string?"scene_name":"scene_id")} = $scene )
@@ -230,12 +230,19 @@ export default abstract class FilesVfs extends BaseVfs{
         mime,
         files.fk_author_id AS author_id,
         (SELECT username FROM users WHERE files.fk_author_id = user_id LIMIT 1) AS author
-      FROM files 
-      INNER JOIN scene ON files.fk_scene_id = scene.scene_id 
+      FROM scene  
+      LEFT JOIN files ON files.fk_scene_id = scene.scene_id 
       WHERE files.name = $name
-      ORDER BY generation DESC
-      LIMIT 1
-    `, {$scene: scene, $name: name});
+      ${(typeof generation!== "undefined")? `
+        AND generation = $generation
+      ` : `
+        ORDER BY generation DESC
+        LIMIT 1`}
+    `, {
+      $scene: scene,
+      $name: name,
+      $generation: generation
+    });
     if(!r || !r.ctime || (!r.hash && !archive)) throw new NotFoundError(`${path.join(scene.toString(), name)}${archive?" incl. archives":""}`);
     return {
       ...r,
@@ -278,7 +285,18 @@ export default abstract class FilesVfs extends BaseVfs{
    * Isolated here so it can easily be replaced to any blob storage external service if needed
    */
   async openFile(file:{hash :string}) :Promise<FileHandle>{
-    return await fs.open(path.join(this.objectsDir, file.hash), constants.O_RDONLY);
+    return await fs.open(this.getPath(file), constants.O_RDONLY);
+  }
+
+  /**
+   * Gets the path to a filesystem-stored file
+   */
+  public getPath(file:{hash:string}) :string{
+    return path.join(this.objectsDir, file.hash)
+  }
+
+  public exists(file :Partial<FileProps>) :file is {hash :string}{
+    return typeof file.hash === "string" && file.data == null;
   }
 
   /**

source/server/vfs/helpers/db.ts

@@ -72,7 +72,10 @@ export default async function open({filename, forceMigration=true} :DbOptions) :
       if(commit) await this.run(`RELEASE SAVEPOINT VFS_TRANSACTION_${transaction_id}`);
       return res;
     }catch(e){
-      if(commit) await this.run(`ROLLBACK TRANSACTION TO VFS_TRANSACTION_${transaction_id}`);
+      if(commit){
+        await this.run(`ROLLBACK TRANSACTION TO VFS_TRANSACTION_${transaction_id}`);
+        await this.run(`RELEASE SAVEPOINT VFS_TRANSACTION_${transaction_id}`);
+      }
       throw e;
     }
   }

source/server/vfs/types.ts

@@ -22,6 +22,7 @@ export interface WriteFileParams extends CommonFileParams{
 export interface GetFileParams extends CommonFileParams{
   /**Also return deleted files */
   archive ?:boolean;
+  generation ?:number;
 }
 
 

source/server/vfs/vfs.test.ts

@@ -902,7 +902,33 @@ describe("Vfs", function(){
           it("throw 404 error if file doesn't exist", async function(){
             await expect(vfs.getFileProps({...props, name: "bar.html"})).to.be.rejectedWith("404");
           });
+
+          it("get archived file", async function(){
+            let id = await vfs.removeFile({...props, user_id: 0});
+            await expect(vfs.getFileProps(props), `File with id ${id} shouldn't be returned`).to.be.rejectedWith("[404]");
+            await expect(vfs.getFileProps({...props, archive: true})).to.eventually.have.property("id", id);
+          });
+
+          it("get by generation", async function(){
+            let r = await vfs.writeFile(dataStream(["foo","\n"]), {...props, user_id: 0});
+            expect(r).to.have.property("generation", 2);
+            await expect(vfs.getFileProps({...props, generation: 2})).to.eventually.have.property("generation", 2);
+            await expect(vfs.getFileProps({...props, generation: 1})).to.eventually.have.property("generation", 1);
+          });
+
+          it("get archived by generation", async function(){
+            await vfs.writeFile(dataStream(["foo","\n"]), {...props, user_id: 0});
+            let id = await vfs.removeFile({...props, user_id: 0});
+            await expect(vfs.getFileProps({...props, archive: true, generation: 3})).to.eventually.have.property("id", id);
+            await expect(vfs.getFileProps({...props, archive: true, generation: 3}, true)).to.eventually.have.property("id", id);
+          });
+
+          it("get document", async function(){
+            let doc = await vfs.writeDoc("{}", {...props, user_id: 0});
+            await expect(vfs.getFileProps({...props, archive: true, generation: doc.generation}, true)).to.eventually.deep.equal({...doc, data: "{}"});
+          });
         });
+
         describe("getFile()", function(){
           it("get a file", async function(){
             let {stream} = await vfs.getFile(props);