using webcodecs for decoding

.github/workflows/deploy.yml

@@ -28,8 +28,6 @@ jobs:
         # However because of the hash, cache only gets restored if the Dockerfile
         # is unchanged.
         run: make -t public/app/bundled/libavjs/version.txt || true
-      - name: install packages
-        run: npm install .
       - name: Build
         run: DOCKER="docker buildx" DOCKER_TMPDIR=/tmp make all
       - name: Set AWS credentials

Makefile

@@ -38,7 +38,12 @@ $(LIBAVJS_TARGET_FILES): libav.js/Dockerfile libav.js/commit.txt
 	@cp -R $(OUTDIR)/dist public/app/bundled/libavjs-$(LIBAVJS_COMMIT)
 	@rm -r "$(OUTDIR)"
 
-public/app/tsc: tsconfig.json $(shell find src) public/app/bundled/libavjs-$(LIBAVJS_COMMIT)/version.txt
+node_modules/tag: package.json
+	@npm install --no-save .
+	@cd node_modules/libavjs-webcodecs-bridge && make all
+	@touch $@
+
+public/app/tsc: tsconfig.json $(shell find src) public/app/bundled/libavjs-$(LIBAVJS_COMMIT)/version.txt node_modules/tag
 	@tsc --noEmit
 	@./node_modules/esbuild/bin/esbuild $(ENTRYPOINTS) --sourcemap --bundle --format=esm --outbase=src --outdir=public/app/
 	@(cd public $(foreach ext,js css,$(foreach outfilebase,$(OUTFILESBASE),&& MD5=$$(md5sum "$(outfilebase).$(ext)" | cut -c-10) && mv "$(outfilebase).$(ext)" "$(outfilebase).$${MD5}.$(ext)" && echo "s|$(outfilebase).$(ext)|$(outfilebase).$${MD5}.$(ext)|g"))) > $@

package.json

@@ -3,9 +3,11 @@
     "@tensorflow/tfjs": "^4.12.0",
     "@tensorflow/tfjs-backend-wasm": "^4.12.0",
     "@tensorflow/tfjs-backend-webgpu": "^4.12.0",
+    "@types/dom-webcodecs": "^0.1.11",
     "@types/react": "^18.2.38",
     "@types/wicg-file-system-access": "^2023.10.4",
     "esbuild": "^0.19.5",
+    "libavjs-webcodecs-bridge": "github:Yahweasel/libavjs-webcodecs-bridge#98fb137f2b06029e376dc252c31d2f0b540a5919",
     "preact": "^10.19.2"
   }
 }

src/infer/tfjs.ts

@@ -58,10 +58,10 @@ export async function convert(
       + `# all coordinates are on frame where left-top = (0, 0) and right-bottom is (1, 1)\n`
   ))
   const MODEL_DIMENSION = 640
-  for await (const imageData of getFrames(file, MODEL_DIMENSION, MODEL_DIMENSION)) {
-    const [boxes, scores, classes] = await infer(model, yoloVersion, imageData)
+  for await (const videoFrame of getFrames(file)) {
+    const [boxes, scores, classes] = await infer(model, yoloVersion, videoFrame)
     if (ctx) {
-      ctx.putImageData(imageData, 0, 0)
+      ctx.drawImage(videoFrame, 0, 0)
       ctx.strokeStyle = "red"
       ctx.lineWidth = 5;
     }
@@ -77,26 +77,28 @@ export async function convert(
       await outputstream.write(textEncoder.encode(line))
       const [cx, cy, w, h] = box
       if (ctx) {
+        const scale = Math.max(videoFrame.displayWidth, videoFrame.displayHeight)
         ctx.strokeRect(
-          (cx - w / 2) * Math.max(imageData.width, imageData.height),
-          (cy - h / 2) * Math.max(imageData.width, imageData.height),
-          w * Math.max(imageData.width, imageData.height),
-          h * Math.max(imageData.width, imageData.height),
+          (cx - w / 2) * scale, (cy - h / 2) * scale, w * scale, h * scale,
         )
       }
     }
+    videoFrame.close()
     onProgress({"converting": Math.min(framenr / numberOfFrames, 1)})
     framenr++
   }
 }
 
 export function preprocess(
-  imageData: ImageData,
+  videoFrame: VideoFrame,
   modelWidth: number,
   modelHeight: number
 ): [tf.Tensor<tf.Rank>, number, number] {
+  const offScreenCanvas = new OffscreenCanvas(videoFrame.displayWidth, videoFrame.displayHeight)
+  const ctx = offScreenCanvas.getContext("2d")!
+  ctx.drawImage(videoFrame, 0, 0)
 
-  const img = tf.browser.fromPixels(imageData);
+  const img = tf.browser.fromPixels(offScreenCanvas.transferToImageBitmap())
 
   const [h, w] = img.shape.slice(0, 2); // get source width and height
   const maxSize = Math.max(w, h); // get max size
@@ -151,9 +153,9 @@ function getBoxesAndScoresAndClassesFromResult(
 export async function infer(
   model: Model,
   yoloVersion: YoloVersion,
-  imageData: ImageData,
+  videoFrame: VideoFrame,
 ): Promise<[Float32Array, Float32Array, Float32Array]> {
-  const [img_tensor, xRatio, yRatio] = tf.tidy(() => preprocess(imageData, 640, 640))
+  const [img_tensor, xRatio, yRatio] = tf.tidy(() => preprocess(videoFrame, 640, 640))
   if (yoloVersion === "v5") {
     const res = await  model.executeAsync(img_tensor)
     const [boxes, scores, classes] = (res as tf.Tensor<tf.Rank>[]).slice(0, 3)