Add h264 recording support to mcap.dev demo (foxglove#1015)

### Public-Facing Changes The recording demo on mcap.dev can now record H.264-encoded CompressedVideo, if supported by the browser. ### Description - Use VideoEncoder to encode h264 - Detection of browser support & workarounds for some Safari issues (related to WebKit/WebKit#15562) Depends on foxglove#1014 (for VideoEncoder type definitions)
pezy · Jan 11, 2024 · b8a12b7 · b8a12b7
1 parent ba6eea0
commit b8a12b7
Show file tree

Hide file tree

Showing 7 changed files with 521 additions and 102 deletions.
diff --git a/website/package.json b/website/package.json
@@ -46,6 +46,7 @@
     "promise-queue": "2.2.5",
     "protobufjs": "7.2.3",
     "react": "17.0.2",
+    "react-async": "10.0.1",
     "react-dom": "17.0.2",
     "typescript": "5.2.2",
     "zustand": "4.3.8"

diff --git a/website/src/components/McapRecordingDemo/McapRecordingDemo.module.css b/website/src/components/McapRecordingDemo/McapRecordingDemo.module.css
@@ -130,6 +130,21 @@
   position: absolute;
   inset: 0;
   object-fit: cover;
+  z-index: 0;
+}
+
+.videoContainer .videoErrorContainer {
+  width: 100%;
+  height: 100%;
+  position: absolute;
+  background-color: #f5f1ffc4;
+  padding: 0.5rem;
+  inset: 0;
+  z-index: 1;
+}
+
+[data-theme="dark"] .videoContainer .videoErrorContainer {
+  background-color: #17151ec4;
 }
 
 .videoPlaceholderText {
@@ -170,6 +185,16 @@
   border-color: #585858;
 }
 
+.h264Warning {
+  font-weight: 600;
+  font-size: 0.8rem;
+  border: 1px solid var(--ifm-color-warning-dark);
+  background-color: var(--ifm-color-warning-contrast-background);
+  padding: 6px 6px 6px 12px;
+  margin-bottom: 16px;
+  color: var(--ifm-color-warning-contrast-foreground);
+}
+
 .downloadInfoCloseButton {
   float: right;
   font-size: 1rem;

diff --git a/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx b/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx
@@ -5,6 +5,7 @@ import { fromMillis } from "@foxglove/rostime";
 import { PoseInFrame } from "@foxglove/schemas";
 import cx from "classnames";
 import React, { useCallback, useEffect, useRef, useState } from "react";
+import { useAsync } from "react-async";
 import { create } from "zustand";
 
 import styles from "./McapRecordingDemo.module.css";
@@ -14,7 +15,12 @@ import {
   Recorder,
   toProtobufTime,
 } from "./Recorder";
-import { startVideoCapture, startVideoStream } from "./videoCapture";
+import {
+  H264Frame,
+  startVideoCapture,
+  startVideoStream,
+  supportsH264Encoding,
+} from "./videoCapture";
 
 type State = {
   bytesWritten: bigint;
@@ -26,7 +32,8 @@ type State = {
 
   addMouseEventMessage: (msg: MouseEventMessage) => void;
   addPoseMessage: (msg: DeviceOrientationEvent) => void;
-  addCameraImage: (blob: Blob) => void;
+  addJpegFrame: (blob: Blob) => void;
+  addH264Frame: (frame: H264Frame) => void;
   closeAndRestart: () => Promise<Blob>;
 };
 
@@ -54,8 +61,11 @@ const useStore = create<State>((set) => {
       void recorder.addPose(deviceOrientationToPose(msg));
       set({ latestOrientation: msg });
     },
-    addCameraImage(blob: Blob) {
-      void recorder.addCameraImage(blob);
+    addJpegFrame(blob: Blob) {
+      void recorder.addJpegFrame(blob);
+    },
+    addH264Frame(frame: H264Frame) {
+      void recorder.addH264Frame(frame);
     },
     async closeAndRestart() {
       return await recorder.closeAndRestart();
@@ -113,20 +123,26 @@ export function McapRecordingDemo(): JSX.Element {
   const [orientationPermissionError, setOrientationPermissionError] =
     useState(false);
 
-  const videoRef = useRef<HTMLVideoElement>(null);
-  const [recordVideo, setRecordVideo] = useState(false);
+  const videoRef = useRef<HTMLVideoElement | undefined>();
+  const videoContainerRef = useRef<HTMLDivElement>(null);
+  const [recordJpeg, setRecordJpeg] = useState(false);
+  const [recordH264, setRecordH264] = useState(false);
   const [recordMouse, setRecordMouse] = useState(true);
   const [recordOrientation, setRecordOrientation] = useState(true);
   const [videoStarted, setVideoStarted] = useState(false);
-  const [videoPermissionError, setVideoPermissionError] = useState(false);
+  const [videoError, setVideoError] = useState<Error | undefined>();
   const [showDownloadInfo, setShowDownloadInfo] = useState(false);
 
-  const { addCameraImage, addMouseEventMessage, addPoseMessage } = state;
+  const { addJpegFrame, addH264Frame, addMouseEventMessage, addPoseMessage } =
+    state;
+
+  const { data: h264Support } = useAsync(supportsH264Encoding);
 
   const canStartRecording =
     recordMouse ||
     (!hasMouse && recordOrientation) ||
-    (recordVideo && !videoPermissionError);
+    (recordH264 && !videoError) ||
+    (recordJpeg && !videoError);
 
   // Automatically pause recording after 30 seconds to avoid unbounded growth
   useEffect(() => {
@@ -172,47 +188,77 @@ export function McapRecordingDemo(): JSX.Element {
     };
   }, [addPoseMessage, recording, recordOrientation]);
 
+  const enableCamera = recordH264 || recordJpeg;
   useEffect(() => {
-    const video = videoRef.current;
-    if (!recordVideo || !video) {
+    const videoContainer = videoContainerRef.current;
+    if (!videoContainer || !enableCamera) {
       return;
     }
 
+    if (videoRef.current) {
+      videoRef.current.remove();
+    }
+    const video = document.createElement("video");
+    video.muted = true;
+    video.playsInline = true;
+    videoRef.current = video;
+    videoContainer.appendChild(video);
+
     const cleanup = startVideoStream({
-      video,
+      video: videoRef.current,
       onStart: () => {
         setVideoStarted(true);
       },
       onError: (err) => {
+        setVideoError(err);
         console.error(err);
-        setVideoPermissionError(true);
       },
     });
 
     return () => {
       cleanup();
+      video.remove();
       setVideoStarted(false);
-      setVideoPermissionError(false);
+      setVideoError(undefined);
     };
-  }, [recordVideo]);
+  }, [enableCamera]);
 
   useEffect(() => {
     const video = videoRef.current;
-    if (!recording || !recordVideo || !video || !videoStarted) {
+    if (!recording || !video || !videoStarted) {
+      return;
+    }
+    if (!recordH264 && !recordJpeg) {
       return;
     }
 
     const stopCapture = startVideoCapture({
       video,
+      enableH264: recordH264,
+      enableJpeg: recordJpeg,
       frameDurationSec: 1 / 30,
-      onFrame: (blob) => {
-        addCameraImage(blob);
+      onJpegFrame: (blob) => {
+        addJpegFrame(blob);
+      },
+      onH264Frame: (frame) => {
+        addH264Frame(frame);
+      },
+      onError: (err) => {
+        setVideoError(err);
+        console.error(err);
       },
     });
     return () => {
       stopCapture();
     };
-  }, [addCameraImage, recordVideo, recording, videoStarted]);
+  }, [
+    addJpegFrame,
+    addH264Frame,
+    recordH264,
+    recording,
+    videoStarted,
+    recordJpeg,
+  ]);
 
   const onRecordClick = useCallback(
     (event: React.MouseEvent) => {
@@ -284,15 +330,27 @@ export function McapRecordingDemo(): JSX.Element {
           </p>
         </header>
         <div className={styles.sensors}>
+          {h264Support?.supported === true && (
+            <label>
+              <input
+                type="checkbox"
+                checked={recordH264}
+                onChange={(event) => {
+                  setRecordH264(event.target.checked);
+                }}
+              />
+              Camera (H.264)
+            </label>
+          )}
           <label>
             <input
               type="checkbox"
-              checked={recordVideo}
+              checked={recordJpeg}
               onChange={(event) => {
-                setRecordVideo(event.target.checked);
+                setRecordJpeg(event.target.checked);
               }}
             />
-            Camera
+            Camera (JPEG)
           </label>
           <label>
             <input
@@ -343,6 +401,13 @@ export function McapRecordingDemo(): JSX.Element {
           </div>
         )}
 
+        {recordH264 && h264Support?.mayUseLotsOfKeyframes === true && (
+          <div className={styles.h264Warning}>
+            Note: This browser may have a bug that causes H.264 encoding to be
+            less efficient.
+          </div>
+        )}
+
         <div className={styles.recordingControls}>
           <div className={styles.recordingControlsColumn}>
             <Link
@@ -427,14 +492,13 @@ export function McapRecordingDemo(): JSX.Element {
           </div>
 
           <div className={styles.recordingControlsColumn}>
-            <div className={styles.videoContainer}>
-              {videoPermissionError ? (
-                <div className={styles.error}>
-                  Allow permission to record camera images
+            <div className={styles.videoContainer} ref={videoContainerRef}>
+              {videoError ? (
+                <div className={cx(styles.error, styles.videoErrorContainer)}>
+                  {videoError.toString()}
                 </div>
-              ) : recordVideo ? (
+              ) : recordH264 || recordJpeg ? (
                 <>
-                  <video ref={videoRef} muted playsInline />
                   {!videoStarted && (
                     <progress className={styles.videoLoadingIndicator} />
                   )}
@@ -443,7 +507,11 @@ export function McapRecordingDemo(): JSX.Element {
                 <span
                   className={styles.videoPlaceholderText}
                   onClick={() => {
-                    setRecordVideo(true);
+                    if (h264Support?.supported === true) {
+                      setRecordH264(true);
+                    } else {
+                      setRecordJpeg(true);
+                    }
                   }}
                 >
                   Enable “Camera” to record video