From 16bc9a8cb1022bd25f860869afa1e13ddab02600 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Mon, 20 Nov 2023 16:21:35 -0800 Subject: [PATCH] Add @mcap/support, @mcap/nodejs, and @mcap/browser libraries (#868) **Public-Facing Changes** - [TypeScript] Added a new package `@mcap/support`. This package provides decompression functions for zstd, lz4, and bz2, and utility functions for working with protobuf descriptors. - [TypeScript] Added a new package `@mcap/nodejs`. This package provides IReadable and IWritable implementations for Node.js `FileHandle`, reducing boilerplate for reading or writing MCAP files. - [TypeScript] Added a new package `@mcap/browser`. This package provides an IReadable implementation for the browser `Blob` (`File`) type, reducing boilerplate for reading MCAP files. - [TypeScript] Examples have been updated to use `@mcap/support` + `@mcap/nodejs` and remove repeated boilerplate **Description** **Note:** This began as un-revert of #838 (see #841) Relates to FG-2274 Resolves #719 Resolves #720 This package has been migrated from the @foxglove/mcap-support internal package in the foxglove/studio repo. Two new additions are `FileHandleWritable` under the `@mcap/nodejs` package, and `protobufDescriptors.ts` to provide helper methods for protobuf descriptors and avoid each project having to include a `protobufjs.d.ts` copy-paste. ~Studio's protobuf timestamp/duration rewriting logic has been replaced with `processRootType` & `processMessageDefinitions` functions that Studio can continue using to implement that behavior without inflicting its opinion upon other users of the library.~ Porting of deserialization for different encodings will be done in a separate PR when we have a chance to revisit the reader APIs to make them more friendly. --------- Co-authored-by: John Hurliman --- .github/workflows/ci.yml | 34 +- .vscode/extensions.json | 3 +- .vscode/settings.json | 16 +- RELEASE.md | 12 - cspell.config.yaml | 3 + package.json | 12 +- python/mcap-protobuf-support/README.md | 2 +- python/mcap-ros1-support/README.md | 2 +- python/mcap-ros2-support/README.md | 2 +- tests/conformance/.eslintrc.js | 5 +- tests/conformance/package.json | 11 +- .../TypescriptIndexedReaderTestRunner.ts | 24 +- tests/conformance/tsconfig.cjs.json | 6 +- tests/conformance/tsconfig.json | 9 +- typescript/CONTRIBUTING.md | 42 +++ typescript/README.md | 35 +- typescript/benchmarks/package.json | 4 +- typescript/browser/.eslintrc.js | 28 ++ typescript/browser/LICENSE | 21 ++ typescript/browser/README.md | 33 ++ typescript/browser/jest.config.json | 19 + typescript/browser/package.json | 55 +++ typescript/browser/src/BlobReadable.ts | 27 ++ typescript/browser/src/index.test.ts | 46 +++ typescript/browser/src/index.ts | 1 + typescript/browser/tsconfig.cjs.json | 7 + typescript/browser/tsconfig.json | 11 + typescript/core/README.md | 10 +- typescript/core/jest.config.json | 16 +- typescript/core/package.json | 11 +- typescript/core/src/TempBuffer.ts | 3 + typescript/core/src/index.ts | 1 + typescript/core/tsconfig.json | 4 +- typescript/examples/bag2mcap/.eslintrc.js | 5 +- typescript/examples/bag2mcap/package.json | 16 +- .../examples/bag2mcap/scripts/bag2mcap.ts | 33 +- typescript/examples/bag2mcap/tsconfig.json | 4 +- typescript/examples/basicwriter/.eslintrc.js | 5 +- typescript/examples/basicwriter/package.json | 10 +- .../examples/basicwriter/scripts/main.ts | 24 +- typescript/examples/basicwriter/tsconfig.json | 4 +- .../examples/flatbufferswriter/.eslintrc.js | 5 +- .../examples/flatbufferswriter/package.json | 9 +- .../flatbufferswriter/scripts/main.ts | 23 +- .../examples/flatbufferswriter/tsconfig.json | 4 +- .../text-annotation-demo/.eslintrc.js | 5 +- .../text-annotation-demo/package.json | 7 +- .../text-annotation-demo/scripts/main.ts | 24 +- .../text-annotation-demo/tsconfig.json | 4 +- typescript/examples/validate/.eslintrc.js | 5 +- typescript/examples/validate/package.json | 12 +- .../examples/validate/scripts/validate.ts | 36 +- typescript/examples/validate/tsconfig.json | 4 +- typescript/jest.config.json | 3 + typescript/nodejs/.eslintrc.js | 28 ++ typescript/nodejs/LICENSE | 21 ++ typescript/nodejs/README.md | 50 +++ typescript/nodejs/jest.config.json | 19 + typescript/nodejs/package.json | 56 +++ typescript/nodejs/src/FileHandleReadable.ts | 37 ++ typescript/nodejs/src/FileHandleWritable.ts | 23 ++ typescript/nodejs/src/index.test.ts | 52 +++ typescript/nodejs/src/index.ts | 2 + typescript/nodejs/tsconfig.cjs.json | 7 + typescript/nodejs/tsconfig.json | 11 + typescript/support/.eslintrc.js | 22 ++ typescript/support/LICENSE | 21 ++ typescript/support/README.md | 68 ++++ typescript/support/jest.config.json | 16 + typescript/support/package.json | 60 ++++ typescript/support/src/decompressHandlers.ts | 30 ++ typescript/support/src/index.ts | 2 + typescript/support/src/protobufDescriptors.ts | 19 + typescript/support/tsconfig.cjs.json | 7 + typescript/support/tsconfig.json | 11 + .../typings/protobufjs.d.ts | 0 typescript/typedoc.json | 5 + website/package.json | 6 +- yarn.lock | 331 ++++++++++++------ 79 files changed, 1285 insertions(+), 376 deletions(-) delete mode 100644 RELEASE.md create mode 100644 typescript/CONTRIBUTING.md create mode 100644 typescript/browser/.eslintrc.js create mode 100644 typescript/browser/LICENSE create mode 100644 typescript/browser/README.md create mode 100644 typescript/browser/jest.config.json create mode 100644 typescript/browser/package.json create mode 100644 typescript/browser/src/BlobReadable.ts create mode 100644 typescript/browser/src/index.test.ts create mode 100644 typescript/browser/src/index.ts create mode 100644 typescript/browser/tsconfig.cjs.json create mode 100644 typescript/browser/tsconfig.json create mode 100644 typescript/jest.config.json create mode 100644 typescript/nodejs/.eslintrc.js create mode 100644 typescript/nodejs/LICENSE create mode 100644 typescript/nodejs/README.md create mode 100644 typescript/nodejs/jest.config.json create mode 100644 typescript/nodejs/package.json create mode 100644 typescript/nodejs/src/FileHandleReadable.ts create mode 100644 typescript/nodejs/src/FileHandleWritable.ts create mode 100644 typescript/nodejs/src/index.test.ts create mode 100644 typescript/nodejs/src/index.ts create mode 100644 typescript/nodejs/tsconfig.cjs.json create mode 100644 typescript/nodejs/tsconfig.json create mode 100644 typescript/support/.eslintrc.js create mode 100644 typescript/support/LICENSE create mode 100644 typescript/support/README.md create mode 100644 typescript/support/jest.config.json create mode 100644 typescript/support/package.json create mode 100644 typescript/support/src/decompressHandlers.ts create mode 100644 typescript/support/src/index.ts create mode 100644 typescript/support/src/protobufDescriptors.ts create mode 100644 typescript/support/tsconfig.cjs.json create mode 100644 typescript/support/tsconfig.json rename typescript/{examples/validate => support}/typings/protobufjs.d.ts (100%) create mode 100644 typescript/typedoc.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0567844017..a57cce882d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: cache: yarn - run: yarn install --immutable - run: yarn workspace @foxglove/mcap-conformance lint:ci - - run: yarn workspace @foxglove/mcap-conformance typecheck + - run: yarn workspace @foxglove/mcap-conformance build conformance-cpp: runs-on: ubuntu-latest @@ -220,15 +220,39 @@ jobs: - run: yarn dedupe --check - run: yarn prettier:check - run: yarn workspace @mcap/core lint:ci - - run: yarn workspace @mcap/core typecheck - - run: yarn workspace @mcap/core test - - - name: Publish to NPM + - run: yarn workspace @mcap/core build + - run: yarn workspace @mcap/support lint:ci + - run: yarn workspace @mcap/support build + - run: yarn workspace @mcap/nodejs lint:ci + - run: yarn workspace @mcap/nodejs build + - run: yarn workspace @mcap/browser lint:ci + - run: yarn workspace @mcap/browser build + - run: yarn typescript:test + + - name: Publish @mcap/core to NPM if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/core/v') }} run: yarn workspace @mcap/core npm publish --access public env: YARN_NPM_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + - name: Publish @mcap/support to NPM + if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/support/v') }} + run: yarn workspace @mcap/support publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + + - name: Publish @mcap/nodejs to NPM + if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/nodejs/v') }} + run: yarn workspace @mcap/nodejs publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + + - name: Publish @mcap/browser to NPM + if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/browser/v') }} + run: yarn workspace @mcap/browser publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + typescript-examples: runs-on: ubuntu-latest steps: diff --git a/.vscode/extensions.json b/.vscode/extensions.json index bb94d4d388..6741286f2e 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -6,6 +6,7 @@ "orta.vscode-jest", "ms-vscode.cpptools-extension-pack", "ms-vscode-remote.vscode-remote-extensionpack", - "streetsidesoftware.code-spell-checker" + "streetsidesoftware.code-spell-checker", + "ms-python.flake8" ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 817d1e7ac8..de4b464e0c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ // -*- jsonc -*- { + "editor.codeActionsOnSave": { "source.fixAll.eslint": true }, "editor.formatOnSave": true, "editor.defaultFormatter": "esbenp.prettier-vscode", @@ -13,7 +14,7 @@ "eslint.options": { "reportUnusedDisableDirectives": "error" }, - "jest.jestCommandLine": "yarn workspace @mcap/core test", + "jest.jestCommandLine": "yarn typescript:test", "cSpell.enabled": true, @@ -21,20 +22,23 @@ "**/node_modules": true, "tests/conformance/data": true, "python/**/build": true, - "python/docs/*-apidoc": true + "python/docs/*-apidoc": true, + ".yarn/**": true, + "yarn.lock": true, + "**/dist": true }, - "python.formatting.provider": "black", + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter" + }, "python.analysis.typeCheckingMode": "strict", - "python.linting.flake8Enabled": true, - "python.linting.enabled": true, - "python.linting.flake8Args": ["--config", "python/.flake8"], "python.analysis.extraPaths": [ "./python/mcap", "./python/mcap-protobuf-support", "./python/mcap-ros1-support", "./python/mcap-ros2-support" ], + "flake8.args": ["--config", "python/.flake8"], // https://github.com/microsoft/vscode-cpptools/issues/722 "C_Cpp.autoAddFileAssociations": false, diff --git a/RELEASE.md b/RELEASE.md deleted file mode 100644 index f3389ea67f..0000000000 --- a/RELEASE.md +++ /dev/null @@ -1,12 +0,0 @@ -# Release - -How to make releases. - -## NPM - -- Checkout the version of the code you want to release -- Update package.json in typescript/core/package.json with the new version. -- Make a PR with your changes to package.json -- Wait for the PR to pass CI and merge -- Checkout main and tag the merged commit with `releases/typescript/core/v#.#.#` (replace #.#.# with the version you used in package.json) -- Push the new tag to the repo with `git push origin releases/typescript/core/v#.#.#` diff --git a/cspell.config.yaml b/cspell.config.yaml index e0f0aff57b..77b1b15ec8 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -14,6 +14,7 @@ ignorePaths: - bin - dist - yarn-error.log + - "*.bfbs" - "*.csv" - "*.Dockerfile" - Dockerfile @@ -88,6 +89,7 @@ words: - rostime - schemaless - serde + - sfixed - srgb - stoull - struct @@ -101,6 +103,7 @@ words: - velodyne - waabi - webp + - xcdr - zstandard - zstd - zustand diff --git a/package.json b/package.json index 208a3ad922..2a936cda28 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,9 @@ "typescript/benchmarks", "typescript/core", "typescript/examples/*", + "typescript/support", + "typescript/nodejs", + "typescript/browser", "website" ] }, @@ -13,16 +16,23 @@ "prettier": "prettier --write .", "prettier:check": "prettier --check .", "docs:swift:start": "swift package --disable-sandbox preview-documentation --target MCAP", - "typedoc": "typedoc --out __docs__/typescript typescript/core/src/index.ts --tsconfig typescript/core/tsconfig.json", + "typedoc": "yarn typescript:build && typedoc --out __docs__/typescript --options typescript/typedoc.json", "start": "yarn workspace website start", "spellcheck": "cspell --relative '**'", + "typescript:test": "yarn jest --config typescript/jest.config.json", + "typescript:build": "yarn workspace @mcap/core build && yarn workspace @mcap/support build && yarn workspace @mcap/nodejs build && yarn workspace @mcap/browser build", + "typescript:clean": "yarn workspace @mcap/core build --clean && yarn workspace @mcap/support build --clean && yarn workspace @mcap/nodejs build --clean && yarn workspace @mcap/browser build --clean", "test:conformance:generate-inputs": "yarn workspace @foxglove/mcap-conformance generate-inputs --data-dir \"$(pwd)/tests/conformance/data\"", "test:conformance": "yarn workspace @foxglove/mcap-conformance run-tests --data-dir \"$(pwd)/tests/conformance/data\"" }, "packageManager": "yarn@3.5.0", "devDependencies": { + "@types/node": "18.13.0", "cspell": "8.0.0", + "jest": "29.7.0", "prettier": "3.1.0", + "ts-jest": "29.1.1", + "ts-node": "10.9.1", "typedoc": "0.25.3", "typescript": "5.2.2" } diff --git a/python/mcap-protobuf-support/README.md b/python/mcap-protobuf-support/README.md index 31698fa70b..9d89c0fa8b 100644 --- a/python/mcap-protobuf-support/README.md +++ b/python/mcap-protobuf-support/README.md @@ -35,4 +35,4 @@ pipenv run python point_cloud_example.py output.mcap ## Stay in touch -Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions, share feedback, and stay up to date on what our team is working on. +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/python/mcap-ros1-support/README.md b/python/mcap-ros1-support/README.md index 6b20603152..48088f23e5 100644 --- a/python/mcap-ros1-support/README.md +++ b/python/mcap-ros1-support/README.md @@ -36,5 +36,5 @@ ros_writer.finish() ## Stay in touch -Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions, +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/python/mcap-ros2-support/README.md b/python/mcap-ros2-support/README.md index 2123ddd7fd..28d11a2d25 100644 --- a/python/mcap-ros2-support/README.md +++ b/python/mcap-ros2-support/README.md @@ -22,5 +22,5 @@ for msg in read_ros2_messages("my_data.mcap"): ## Stay in touch -Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions, +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/tests/conformance/.eslintrc.js b/tests/conformance/.eslintrc.js index 99e1d0a0dc..be9e90eaf6 100644 --- a/tests/conformance/.eslintrc.js +++ b/tests/conformance/.eslintrc.js @@ -8,8 +8,11 @@ module.exports = { files: ["*.ts", "*.tsx"], extends: ["plugin:@foxglove/typescript"], parserOptions: { - project: "tsconfig.json", + project: ["tsconfig.json", "../../typescript/*/tsconfig.json"], tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, }, }, ], diff --git a/tests/conformance/package.json b/tests/conformance/package.json index cdc925c720..a7bb815dde 100644 --- a/tests/conformance/package.json +++ b/tests/conformance/package.json @@ -3,17 +3,18 @@ "version": "0.0.0", "private": true, "scripts": { - "typecheck": "tsc -p tsconfig.json --noEmit", + "build": "tsc -b tsconfig.json tsconfig.cjs.json", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", "generate-inputs": "ts-node --files --project tsconfig.cjs.json scripts/generate-inputs", - "run-tests": "ts-node --files --project tsconfig.cjs.json scripts/run-tests" + "run-tests": "tsc -b tsconfig.json tsconfig.cjs.json && ts-node --files --project tsconfig.cjs.json scripts/run-tests" }, "devDependencies": { "@foxglove/crc": "^0.0.3", - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/tsconfig": "1.1.0", - "@mcap/core": "*", + "@mcap/core": "workspace:*", + "@mcap/support": "workspace:*", "@types/diff": "^5.0.2", "@types/js-yaml": "^4.0.5", "@types/node": "18.13.0", @@ -22,7 +23,7 @@ "colors": "1.4.0", "commander": "11.1.0", "diff": "^5.1.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/tests/conformance/scripts/run-tests/runners/TypescriptIndexedReaderTestRunner.ts b/tests/conformance/scripts/run-tests/runners/TypescriptIndexedReaderTestRunner.ts index 905c1a1441..f2e6f04949 100644 --- a/tests/conformance/scripts/run-tests/runners/TypescriptIndexedReaderTestRunner.ts +++ b/tests/conformance/scripts/run-tests/runners/TypescriptIndexedReaderTestRunner.ts @@ -1,4 +1,5 @@ import { McapIndexedReader } from "@mcap/core"; +import { FileHandleReadable } from "@mcap/nodejs"; import fs from "fs/promises"; import { TestFeatures, TestVariant } from "variants/types"; @@ -41,28 +42,7 @@ export default class TypescriptIndexedReaderTestRunner extends IndexedReadTestRu } async #run(fileHandle: fs.FileHandle): Promise { - let buffer = new ArrayBuffer(4096); - const readable = { - size: async () => BigInt((await fileHandle.stat()).size), - read: async (offset: bigint, length: bigint) => { - if (offset > Number.MAX_SAFE_INTEGER || length > Number.MAX_SAFE_INTEGER) { - throw new Error(`Read too large: offset ${offset}, length ${length}`); - } - if (length > buffer.byteLength) { - buffer = new ArrayBuffer(Number(length * 2n)); - } - const result = await fileHandle.read({ - buffer: new DataView(buffer, 0, Number(length)), - position: Number(offset), - }); - if (result.bytesRead !== Number(length)) { - throw new Error( - `Read only ${result.bytesRead} bytes from offset ${offset}, expected ${length}`, - ); - } - return new Uint8Array(result.buffer.buffer, result.buffer.byteOffset, result.bytesRead); - }, - }; + const readable = new FileHandleReadable(fileHandle); const reader = await McapIndexedReader.Initialize({ readable }); if (reader.chunkIndexes.length === 0) { diff --git a/tests/conformance/tsconfig.cjs.json b/tests/conformance/tsconfig.cjs.json index ac31cb3343..916ffa5a73 100644 --- a/tests/conformance/tsconfig.cjs.json +++ b/tests/conformance/tsconfig.cjs.json @@ -3,5 +3,9 @@ "compilerOptions": { "outDir": "./dist/cjs", "module": "commonjs" - } + }, + "references": [ + { "path": "../../typescript/core/tsconfig.cjs.json" }, + { "path": "../../typescript/support/tsconfig.cjs.json" } + ] } diff --git a/tests/conformance/tsconfig.json b/tests/conformance/tsconfig.json index a8c9f12ab0..2deb6fae3e 100644 --- a/tests/conformance/tsconfig.json +++ b/tests/conformance/tsconfig.json @@ -6,12 +6,19 @@ "noEmit": true, "lib": ["es2020", "dom"], "paths": { - "@mcap/core": ["../../typescript/core/src"] + "@mcap/core": ["../../typescript/core/src"], + "@mcap/nodejs": ["../../typescript/nodejs/src"], + "@mcap/support": ["../../typescript/support/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 "baseUrl": "." }, + "references": [ + { "path": "../../typescript/core" }, + { "path": "../../typescript/nodejs" }, + { "path": "../../typescript/support" } + ], "ts-node": { "require": ["tsconfig-paths/register"] } diff --git a/typescript/CONTRIBUTING.md b/typescript/CONTRIBUTING.md new file mode 100644 index 0000000000..23052bd3e0 --- /dev/null +++ b/typescript/CONTRIBUTING.md @@ -0,0 +1,42 @@ +# Development guide + +Install dependencies: + +``` +corepack enable +yarn install +``` + +Run lint/tests: + +``` +yarn workspace @mcap/core lint +yarn workspace @mcap/core test +``` + +Read and validate an MCAP file: + +``` +yarn workspace @foxglove/mcap-example-validate validate file.mcap +``` + +Run benchmarks: + +``` +yarn workspace @foxglove/mcap-benchmarks bench +``` + +Run benchmarks with Chrome debugger attached to use profiling tools: + +``` +yarn workspace @foxglove/mcap-benchmarks bench:debug +``` + +## Releasing to NPM + +- Check out the version of the code you want to release +- Update package.json in `typescript/{pkg}/package.json` with the new version. +- Make a PR with your changes to package.json +- Wait for the PR to pass CI and merge +- Checkout main and tag the merged commit with `releases/typescript/{pkg}/v#.#.#` (replace #.#.# with the version you used in package.json) +- Push the new tag to the repo with `git push origin releases/typescript/{pkg}/v#.#.#` diff --git a/typescript/README.md b/typescript/README.md index f05e71c0aa..ebcbd87b25 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -1,33 +1,10 @@ # TypeScript libraries for MCAP -Install dependencies: +[MCAP](https://mcap.dev/) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. -``` -corepack enable -yarn install -``` +The following NPM packages are provided for use with JavaScript and TypeScript: -Run lint/tests: - -``` -yarn workspace @mcap/core lint -yarn workspace @mcap/core test -``` - -Read and validate an MCAP file: - -``` -yarn workspace @foxglove/mcap-example-validate validate file.mcap -``` - -Run benchmarks: - -``` -yarn workspace @foxglove/mcap-benchmarks bench -``` - -Run benchmarks with Chrome debugger attached to use profiling tools: - -``` -yarn workspace @foxglove/mcap-benchmarks bench:debug -``` +- **@mcap/core** – low-level readers and writers +- **@mcap/support** – support for well-known compression formats +- **@mcap/nodejs** – support for Node.js environment +- **@mcap/browser** – support for browser environment diff --git a/typescript/benchmarks/package.json b/typescript/benchmarks/package.json index 2129ea854b..925a49dce3 100644 --- a/typescript/benchmarks/package.json +++ b/typescript/benchmarks/package.json @@ -21,14 +21,14 @@ "bench:debug": "NODE_OPTIONS='--inspect-brk' ts-node --files --project tsconfig.cjs.json index.ts" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/tsconfig": "1.1.0", "@mcap/core": "*", "@types/node": "18.13.0", "@typescript-eslint/eslint-plugin": "6.11.0", "@typescript-eslint/parser": "6.11.0", "benny": "^3.7.1", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/typescript/browser/.eslintrc.js b/typescript/browser/.eslintrc.js new file mode 100644 index 0000000000..85bdbf92d7 --- /dev/null +++ b/typescript/browser/.eslintrc.js @@ -0,0 +1,28 @@ +/* eslint-env node */ +module.exports = { + env: { es2020: true }, + ignorePatterns: ["dist"], + extends: ["plugin:@foxglove/base", "plugin:@foxglove/jest", "plugin:import/recommended"], + overrides: [ + { + files: ["*.ts", "*.tsx"], + extends: ["plugin:@foxglove/typescript"], + parserOptions: { + project: "../*/tsconfig.json", + tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, + }, + }, + ], + rules: { + "no-warning-comments": ["error", { terms: ["fixme"], location: "anywhere" }], + }, + settings: { + "import/resolver": { + typescript: true, + node: true, + }, + }, +}; diff --git a/typescript/browser/LICENSE b/typescript/browser/LICENSE new file mode 100644 index 0000000000..0aa593ea8c --- /dev/null +++ b/typescript/browser/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Foxglove Technologies Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/typescript/browser/README.md b/typescript/browser/README.md new file mode 100644 index 0000000000..d3050d47f4 --- /dev/null +++ b/typescript/browser/README.md @@ -0,0 +1,33 @@ +# @mcap/browser + +[MCAP](https://mcap.dev/) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. + +The `@mcap/browser` package provides utilities for working with MCAP files in browsers. + +## Usage examples + +### Reading MCAP files in a browser + +```ts +import { loadDecompressHandlers } from "@mcap/support"; +import { BlobReadable } from "@mcap/browser"; +import { McapIndexedReader } from "@mcap/core"; +import { open } from "fs/promises"; + +async function onInputOrDrop(event: InputEvent | DragEvent) { + const file = event.dataTransfer.files[0]; + const decompressHandlers = await loadDecompressHandlers(); + const reader = await McapIndexedReader.Initialize({ + readable: new BlobReadable(file), + decompressHandlers, + }); +} +``` + +## License + +`@mcap/browser` is licensed under the [MIT License](https://opensource.org/licenses/MIT). + +## Stay in touch + +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/typescript/browser/jest.config.json b/typescript/browser/jest.config.json new file mode 100644 index 0000000000..09337c2e52 --- /dev/null +++ b/typescript/browser/jest.config.json @@ -0,0 +1,19 @@ +{ + "testMatch": ["/src/**/*.test.ts"], + "transform": { + "^.+\\.ts$": [ + "ts-jest", + { + "diagnostics": { + "//": "add 6133 (unused variables) to default ignore codes", + "ignoreCodes": [6059, 18002, 18003, 6133] + } + } + ] + }, + "moduleNameMapper": { + "^@mcap/core$": "/../core/src" + }, + "//": "Native find is slow because it does not exclude files: https://github.com/facebook/jest/pull/11264#issuecomment-825377579", + "haste": { "forceNodeFilesystemAPI": true } +} diff --git a/typescript/browser/package.json b/typescript/browser/package.json new file mode 100644 index 0000000000..786bba60c4 --- /dev/null +++ b/typescript/browser/package.json @@ -0,0 +1,55 @@ +{ + "name": "@mcap/browser", + "version": "1.0.0", + "description": "Support library for using MCAP in the browser", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/foxglove/mcap.git" + }, + "author": { + "name": "Foxglove Technologies", + "email": "support@foxglove.dev" + }, + "homepage": "https://foxglove.dev/", + "module": "dist/esm/src/index.js", + "main": "dist/cjs/src/index.js", + "typings": "dist/esm/src/index.d.ts", + "typedoc": { + "entryPoint": "src/index.ts" + }, + "files": [ + "dist", + "src" + ], + "engines": { + "node": ">=14.0.0" + }, + "scripts": { + "build": "tsc -b tsconfig.json tsconfig.cjs.json", + "prepack": "yarn build", + "lint:ci": "eslint --report-unused-disable-directives .", + "lint": "eslint --report-unused-disable-directives --fix .", + "test": "jest" + }, + "devDependencies": { + "@foxglove/eslint-plugin": "1.0.1", + "@foxglove/tsconfig": "1.1.0", + "@mcap/core": "workspace:*", + "@types/jest": "29.5.8", + "@typescript-eslint/eslint-plugin": "6.11.0", + "@typescript-eslint/parser": "6.11.0", + "eslint": "8.54.0", + "eslint-config-prettier": "9.0.0", + "eslint-import-resolver-typescript": "3.6.1", + "eslint-plugin-es": "4.1.0", + "eslint-plugin-filenames": "1.3.2", + "eslint-plugin-import": "2.29.0", + "eslint-plugin-jest": "27.6.0", + "eslint-plugin-prettier": "5.0.1", + "jest": "29.7.0", + "prettier": "3.1.0", + "ts-jest": "29.1.1", + "typescript": "5.2.2" + } +} diff --git a/typescript/browser/src/BlobReadable.ts b/typescript/browser/src/BlobReadable.ts new file mode 100644 index 0000000000..51a438d998 --- /dev/null +++ b/typescript/browser/src/BlobReadable.ts @@ -0,0 +1,27 @@ +import type { McapTypes } from "@mcap/core"; + +/** + * IReadable implementation for Blob (and File, which is a Blob). + */ +export class BlobReadable implements McapTypes.IReadable { + #blob: Blob; + + public constructor(blob: Blob) { + this.#blob = blob; + } + + public async size(): Promise { + return BigInt(this.#blob.size); + } + + public async read(offset: bigint, size: bigint): Promise { + if (offset + size > this.#blob.size) { + throw new Error( + `Read of ${size} bytes at offset ${offset} exceeds file size ${this.#blob.size}`, + ); + } + return new Uint8Array( + await this.#blob.slice(Number(offset), Number(offset + size)).arrayBuffer(), + ); + } +} diff --git a/typescript/browser/src/index.test.ts b/typescript/browser/src/index.test.ts new file mode 100644 index 0000000000..77e95f993e --- /dev/null +++ b/typescript/browser/src/index.test.ts @@ -0,0 +1,46 @@ +import { McapWriter, McapIndexedReader, TempBuffer } from "@mcap/core"; +import { Blob } from "buffer"; + +import { BlobReadable } from "./BlobReadable"; + +async function collect(iterable: AsyncIterable): Promise { + const result: T[] = []; + for await (const item of iterable) { + result.push(item); + } + return result; +} + +describe("BlobReadable", () => { + it("reads blob", async () => { + const tempBuffer = new TempBuffer(); + + const header = { library: "lib", profile: "prof" }; + const writer = new McapWriter({ writable: tempBuffer }); + await writer.start(header); + const channel = { + topic: "foo", + schemaId: 0, + messageEncoding: "enc", + metadata: new Map(), + }; + const channelId = await writer.registerChannel(channel); + const message = { + channelId, + sequence: 1, + logTime: 1n, + publishTime: 2n, + data: new Uint8Array([1, 2, 3]), + }; + await writer.addMessage(message); + await writer.end(); + + const blob = new Blob([tempBuffer.get()]); + + const reader = await McapIndexedReader.Initialize({ + readable: new BlobReadable(blob as unknown as ConstructorParameters[0]), + }); + expect(reader.header).toEqual({ ...header, type: "Header" }); + expect(await collect(reader.readMessages())).toEqual([{ ...message, type: "Message" }]); + }); +}); diff --git a/typescript/browser/src/index.ts b/typescript/browser/src/index.ts new file mode 100644 index 0000000000..cd3b52e1bd --- /dev/null +++ b/typescript/browser/src/index.ts @@ -0,0 +1 @@ +export * from "./BlobReadable"; diff --git a/typescript/browser/tsconfig.cjs.json b/typescript/browser/tsconfig.cjs.json new file mode 100644 index 0000000000..ac31cb3343 --- /dev/null +++ b/typescript/browser/tsconfig.cjs.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist/cjs", + "module": "commonjs" + } +} diff --git a/typescript/browser/tsconfig.json b/typescript/browser/tsconfig.json new file mode 100644 index 0000000000..6a6b398a2e --- /dev/null +++ b/typescript/browser/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "@foxglove/tsconfig/base", + "include": ["./src/**/*"], + "compilerOptions": { + "outDir": "./dist/esm", + "lib": ["es2020", "dom"], + "composite": true, + "incremental": true + }, + "references": [{ "path": "../core" }] +} diff --git a/typescript/core/README.md b/typescript/core/README.md index 160403fd35..2e007000cd 100644 --- a/typescript/core/README.md +++ b/typescript/core/README.md @@ -1,9 +1,17 @@ # @mcap/core -[MCAP](https://github.com/foxglove/mcap) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. +[MCAP](https://mcap.dev/) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. The `@mcap/core` package provides low-level readers and writers for the MCAP format in TypeScript. ## Examples Examples of how to use the `@mcap/core` APIs can be found in the [TypeScript examples folder](https://github.com/foxglove/mcap/tree/main/typescript/examples) in the MCAP repo. + +## License + +`@mcap/core` is licensed under the [MIT License](https://opensource.org/licenses/MIT). + +## Stay in touch + +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/typescript/core/jest.config.json b/typescript/core/jest.config.json index c5dcd8fa38..57e9583c8d 100644 --- a/typescript/core/jest.config.json +++ b/typescript/core/jest.config.json @@ -1,15 +1,15 @@ { "testMatch": ["/src/**/*.test.ts"], "transform": { - "^.+\\.ts$": "ts-jest" - }, - "globals": { - "ts-jest": { - "diagnostics": { - "//": "add 6133 (unused variables) to default ignore codes", - "ignoreCodes": [6059, 18002, 18003, 6133] + "^.+\\.ts$": [ + "ts-jest", + { + "diagnostics": { + "//": "add 6133 (unused variables) to default ignore codes", + "ignoreCodes": [6059, 18002, 18003, 6133] + } } - } + ] }, "//": "Native find is slow because it does not exclude files: https://github.com/facebook/jest/pull/11264#issuecomment-825377579", "haste": { "forceNodeFilesystemAPI": true } diff --git a/typescript/core/package.json b/typescript/core/package.json index bb66139bcf..bbaea2cc50 100644 --- a/typescript/core/package.json +++ b/typescript/core/package.json @@ -15,27 +15,30 @@ "module": "dist/esm/src/index.js", "main": "dist/cjs/src/index.js", "typings": "dist/esm/src/index.d.ts", - "typedocMain": "src/index.ts", + "typedoc": { + "entryPoint": "src/index.ts" + }, "files": [ "dist", "src" ], "scripts": { - "prepack": "tsc -b tsconfig.json tsconfig.cjs.json", + "build": "tsc -b tsconfig.json tsconfig.cjs.json", + "prepack": "yarn build", "typecheck": "tsc -p tsconfig.json --noEmit", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", "test": "jest" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/tsconfig": "1.1.0", "@types/jest": "29.5.8", "@types/lodash": "4.14.191", "@types/node": "18.13.0", "@typescript-eslint/eslint-plugin": "6.11.0", "@typescript-eslint/parser": "6.11.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/typescript/core/src/TempBuffer.ts b/typescript/core/src/TempBuffer.ts index a36dd29d3e..e81ba1712e 100644 --- a/typescript/core/src/TempBuffer.ts +++ b/typescript/core/src/TempBuffer.ts @@ -1,6 +1,9 @@ import { IWritable } from "./IWritable"; import { IReadable } from "./types"; +/** + * In-memory buffer used for reading and writing MCAP files in tests. Can be used as both an IReadable and an IWritable. + */ export class TempBuffer implements IReadable, IWritable { #buffer = new ArrayBuffer(1024); #size = 0; diff --git a/typescript/core/src/index.ts b/typescript/core/src/index.ts index 8edfe0416a..051122da3b 100644 --- a/typescript/core/src/index.ts +++ b/typescript/core/src/index.ts @@ -10,3 +10,4 @@ export type { IWritable } from "./IWritable"; export * from "./hasMcapPrefix"; export * from "./parse"; +export * from "./TempBuffer"; diff --git a/typescript/core/tsconfig.json b/typescript/core/tsconfig.json index 4e3cd642e8..260bc429e0 100644 --- a/typescript/core/tsconfig.json +++ b/typescript/core/tsconfig.json @@ -6,6 +6,8 @@ "outDir": "./dist/esm", "module": "es2022", "target": "es2022", - "lib": ["es2022", "dom"] + "lib": ["es2022", "dom"], + "composite": true, + "incremental": true } } diff --git a/typescript/examples/bag2mcap/.eslintrc.js b/typescript/examples/bag2mcap/.eslintrc.js index 8eb03eee3d..cabff403f4 100644 --- a/typescript/examples/bag2mcap/.eslintrc.js +++ b/typescript/examples/bag2mcap/.eslintrc.js @@ -8,8 +8,11 @@ module.exports = { files: ["*.ts", "*.tsx"], extends: ["plugin:@foxglove/typescript"], parserOptions: { - project: "tsconfig.json", + project: ["tsconfig.json", "../../*/tsconfig.json"], tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, }, }, ], diff --git a/typescript/examples/bag2mcap/package.json b/typescript/examples/bag2mcap/package.json index 9915a07176..658365ef80 100644 --- a/typescript/examples/bag2mcap/package.json +++ b/typescript/examples/bag2mcap/package.json @@ -17,10 +17,10 @@ "typecheck": "tsc -p tsconfig.json --noEmit", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", - "bag2mcap": "ts-node --files --project tsconfig.cjs.json scripts/bag2mcap.ts" + "bag2mcap": "yarn typescript:build && ts-node --files --project tsconfig.cjs.json scripts/bag2mcap.ts" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/rosbag": "0.2.3", "@foxglove/rosmsg": "3.1.0", "@foxglove/rosmsg-serialization": "1.5.3", @@ -29,25 +29,23 @@ "@foxglove/wasm-bz2": "0.1.1", "@foxglove/wasm-lz4": "1.0.2", "@foxglove/wasm-zstd": "1.0.1", - "@mcap/core": "*", - "@types/jest": "29.5.8", + "@mcap/core": "workspace:*", + "@mcap/nodejs": "workspace:*", + "@mcap/support": "workspace:*", "@types/lodash": "4.14.191", "@types/node": "18.13.0", "@typescript-eslint/eslint-plugin": "6.11.0", "@typescript-eslint/parser": "6.11.0", "commander": "11.1.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", "eslint-plugin-import": "2.29.0", - "eslint-plugin-jest": "27.6.0", "eslint-plugin-prettier": "5.0.1", - "jest": "29.7.0", "lodash": "4.17.21", "prettier": "3.1.0", - "protobufjs": "7.2.2", - "ts-jest": "29.1.1", + "protobufjs": "7.2.5", "ts-node": "10.9.1", "tsconfig-paths": "4.1.2", "typescript": "5.2.2" diff --git a/typescript/examples/bag2mcap/scripts/bag2mcap.ts b/typescript/examples/bag2mcap/scripts/bag2mcap.ts index aa734ef09d..86e54e9b0e 100644 --- a/typescript/examples/bag2mcap/scripts/bag2mcap.ts +++ b/typescript/examples/bag2mcap/scripts/bag2mcap.ts @@ -1,7 +1,3 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/ - // convert a ROS1 .bag file to an mcap file with protobuf schema and message encoding import { Bag } from "@foxglove/rosbag"; @@ -12,9 +8,11 @@ import { toNanoSec } from "@foxglove/rostime"; import Bzip2 from "@foxglove/wasm-bz2"; import decompressLZ4 from "@foxglove/wasm-lz4"; import zstd from "@foxglove/wasm-zstd"; -import { McapWriter, IWritable, McapTypes } from "@mcap/core"; +import { McapWriter, McapTypes } from "@mcap/core"; +import { FileHandleWritable } from "@mcap/nodejs"; +import { ProtobufDescriptor, protobufToDescriptor } from "@mcap/support"; import { program } from "commander"; -import { open, FileHandle } from "fs/promises"; +import { open } from "fs/promises"; import protobufjs from "protobufjs"; import descriptor from "protobufjs/ext/descriptor"; @@ -55,7 +53,7 @@ function rosMsgDefinitionToProto( msgDef: string, ): { rootType: protobufjs.Type; - descriptorSet: ReturnType; + descriptorSet: ProtobufDescriptor; schemaName: string; } { const definitionArr = parseMessageDefinition(msgDef); @@ -153,7 +151,7 @@ function rosMsgDefinitionToProto( const rootType = root.lookupType(schemaName); // create a descriptor message for the root - const descriptorSet = root.toDescriptor("proto3"); + const descriptorSet = protobufToDescriptor(root); for (const file of descriptorSet.file) { // Strip leading `.` from the package names to make them relative to the descriptor file.package = file.package?.substring(1); @@ -193,25 +191,6 @@ function convertTypedArrays(msg: Record): Record { - const written = await this.#handle.write(buffer); - this.#totalBytesWritten += written.bytesWritten; - } - - position(): bigint { - return BigInt(this.#totalBytesWritten); - } -} - async function convert(filePath: string, options: { indexed: boolean }) { await decompressLZ4.isLoaded; await zstd.isLoaded; diff --git a/typescript/examples/bag2mcap/tsconfig.json b/typescript/examples/bag2mcap/tsconfig.json index 96c3435379..466eaa4a9a 100644 --- a/typescript/examples/bag2mcap/tsconfig.json +++ b/typescript/examples/bag2mcap/tsconfig.json @@ -8,7 +8,9 @@ "target": "es2022", "lib": ["es2022", "dom"], "paths": { - "@mcap/core": ["../../core/src"] + "@mcap/core": ["../../core/src"], + "@mcap/support": ["../../support/src"], + "@mcap/nodejs": ["../../nodejs/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/examples/basicwriter/.eslintrc.js b/typescript/examples/basicwriter/.eslintrc.js index 8eb03eee3d..cabff403f4 100644 --- a/typescript/examples/basicwriter/.eslintrc.js +++ b/typescript/examples/basicwriter/.eslintrc.js @@ -8,8 +8,11 @@ module.exports = { files: ["*.ts", "*.tsx"], extends: ["plugin:@foxglove/typescript"], parserOptions: { - project: "tsconfig.json", + project: ["tsconfig.json", "../../*/tsconfig.json"], tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, }, }, ], diff --git a/typescript/examples/basicwriter/package.json b/typescript/examples/basicwriter/package.json index f8f21b548a..ab294dbca7 100644 --- a/typescript/examples/basicwriter/package.json +++ b/typescript/examples/basicwriter/package.json @@ -17,13 +17,15 @@ "typecheck": "tsc -p tsconfig.json --noEmit", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", - "main": "ts-node --files --project tsconfig.cjs.json scripts/main.ts" + "main": "yarn typescript:build && ts-node --files --project tsconfig.cjs.json scripts/main.ts" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", - "@mcap/core": "*", + "@foxglove/eslint-plugin": "1.0.1", + "@mcap/core": "workspace:*", + "@mcap/nodejs": "workspace:*", + "@mcap/support": "workspace:*", "@types/node": "18.13.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/typescript/examples/basicwriter/scripts/main.ts b/typescript/examples/basicwriter/scripts/main.ts index fec92cfdfe..3e6f6c9858 100644 --- a/typescript/examples/basicwriter/scripts/main.ts +++ b/typescript/examples/basicwriter/scripts/main.ts @@ -1,24 +1,6 @@ -import { McapWriter, IWritable } from "@mcap/core"; -import { open, FileHandle } from "fs/promises"; - -// Mcap IWritable interface for nodejs FileHandle -class FileHandleWritable implements IWritable { - #handle: FileHandle; - #totalBytesWritten = 0; - - constructor(handle: FileHandle) { - this.#handle = handle; - } - - async write(buffer: Uint8Array): Promise { - const written = await this.#handle.write(buffer); - this.#totalBytesWritten += written.bytesWritten; - } - - position(): bigint { - return BigInt(this.#totalBytesWritten); - } -} +import { McapWriter } from "@mcap/core"; +import { FileHandleWritable } from "@mcap/nodejs"; +import { open } from "fs/promises"; async function main() { const mcapFilePath = "output.mcap"; diff --git a/typescript/examples/basicwriter/tsconfig.json b/typescript/examples/basicwriter/tsconfig.json index 96c3435379..466eaa4a9a 100644 --- a/typescript/examples/basicwriter/tsconfig.json +++ b/typescript/examples/basicwriter/tsconfig.json @@ -8,7 +8,9 @@ "target": "es2022", "lib": ["es2022", "dom"], "paths": { - "@mcap/core": ["../../core/src"] + "@mcap/core": ["../../core/src"], + "@mcap/support": ["../../support/src"], + "@mcap/nodejs": ["../../nodejs/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/examples/flatbufferswriter/.eslintrc.js b/typescript/examples/flatbufferswriter/.eslintrc.js index 8eb03eee3d..cabff403f4 100644 --- a/typescript/examples/flatbufferswriter/.eslintrc.js +++ b/typescript/examples/flatbufferswriter/.eslintrc.js @@ -8,8 +8,11 @@ module.exports = { files: ["*.ts", "*.tsx"], extends: ["plugin:@foxglove/typescript"], parserOptions: { - project: "tsconfig.json", + project: ["tsconfig.json", "../../*/tsconfig.json"], tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, }, }, ], diff --git a/typescript/examples/flatbufferswriter/package.json b/typescript/examples/flatbufferswriter/package.json index 13e2a391be..1d0221bc82 100644 --- a/typescript/examples/flatbufferswriter/package.json +++ b/typescript/examples/flatbufferswriter/package.json @@ -17,14 +17,15 @@ "typecheck": "tsc -p tsconfig.json --noEmit", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", - "main": "ts-node --files --project tsconfig.cjs.json scripts/main.ts" + "main": "yarn typescript:build && ts-node --files --project tsconfig.cjs.json scripts/main.ts" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/schemas": "^1.0.0", - "@mcap/core": "*", + "@mcap/core": "workspace:*", + "@mcap/nodejs": "workspace:*", "@types/node": "18.13.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/typescript/examples/flatbufferswriter/scripts/main.ts b/typescript/examples/flatbufferswriter/scripts/main.ts index 3c45f291b0..59ba0c4144 100644 --- a/typescript/examples/flatbufferswriter/scripts/main.ts +++ b/typescript/examples/flatbufferswriter/scripts/main.ts @@ -1,31 +1,14 @@ import { Grid, NumericType } from "@foxglove/schemas"; -import { McapWriter, IWritable } from "@mcap/core"; +import { McapWriter } from "@mcap/core"; +import { FileHandleWritable } from "@mcap/nodejs"; import { Builder } from "flatbuffers"; import fs from "fs"; -import { open, FileHandle } from "fs/promises"; +import { open } from "fs/promises"; import { buildGridMessage, buildTfMessage } from "./flatbufferUtils"; const QUAT_IDENTITY = { x: 0, y: 0, z: 0, w: 1 }; -// Mcap IWritable interface for nodejs FileHandle -class FileHandleWritable implements IWritable { - #handle: FileHandle; - #totalBytesWritten = 0; - - constructor(handle: FileHandle) { - this.#handle = handle; - } - - async write(buffer: Uint8Array): Promise { - const written = await this.#handle.write(buffer); - this.#totalBytesWritten += written.bytesWritten; - } - - position(): bigint { - return BigInt(this.#totalBytesWritten); - } -} function nextPowerOfTwo(numToRound: number) { let nextPower = 1; while (nextPower < numToRound) { diff --git a/typescript/examples/flatbufferswriter/tsconfig.json b/typescript/examples/flatbufferswriter/tsconfig.json index 96c3435379..466eaa4a9a 100644 --- a/typescript/examples/flatbufferswriter/tsconfig.json +++ b/typescript/examples/flatbufferswriter/tsconfig.json @@ -8,7 +8,9 @@ "target": "es2022", "lib": ["es2022", "dom"], "paths": { - "@mcap/core": ["../../core/src"] + "@mcap/core": ["../../core/src"], + "@mcap/support": ["../../support/src"], + "@mcap/nodejs": ["../../nodejs/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/examples/text-annotation-demo/.eslintrc.js b/typescript/examples/text-annotation-demo/.eslintrc.js index 8eb03eee3d..cabff403f4 100644 --- a/typescript/examples/text-annotation-demo/.eslintrc.js +++ b/typescript/examples/text-annotation-demo/.eslintrc.js @@ -8,8 +8,11 @@ module.exports = { files: ["*.ts", "*.tsx"], extends: ["plugin:@foxglove/typescript"], parserOptions: { - project: "tsconfig.json", + project: ["tsconfig.json", "../../*/tsconfig.json"], tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, }, }, ], diff --git a/typescript/examples/text-annotation-demo/package.json b/typescript/examples/text-annotation-demo/package.json index 81f73f6f73..a56a28853c 100644 --- a/typescript/examples/text-annotation-demo/package.json +++ b/typescript/examples/text-annotation-demo/package.json @@ -20,10 +20,11 @@ "main": "ts-node --files --project tsconfig.cjs.json scripts/main.ts" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", - "@mcap/core": "*", + "@foxglove/eslint-plugin": "1.0.1", + "@mcap/core": "workspace:*", + "@mcap/nodejs": "workspace:*", "@types/node": "18.13.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/typescript/examples/text-annotation-demo/scripts/main.ts b/typescript/examples/text-annotation-demo/scripts/main.ts index 2b15299e5a..4bb99b0121 100644 --- a/typescript/examples/text-annotation-demo/scripts/main.ts +++ b/typescript/examples/text-annotation-demo/scripts/main.ts @@ -4,30 +4,12 @@ import { ImageAnnotations as ImageAnnotationsSchema, } from "@foxglove/schemas/jsonschema"; import { Time } from "@foxglove/schemas/schemas/typescript/Time"; -import { McapWriter, IWritable } from "@mcap/core"; -import { open, FileHandle } from "fs/promises"; +import { McapWriter } from "@mcap/core"; +import { FileHandleWritable } from "@mcap/nodejs"; +import { open } from "fs/promises"; import Scene from "./Scene"; -// Mcap IWritable interface for nodejs FileHandle -class FileHandleWritable implements IWritable { - #handle: FileHandle; - #totalBytesWritten = 0; - - constructor(handle: FileHandle) { - this.#handle = handle; - } - - async write(buffer: Uint8Array): Promise { - const written = await this.#handle.write(buffer); - this.#totalBytesWritten += written.bytesWritten; - } - - position(): bigint { - return BigInt(this.#totalBytesWritten); - } -} - const framesPerSecond = 30; const lengthSeconds = 10; // seconds diff --git a/typescript/examples/text-annotation-demo/tsconfig.json b/typescript/examples/text-annotation-demo/tsconfig.json index 96c3435379..466eaa4a9a 100644 --- a/typescript/examples/text-annotation-demo/tsconfig.json +++ b/typescript/examples/text-annotation-demo/tsconfig.json @@ -8,7 +8,9 @@ "target": "es2022", "lib": ["es2022", "dom"], "paths": { - "@mcap/core": ["../../core/src"] + "@mcap/core": ["../../core/src"], + "@mcap/support": ["../../support/src"], + "@mcap/nodejs": ["../../nodejs/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/examples/validate/.eslintrc.js b/typescript/examples/validate/.eslintrc.js index 8eb03eee3d..cabff403f4 100644 --- a/typescript/examples/validate/.eslintrc.js +++ b/typescript/examples/validate/.eslintrc.js @@ -8,8 +8,11 @@ module.exports = { files: ["*.ts", "*.tsx"], extends: ["plugin:@foxglove/typescript"], parserOptions: { - project: "tsconfig.json", + project: ["tsconfig.json", "../../*/tsconfig.json"], tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, }, }, ], diff --git a/typescript/examples/validate/package.json b/typescript/examples/validate/package.json index 97f1fdbc78..cdea642010 100644 --- a/typescript/examples/validate/package.json +++ b/typescript/examples/validate/package.json @@ -17,10 +17,10 @@ "typecheck": "tsc -p tsconfig.json --noEmit", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", - "validate": "ts-node --files --project tsconfig.cjs.json scripts/validate.ts" + "validate": "yarn typescript:build && ts-node --files --project tsconfig.cjs.json scripts/validate.ts" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/rosbag": "0.2.3", "@foxglove/rosmsg": "3.1.0", "@foxglove/rosmsg-serialization": "1.5.3", @@ -29,14 +29,16 @@ "@foxglove/wasm-bz2": "0.1.1", "@foxglove/wasm-lz4": "1.0.2", "@foxglove/wasm-zstd": "1.0.1", - "@mcap/core": "*", + "@mcap/core": "workspace:*", + "@mcap/nodejs": "workspace:*", + "@mcap/support": "workspace:*", "@types/jest": "29.5.8", "@types/lodash": "4.14.191", "@types/node": "18.13.0", "@typescript-eslint/eslint-plugin": "6.11.0", "@typescript-eslint/parser": "6.11.0", "commander": "11.1.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", @@ -46,7 +48,7 @@ "jest": "29.7.0", "lodash": "4.17.21", "prettier": "3.1.0", - "protobufjs": "7.2.2", + "protobufjs": "7.2.5", "ts-jest": "29.1.1", "ts-node": "10.9.1", "tsconfig-paths": "4.1.2", diff --git a/typescript/examples/validate/scripts/validate.ts b/typescript/examples/validate/scripts/validate.ts index e4b35a69d5..150186f794 100644 --- a/typescript/examples/validate/scripts/validate.ts +++ b/typescript/examples/validate/scripts/validate.ts @@ -1,8 +1,6 @@ import { parse as parseMessageDefinition } from "@foxglove/rosmsg"; import { LazyMessageReader as ROS1LazyMessageReader } from "@foxglove/rosmsg-serialization"; import { MessageReader as ROS2MessageReader } from "@foxglove/rosmsg2-serialization"; -import decompressLZ4 from "@foxglove/wasm-lz4"; -import zstd from "@foxglove/wasm-zstd"; import { hasMcapPrefix, McapConstants, @@ -10,6 +8,8 @@ import { McapStreamReader, McapTypes, } from "@mcap/core"; +import { FileHandleReadable } from "@mcap/nodejs"; +import { loadDecompressHandlers } from "@mcap/support"; import { program } from "commander"; import { createReadStream } from "fs"; import fs from "fs/promises"; @@ -19,7 +19,6 @@ import protobufjs from "protobufjs"; import { FileDescriptorSet } from "protobufjs/ext/descriptor"; type Channel = McapTypes.Channel; -type DecompressHandlers = McapTypes.DecompressHandlers; type TypedMcapRecord = McapTypes.TypedMcapRecord; function log(...data: unknown[]) { @@ -91,13 +90,7 @@ async function validate( filePath: string, { deserialize, dump, stream }: { deserialize: boolean; dump: boolean; stream: boolean }, ) { - await decompressLZ4.isLoaded; - await zstd.isLoaded; - - const decompressHandlers: DecompressHandlers = { - lz4: (buffer, decompressedSize) => decompressLZ4(buffer, Number(decompressedSize)), - zstd: (buffer, decompressedSize) => zstd.decompress(buffer, Number(decompressedSize)), - }; + const decompressHandlers = await loadDecompressHandlers(); const recordCounts = new Map(); const schemasById = new Map(); @@ -230,29 +223,8 @@ async function validate( if (!stream) { const handle = await fs.open(filePath, "r"); try { - let buffer = new ArrayBuffer(4096); const reader = await McapIndexedReader.Initialize({ - readable: { - size: async () => BigInt((await handle.stat()).size), - read: async (offset, length) => { - if (offset > Number.MAX_SAFE_INTEGER || length > Number.MAX_SAFE_INTEGER) { - throw new Error(`Read too large: offset ${offset}, length ${length}`); - } - if (length > buffer.byteLength) { - buffer = new ArrayBuffer(Number(length * 2n)); - } - const result = await handle.read({ - buffer: new DataView(buffer, 0, Number(length)), - position: Number(offset), - }); - if (result.bytesRead !== Number(length)) { - throw new Error( - `Read only ${result.bytesRead} bytes from offset ${offset}, expected ${length}`, - ); - } - return new Uint8Array(result.buffer.buffer, result.buffer.byteOffset, result.bytesRead); - }, - }, + readable: new FileHandleReadable(handle), decompressHandlers, }); for (const record of reader.schemasById.values()) { diff --git a/typescript/examples/validate/tsconfig.json b/typescript/examples/validate/tsconfig.json index 96c3435379..466eaa4a9a 100644 --- a/typescript/examples/validate/tsconfig.json +++ b/typescript/examples/validate/tsconfig.json @@ -8,7 +8,9 @@ "target": "es2022", "lib": ["es2022", "dom"], "paths": { - "@mcap/core": ["../../core/src"] + "@mcap/core": ["../../core/src"], + "@mcap/support": ["../../support/src"], + "@mcap/nodejs": ["../../nodejs/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/jest.config.json b/typescript/jest.config.json new file mode 100644 index 0000000000..f631460334 --- /dev/null +++ b/typescript/jest.config.json @@ -0,0 +1,3 @@ +{ + "projects": ["/*/jest.config.json"] +} diff --git a/typescript/nodejs/.eslintrc.js b/typescript/nodejs/.eslintrc.js new file mode 100644 index 0000000000..85bdbf92d7 --- /dev/null +++ b/typescript/nodejs/.eslintrc.js @@ -0,0 +1,28 @@ +/* eslint-env node */ +module.exports = { + env: { es2020: true }, + ignorePatterns: ["dist"], + extends: ["plugin:@foxglove/base", "plugin:@foxglove/jest", "plugin:import/recommended"], + overrides: [ + { + files: ["*.ts", "*.tsx"], + extends: ["plugin:@foxglove/typescript"], + parserOptions: { + project: "../*/tsconfig.json", + tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, + }, + }, + ], + rules: { + "no-warning-comments": ["error", { terms: ["fixme"], location: "anywhere" }], + }, + settings: { + "import/resolver": { + typescript: true, + node: true, + }, + }, +}; diff --git a/typescript/nodejs/LICENSE b/typescript/nodejs/LICENSE new file mode 100644 index 0000000000..0aa593ea8c --- /dev/null +++ b/typescript/nodejs/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Foxglove Technologies Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/typescript/nodejs/README.md b/typescript/nodejs/README.md new file mode 100644 index 0000000000..b175e47638 --- /dev/null +++ b/typescript/nodejs/README.md @@ -0,0 +1,50 @@ +# @mcap/nodejs + +[MCAP](https://mcap.dev/) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. + +The `@mcap/nodejs` package provides utilities for working with MCAP files from Node.js. + +## Usage examples + +### Reading MCAP files + +```ts +import { loadDecompressHandlers } from "@mcap/support"; +import { FileHandleReadable } from "@mcap/nodejs"; +import { McapIndexedReader } from "@mcap/core"; +import { open } from "fs/promises"; + +const decompressHandlers = await loadDecompressHandlers(); +const fileHandle = await open("file.mcap", "r"); +const reader = await McapIndexedReader.Initialize({ + readable: new FileHandleReadable(fileHandle), + decompressHandlers, +}); +``` + +### Writing MCAP files + +```ts +import zstd from "@foxglove/wasm-zstd"; +import { FileHandleWritable } from "@mcap/support/nodejs"; +import { McapWriter } from "@mcap/core"; +import { open } from "fs/promises"; + +await zstd.isLoaded; +const fileHandle = await open("file.mcap", "wx"); +const writer = new McapWriter({ + writable: new FileHandleWritable(fileHandle), + compressChunk: (data) => ({ + compression: "zstd", + compressedData: zstd.compress(data), + }), +}); +``` + +## License + +`@mcap/nodejs` is licensed under the [MIT License](https://opensource.org/licenses/MIT). + +## Stay in touch + +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/typescript/nodejs/jest.config.json b/typescript/nodejs/jest.config.json new file mode 100644 index 0000000000..09337c2e52 --- /dev/null +++ b/typescript/nodejs/jest.config.json @@ -0,0 +1,19 @@ +{ + "testMatch": ["/src/**/*.test.ts"], + "transform": { + "^.+\\.ts$": [ + "ts-jest", + { + "diagnostics": { + "//": "add 6133 (unused variables) to default ignore codes", + "ignoreCodes": [6059, 18002, 18003, 6133] + } + } + ] + }, + "moduleNameMapper": { + "^@mcap/core$": "/../core/src" + }, + "//": "Native find is slow because it does not exclude files: https://github.com/facebook/jest/pull/11264#issuecomment-825377579", + "haste": { "forceNodeFilesystemAPI": true } +} diff --git a/typescript/nodejs/package.json b/typescript/nodejs/package.json new file mode 100644 index 0000000000..f1f27f763a --- /dev/null +++ b/typescript/nodejs/package.json @@ -0,0 +1,56 @@ +{ + "name": "@mcap/nodejs", + "version": "1.0.0", + "description": "Support library for using MCAP in Node.js", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/foxglove/mcap.git" + }, + "author": { + "name": "Foxglove Technologies", + "email": "support@foxglove.dev" + }, + "homepage": "https://foxglove.dev/", + "module": "dist/esm/src/index.js", + "main": "dist/cjs/src/index.js", + "typings": "dist/esm/src/index.d.ts", + "typedoc": { + "entryPoint": "src/index.ts" + }, + "files": [ + "dist", + "src" + ], + "engines": { + "node": ">=14.0.0" + }, + "scripts": { + "build": "tsc -b tsconfig.json tsconfig.cjs.json", + "prepack": "yarn build", + "lint:ci": "eslint --report-unused-disable-directives .", + "lint": "eslint --report-unused-disable-directives --fix .", + "test": "jest" + }, + "devDependencies": { + "@foxglove/eslint-plugin": "1.0.1", + "@foxglove/tsconfig": "1.1.0", + "@mcap/core": "workspace:*", + "@types/jest": "29.5.8", + "@types/node": "18.13.0", + "@typescript-eslint/eslint-plugin": "6.11.0", + "@typescript-eslint/parser": "6.11.0", + "eslint": "8.54.0", + "eslint-config-prettier": "9.0.0", + "eslint-import-resolver-typescript": "3.6.1", + "eslint-plugin-es": "4.1.0", + "eslint-plugin-filenames": "1.3.2", + "eslint-plugin-import": "2.29.0", + "eslint-plugin-jest": "27.6.0", + "eslint-plugin-prettier": "5.0.1", + "jest": "29.7.0", + "prettier": "3.1.0", + "ts-jest": "29.1.1", + "typescript": "5.2.2" + } +} diff --git a/typescript/nodejs/src/FileHandleReadable.ts b/typescript/nodejs/src/FileHandleReadable.ts new file mode 100644 index 0000000000..9de82e3e79 --- /dev/null +++ b/typescript/nodejs/src/FileHandleReadable.ts @@ -0,0 +1,37 @@ +import type { McapTypes } from "@mcap/core"; +import { FileHandle } from "fs/promises"; + +/** + * IReadable implementation for FileHandle. + */ +export class FileHandleReadable implements McapTypes.IReadable { + #handle: FileHandle; + #buffer = new ArrayBuffer(4096); + + constructor(handle: FileHandle) { + this.#handle = handle; + } + + async size(): Promise { + return BigInt((await this.#handle.stat()).size); + } + + async read(offset: bigint, length: bigint): Promise { + if (offset > Number.MAX_SAFE_INTEGER || length > Number.MAX_SAFE_INTEGER) { + throw new Error(`Read too large: offset ${offset}, length ${length}`); + } + if (length > this.#buffer.byteLength) { + this.#buffer = new ArrayBuffer(Number(length * 2n)); + } + const result = await this.#handle.read({ + buffer: new DataView(this.#buffer, 0, Number(length)), + position: Number(offset), + }); + if (result.bytesRead !== Number(length)) { + throw new Error( + `Read only ${result.bytesRead} bytes from offset ${offset}, expected ${length}`, + ); + } + return new Uint8Array(result.buffer.buffer, result.buffer.byteOffset, result.bytesRead); + } +} diff --git a/typescript/nodejs/src/FileHandleWritable.ts b/typescript/nodejs/src/FileHandleWritable.ts new file mode 100644 index 0000000000..6b8a7f095b --- /dev/null +++ b/typescript/nodejs/src/FileHandleWritable.ts @@ -0,0 +1,23 @@ +import type { IWritable } from "@mcap/core"; +import { FileHandle } from "fs/promises"; + +/** + * IWritable implementation for FileHandle. + */ +export class FileHandleWritable implements IWritable { + #handle: FileHandle; + #totalBytesWritten = 0; + + constructor(handle: FileHandle) { + this.#handle = handle; + } + + async write(buffer: Uint8Array): Promise { + const written = await this.#handle.write(buffer); + this.#totalBytesWritten += written.bytesWritten; + } + + position(): bigint { + return BigInt(this.#totalBytesWritten); + } +} diff --git a/typescript/nodejs/src/index.test.ts b/typescript/nodejs/src/index.test.ts new file mode 100644 index 0000000000..b9969eb09d --- /dev/null +++ b/typescript/nodejs/src/index.test.ts @@ -0,0 +1,52 @@ +import { McapWriter, McapIndexedReader } from "@mcap/core"; +import { mkdtemp, open, rm } from "fs/promises"; +import { tmpdir } from "os"; +import path from "path"; + +import { FileHandleReadable } from "./FileHandleReadable"; +import { FileHandleWritable } from "./FileHandleWritable"; + +async function collect(iterable: AsyncIterable): Promise { + const result: T[] = []; + for await (const item of iterable) { + result.push(item); + } + return result; +} + +describe("FileHandleReadable & FileHandleWritable", () => { + it("roundtrips", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "mcap-test")); + try { + const handle = await open(path.join(dir, "test.mcap"), "wx+"); + + const header = { library: "lib", profile: "prof" }; + const writer = new McapWriter({ writable: new FileHandleWritable(handle) }); + await writer.start(header); + const channel = { + topic: "foo", + schemaId: 0, + messageEncoding: "enc", + metadata: new Map(), + }; + const channelId = await writer.registerChannel(channel); + const message = { + channelId, + sequence: 1, + logTime: 1n, + publishTime: 2n, + data: new Uint8Array([1, 2, 3]), + }; + await writer.addMessage(message); + await writer.end(); + + const reader = await McapIndexedReader.Initialize({ + readable: new FileHandleReadable(handle), + }); + expect(reader.header).toEqual({ ...header, type: "Header" }); + expect(await collect(reader.readMessages())).toEqual([{ ...message, type: "Message" }]); + } finally { + await rm(dir, { recursive: true }); + } + }); +}); diff --git a/typescript/nodejs/src/index.ts b/typescript/nodejs/src/index.ts new file mode 100644 index 0000000000..cec1cee26c --- /dev/null +++ b/typescript/nodejs/src/index.ts @@ -0,0 +1,2 @@ +export * from "./FileHandleReadable"; +export * from "./FileHandleWritable"; diff --git a/typescript/nodejs/tsconfig.cjs.json b/typescript/nodejs/tsconfig.cjs.json new file mode 100644 index 0000000000..ac31cb3343 --- /dev/null +++ b/typescript/nodejs/tsconfig.cjs.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist/cjs", + "module": "commonjs" + } +} diff --git a/typescript/nodejs/tsconfig.json b/typescript/nodejs/tsconfig.json new file mode 100644 index 0000000000..ff2ce52dc6 --- /dev/null +++ b/typescript/nodejs/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "@foxglove/tsconfig/base", + "include": ["./src/**/*"], + "compilerOptions": { + "outDir": "./dist/esm", + "lib": ["es2020"], + "composite": true, + "incremental": true + }, + "references": [{ "path": "../core" }] +} diff --git a/typescript/support/.eslintrc.js b/typescript/support/.eslintrc.js new file mode 100644 index 0000000000..1b945176f6 --- /dev/null +++ b/typescript/support/.eslintrc.js @@ -0,0 +1,22 @@ +/* eslint-env node */ +module.exports = { + env: { es2020: true }, + ignorePatterns: ["dist", "nodejs.d.ts", "nodejs.js"], + extends: ["plugin:@foxglove/base", "plugin:@foxglove/jest"], + overrides: [ + { + files: ["*.ts", "*.tsx"], + extends: ["plugin:@foxglove/typescript"], + parserOptions: { + project: "../*/tsconfig.json", + tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, + }, + }, + ], + rules: { + "no-warning-comments": ["error", { terms: ["fixme"], location: "anywhere" }], + }, +}; diff --git a/typescript/support/LICENSE b/typescript/support/LICENSE new file mode 100644 index 0000000000..0aa593ea8c --- /dev/null +++ b/typescript/support/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Foxglove Technologies Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/typescript/support/README.md b/typescript/support/README.md new file mode 100644 index 0000000000..d22885fd8a --- /dev/null +++ b/typescript/support/README.md @@ -0,0 +1,68 @@ +# @mcap/support + +[MCAP](https://mcap.dev/) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. + +The `@mcap/support` package provides utilities for working with MCAP files that use [well-known compression formats](https://mcap.dev/specification/appendix.html), from Node.js and browsers. + +## Usage examples + +### Reading MCAP files in a browser + +```ts +import { loadDecompressHandlers } from "@mcap/support"; +import { BlobReadable } from "@mcap/browser"; +import { McapIndexedReader } from "@mcap/core"; +import { open } from "fs/promises"; + +async function onInputOrDrop(event: InputEvent | DragEvent) { + const file = event.dataTransfer.files[0]; + const decompressHandlers = await loadDecompressHandlers(); + const reader = await McapIndexedReader.Initialize({ + readable: new BlobReadable(file), + decompressHandlers, + }); +} +``` + +### Reading MCAP files in Node.js + +```ts +import { loadDecompressHandlers } from "@mcap/support"; +import { FileHandleReadable } from "@mcap/nodejs"; +import { McapIndexedReader } from "@mcap/core"; +import { open } from "fs/promises"; + +const decompressHandlers = await loadDecompressHandlers(); +const fileHandle = await open("file.mcap", "r"); +const reader = await McapIndexedReader.Initialize({ + readable: new FileHandleReadable(fileHandle), + decompressHandlers, +}); +``` + +### Writing MCAP files with Node.js + +```ts +import zstd from "@foxglove/wasm-zstd"; +import { FileHandleWritable } from "@mcap/support/nodejs"; +import { open } from "fs/promises"; + +const fileHandle = await open("file.mcap", "w"); + +await zstd.isLoaded; +const writer = new McapWriter({ + writable: new FileHandleWritable(fileHandle), + compressChunk: (data) => ({ + compression: "zstd", + compressedData: zstd.compress(data), + }), +}); +``` + +## License + +`@mcap/support` is licensed under the [MIT License](https://opensource.org/licenses/MIT). + +## Stay in touch + +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/typescript/support/jest.config.json b/typescript/support/jest.config.json new file mode 100644 index 0000000000..57e9583c8d --- /dev/null +++ b/typescript/support/jest.config.json @@ -0,0 +1,16 @@ +{ + "testMatch": ["/src/**/*.test.ts"], + "transform": { + "^.+\\.ts$": [ + "ts-jest", + { + "diagnostics": { + "//": "add 6133 (unused variables) to default ignore codes", + "ignoreCodes": [6059, 18002, 18003, 6133] + } + } + ] + }, + "//": "Native find is slow because it does not exclude files: https://github.com/facebook/jest/pull/11264#issuecomment-825377579", + "haste": { "forceNodeFilesystemAPI": true } +} diff --git a/typescript/support/package.json b/typescript/support/package.json new file mode 100644 index 0000000000..cdb91e4f6e --- /dev/null +++ b/typescript/support/package.json @@ -0,0 +1,60 @@ +{ + "name": "@mcap/support", + "version": "1.0.1", + "description": "Common decompression for use with MCAP files", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/foxglove/mcap.git" + }, + "author": { + "name": "Foxglove Technologies", + "email": "support@foxglove.dev" + }, + "homepage": "https://foxglove.dev/", + "module": "dist/esm/src/index.js", + "main": "dist/cjs/src/index.js", + "typings": "dist/esm/src/index.d.ts", + "typedoc": { + "entryPoint": "src/index.ts" + }, + "files": [ + "dist", + "src", + "nodejs.d.ts", + "nodejs.js" + ], + "scripts": { + "build": "tsc -b tsconfig.json tsconfig.cjs.json", + "prepack": "yarn build", + "lint:ci": "eslint --report-unused-disable-directives .", + "lint": "eslint --report-unused-disable-directives --fix .", + "test": "jest" + }, + "devDependencies": { + "@foxglove/eslint-plugin": "1.0.1", + "@foxglove/tsconfig": "1.1.0", + "@mcap/core": "workspace:*", + "@types/jest": "29.5.8", + "@types/node": "18.13.0", + "@typescript-eslint/eslint-plugin": "6.11.0", + "@typescript-eslint/parser": "6.11.0", + "eslint": "8.54.0", + "eslint-config-prettier": "9.0.0", + "eslint-plugin-es": "4.1.0", + "eslint-plugin-filenames": "1.3.2", + "eslint-plugin-import": "2.29.0", + "eslint-plugin-jest": "27.6.0", + "eslint-plugin-prettier": "5.0.1", + "jest": "29.7.0", + "prettier": "3.1.0", + "ts-jest": "29.1.1", + "typescript": "5.2.2" + }, + "dependencies": { + "@foxglove/wasm-bz2": "^0.1.1", + "@foxglove/wasm-lz4": "^1.0.2", + "@foxglove/wasm-zstd": "^1.0.1", + "protobufjs": "^7.2.5" + } +} diff --git a/typescript/support/src/decompressHandlers.ts b/typescript/support/src/decompressHandlers.ts new file mode 100644 index 0000000000..a71bab5112 --- /dev/null +++ b/typescript/support/src/decompressHandlers.ts @@ -0,0 +1,30 @@ +import type { McapTypes } from "@mcap/core"; + +let handlersPromise: Promise | undefined; +export async function loadDecompressHandlers(): Promise { + return await (handlersPromise ??= _loadDecompressHandlers()); +} + +// eslint-disable-next-line no-underscore-dangle +async function _loadDecompressHandlers(): Promise { + const [decompressZstd, decompressLZ4, bzip2] = await Promise.all([ + import("@foxglove/wasm-zstd").then(async (mod) => { + await mod.isLoaded; + return mod.decompress; + }), + import("@foxglove/wasm-lz4").then(async (mod) => { + await mod.default.isLoaded; + return mod.default; + }), + import("@foxglove/wasm-bz2").then(async (mod) => await mod.default.init()), + ]); + + return { + lz4: (buffer, decompressedSize) => decompressLZ4(buffer, Number(decompressedSize)), + + bz2: (buffer, decompressedSize) => + bzip2.decompress(buffer, Number(decompressedSize), { small: false }), + + zstd: (buffer, decompressedSize) => decompressZstd(buffer, Number(decompressedSize)), + }; +} diff --git a/typescript/support/src/index.ts b/typescript/support/src/index.ts new file mode 100644 index 0000000000..f469258d50 --- /dev/null +++ b/typescript/support/src/index.ts @@ -0,0 +1,2 @@ +export * from "./decompressHandlers"; +export * from "./protobufDescriptors"; diff --git a/typescript/support/src/protobufDescriptors.ts b/typescript/support/src/protobufDescriptors.ts new file mode 100644 index 0000000000..fb572a7a5b --- /dev/null +++ b/typescript/support/src/protobufDescriptors.ts @@ -0,0 +1,19 @@ +// eslint-disable-next-line @typescript-eslint/triple-slash-reference +/// + +import protobufjs from "protobufjs"; +import { FileDescriptorSet } from "protobufjs/ext/descriptor"; + +export type ProtobufDescriptor = ReturnType; + +export function protobufToDescriptor(root: protobufjs.Root): ProtobufDescriptor { + return root.toDescriptor("proto3"); +} + +export function protobufFromDescriptor(descriptorSet: protobufjs.Message): protobufjs.Root { + return protobufjs.Root.fromDescriptor(descriptorSet); +} + +export function protobufFromBinaryDescriptor(schemaData: Uint8Array): protobufjs.Root { + return protobufFromDescriptor(FileDescriptorSet.decode(schemaData)); +} diff --git a/typescript/support/tsconfig.cjs.json b/typescript/support/tsconfig.cjs.json new file mode 100644 index 0000000000..ac31cb3343 --- /dev/null +++ b/typescript/support/tsconfig.cjs.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist/cjs", + "module": "commonjs" + } +} diff --git a/typescript/support/tsconfig.json b/typescript/support/tsconfig.json new file mode 100644 index 0000000000..6a6b398a2e --- /dev/null +++ b/typescript/support/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "@foxglove/tsconfig/base", + "include": ["./src/**/*"], + "compilerOptions": { + "outDir": "./dist/esm", + "lib": ["es2020", "dom"], + "composite": true, + "incremental": true + }, + "references": [{ "path": "../core" }] +} diff --git a/typescript/examples/validate/typings/protobufjs.d.ts b/typescript/support/typings/protobufjs.d.ts similarity index 100% rename from typescript/examples/validate/typings/protobufjs.d.ts rename to typescript/support/typings/protobufjs.d.ts diff --git a/typescript/typedoc.json b/typescript/typedoc.json new file mode 100644 index 0000000000..461c973119 --- /dev/null +++ b/typescript/typedoc.json @@ -0,0 +1,5 @@ +{ + "name": "MCAP TypeScript SDK", + "entryPointStrategy": "packages", + "entryPoints": ["./core", "./support", "./nodejs", "./browser"] +} diff --git a/website/package.json b/website/package.json index d0c5f68edb..bdfb4fa619 100644 --- a/website/package.json +++ b/website/package.json @@ -20,7 +20,7 @@ "@docusaurus/core": "2.4.1", "@docusaurus/module-type-aliases": "2.4.1", "@docusaurus/preset-classic": "2.4.1", - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/rostime": "1.1.2", "@foxglove/schemas": "1.6.2", "@foxglove/tsconfig": "2.0.0", @@ -30,7 +30,7 @@ "@types/promise-queue": "2.2.0", "buffer": "6.0.3", "classnames": "2.3.2", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", @@ -44,7 +44,7 @@ "prism-react-renderer": "1.3.5", "process": "0.11.10", "promise-queue": "2.2.5", - "protobufjs": "7.2.3", + "protobufjs": "7.2.5", "react": "17.0.2", "react-async": "10.0.1", "react-dom": "17.0.2", diff --git a/yarn.lock b/yarn.lock index 9a963f8684..cd61613919 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2782,10 +2782,10 @@ __metadata: languageName: node linkType: hard -"@eslint/js@npm:8.53.0": - version: 8.53.0 - resolution: "@eslint/js@npm:8.53.0" - checksum: e0d5cfb0000aaee237c8e6d6d6e366faa60b1ef7f928ce17778373aa44d3b886368f6d5e1f97f913f0f16801aad016db8b8df78418c9d18825c15590328028af +"@eslint/js@npm:8.54.0": + version: 8.54.0 + resolution: "@eslint/js@npm:8.54.0" + checksum: 6d88a6f711ef0133566b5340e3178a178fbb297585766460f195d0a9db85688f1e5cf8559fd5748aeb3131e2096c66595b323d8edab22df015acda68f1ebde92 languageName: node linkType: hard @@ -2803,22 +2803,22 @@ __metadata: languageName: node linkType: hard -"@foxglove/eslint-plugin@npm:1.0.0": - version: 1.0.0 - resolution: "@foxglove/eslint-plugin@npm:1.0.0" +"@foxglove/eslint-plugin@npm:1.0.1": + version: 1.0.1 + resolution: "@foxglove/eslint-plugin@npm:1.0.1" dependencies: "@typescript-eslint/utils": ^6 tsutils: ^3 typescript: ^4 || ^5 peerDependencies: eslint: ^7 || ^8 - eslint-config-prettier: ^8 + eslint-config-prettier: ^8 || ^9 eslint-plugin-es: ^4 eslint-plugin-filenames: ^1 eslint-plugin-import: ^2 eslint-plugin-prettier: ^5 prettier: ^3 - checksum: 455819c6d2a0de3901fee957688433d631dd7ecd5ad691c838b853cd63a1e2e94362b0b065e83410d979deccabea5959b46d5de994006d80d287c3afc3e4aed0 + checksum: 53a802c11a373545932671151c1ab432fd3c9af9eef124a462ceff1bbc0c04605dfba72e7e078c38f56ff873baa8e135a24440ad1b69ef7d2fd78239795d9589 languageName: node linkType: hard @@ -2826,14 +2826,14 @@ __metadata: version: 0.0.0-use.local resolution: "@foxglove/mcap-benchmarks@workspace:typescript/benchmarks" dependencies: - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/tsconfig": 1.1.0 "@mcap/core": "*" "@types/node": 18.13.0 "@typescript-eslint/eslint-plugin": 6.11.0 "@typescript-eslint/parser": 6.11.0 benny: ^3.7.1 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -2851,9 +2851,10 @@ __metadata: resolution: "@foxglove/mcap-conformance@workspace:tests/conformance" dependencies: "@foxglove/crc": ^0.0.3 - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/tsconfig": 1.1.0 - "@mcap/core": "*" + "@mcap/core": "workspace:*" + "@mcap/support": "workspace:*" "@types/diff": ^5.0.2 "@types/js-yaml": ^4.0.5 "@types/node": 18.13.0 @@ -2862,7 +2863,7 @@ __metadata: colors: 1.4.0 commander: 11.1.0 diff: ^5.1.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -2885,7 +2886,7 @@ __metadata: version: 0.0.0-use.local resolution: "@foxglove/mcap-example-bag2mcap@workspace:typescript/examples/bag2mcap" dependencies: - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/rosbag": 0.2.3 "@foxglove/rosmsg": 3.1.0 "@foxglove/rosmsg-serialization": 1.5.3 @@ -2894,25 +2895,23 @@ __metadata: "@foxglove/wasm-bz2": 0.1.1 "@foxglove/wasm-lz4": 1.0.2 "@foxglove/wasm-zstd": 1.0.1 - "@mcap/core": "*" - "@types/jest": 29.5.8 + "@mcap/core": "workspace:*" + "@mcap/nodejs": "workspace:*" + "@mcap/support": "workspace:*" "@types/lodash": 4.14.191 "@types/node": 18.13.0 "@typescript-eslint/eslint-plugin": 6.11.0 "@typescript-eslint/parser": 6.11.0 commander: 11.1.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 eslint-plugin-import: 2.29.0 - eslint-plugin-jest: 27.6.0 eslint-plugin-prettier: 5.0.1 - jest: 29.7.0 lodash: 4.17.21 prettier: 3.1.0 - protobufjs: 7.2.2 - ts-jest: 29.1.1 + protobufjs: 7.2.5 ts-node: 10.9.1 tsconfig-paths: 4.1.2 typescript: 5.2.2 @@ -2923,10 +2922,12 @@ __metadata: version: 0.0.0-use.local resolution: "@foxglove/mcap-example-basicwriter@workspace:typescript/examples/basicwriter" dependencies: - "@foxglove/eslint-plugin": 1.0.0 - "@mcap/core": "*" + "@foxglove/eslint-plugin": 1.0.1 + "@mcap/core": "workspace:*" + "@mcap/nodejs": "workspace:*" + "@mcap/support": "workspace:*" "@types/node": 18.13.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -2942,11 +2943,12 @@ __metadata: version: 0.0.0-use.local resolution: "@foxglove/mcap-example-flatbufferswriter@workspace:typescript/examples/flatbufferswriter" dependencies: - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/schemas": ^1.0.0 - "@mcap/core": "*" + "@mcap/core": "workspace:*" + "@mcap/nodejs": "workspace:*" "@types/node": 18.13.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -2963,11 +2965,12 @@ __metadata: version: 0.0.0-use.local resolution: "@foxglove/mcap-example-text-annotation-demo@workspace:typescript/examples/text-annotation-demo" dependencies: - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/schemas": 1.3.0 - "@mcap/core": "*" + "@mcap/core": "workspace:*" + "@mcap/nodejs": "workspace:*" "@types/node": 18.13.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -2983,7 +2986,7 @@ __metadata: version: 0.0.0-use.local resolution: "@foxglove/mcap-example-validate@workspace:typescript/examples/validate" dependencies: - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/rosbag": 0.2.3 "@foxglove/rosmsg": 3.1.0 "@foxglove/rosmsg-serialization": 1.5.3 @@ -2992,14 +2995,16 @@ __metadata: "@foxglove/wasm-bz2": 0.1.1 "@foxglove/wasm-lz4": 1.0.2 "@foxglove/wasm-zstd": 1.0.1 - "@mcap/core": "*" + "@mcap/core": "workspace:*" + "@mcap/nodejs": "workspace:*" + "@mcap/support": "workspace:*" "@types/jest": 29.5.8 "@types/lodash": 4.14.191 "@types/node": 18.13.0 "@typescript-eslint/eslint-plugin": 6.11.0 "@typescript-eslint/parser": 6.11.0 commander: 11.1.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -3009,7 +3014,7 @@ __metadata: jest: 29.7.0 lodash: 4.17.21 prettier: 3.1.0 - protobufjs: 7.2.2 + protobufjs: 7.2.5 ts-jest: 29.1.1 ts-node: 10.9.1 tsconfig-paths: 4.1.2 @@ -3128,7 +3133,7 @@ __metadata: languageName: node linkType: hard -"@foxglove/wasm-bz2@npm:0.1.1": +"@foxglove/wasm-bz2@npm:0.1.1, @foxglove/wasm-bz2@npm:^0.1.1": version: 0.1.1 resolution: "@foxglove/wasm-bz2@npm:0.1.1" dependencies: @@ -3137,14 +3142,14 @@ __metadata: languageName: node linkType: hard -"@foxglove/wasm-lz4@npm:1.0.2": +"@foxglove/wasm-lz4@npm:1.0.2, @foxglove/wasm-lz4@npm:^1.0.2": version: 1.0.2 resolution: "@foxglove/wasm-lz4@npm:1.0.2" checksum: 1148ae84be22830382995ef19912dd033f009c0f16690dd942acce2efee02035b035f5910690c7e37c9435e7636b94006ef33220f86e98bc281129e50445bb19 languageName: node linkType: hard -"@foxglove/wasm-zstd@npm:1.0.1": +"@foxglove/wasm-zstd@npm:1.0.1, @foxglove/wasm-zstd@npm:^1.0.1": version: 1.0.1 resolution: "@foxglove/wasm-zstd@npm:1.0.1" checksum: 91fffc0970e4dad0550e054ae1647730e09876ef0d6fb15c70d800e9027e38d8372e990dde98542a95eb00d88d483c9e1227be0ea958f9d3f41693dc78311027 @@ -3528,19 +3533,44 @@ __metadata: languageName: node linkType: hard +"@mcap/browser@workspace:typescript/browser": + version: 0.0.0-use.local + resolution: "@mcap/browser@workspace:typescript/browser" + dependencies: + "@foxglove/eslint-plugin": 1.0.1 + "@foxglove/tsconfig": 1.1.0 + "@mcap/core": "workspace:*" + "@types/jest": 29.5.8 + "@typescript-eslint/eslint-plugin": 6.11.0 + "@typescript-eslint/parser": 6.11.0 + eslint: 8.54.0 + eslint-config-prettier: 9.0.0 + eslint-import-resolver-typescript: 3.6.1 + eslint-plugin-es: 4.1.0 + eslint-plugin-filenames: 1.3.2 + eslint-plugin-import: 2.29.0 + eslint-plugin-jest: 27.6.0 + eslint-plugin-prettier: 5.0.1 + jest: 29.7.0 + prettier: 3.1.0 + ts-jest: 29.1.1 + typescript: 5.2.2 + languageName: unknown + linkType: soft + "@mcap/core@*, @mcap/core@workspace:*, @mcap/core@workspace:typescript/core": version: 0.0.0-use.local resolution: "@mcap/core@workspace:typescript/core" dependencies: "@foxglove/crc": ^0.0.3 - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/tsconfig": 1.1.0 "@types/jest": 29.5.8 "@types/lodash": 4.14.191 "@types/node": 18.13.0 "@typescript-eslint/eslint-plugin": 6.11.0 "@typescript-eslint/parser": 6.11.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -3558,6 +3588,61 @@ __metadata: languageName: unknown linkType: soft +"@mcap/nodejs@workspace:*, @mcap/nodejs@workspace:typescript/nodejs": + version: 0.0.0-use.local + resolution: "@mcap/nodejs@workspace:typescript/nodejs" + dependencies: + "@foxglove/eslint-plugin": 1.0.1 + "@foxglove/tsconfig": 1.1.0 + "@mcap/core": "workspace:*" + "@types/jest": 29.5.8 + "@types/node": 18.13.0 + "@typescript-eslint/eslint-plugin": 6.11.0 + "@typescript-eslint/parser": 6.11.0 + eslint: 8.54.0 + eslint-config-prettier: 9.0.0 + eslint-import-resolver-typescript: 3.6.1 + eslint-plugin-es: 4.1.0 + eslint-plugin-filenames: 1.3.2 + eslint-plugin-import: 2.29.0 + eslint-plugin-jest: 27.6.0 + eslint-plugin-prettier: 5.0.1 + jest: 29.7.0 + prettier: 3.1.0 + ts-jest: 29.1.1 + typescript: 5.2.2 + languageName: unknown + linkType: soft + +"@mcap/support@workspace:*, @mcap/support@workspace:typescript/support": + version: 0.0.0-use.local + resolution: "@mcap/support@workspace:typescript/support" + dependencies: + "@foxglove/eslint-plugin": 1.0.1 + "@foxglove/tsconfig": 1.1.0 + "@foxglove/wasm-bz2": ^0.1.1 + "@foxglove/wasm-lz4": ^1.0.2 + "@foxglove/wasm-zstd": ^1.0.1 + "@mcap/core": "workspace:*" + "@types/jest": 29.5.8 + "@types/node": 18.13.0 + "@typescript-eslint/eslint-plugin": 6.11.0 + "@typescript-eslint/parser": 6.11.0 + eslint: 8.54.0 + eslint-config-prettier: 9.0.0 + eslint-plugin-es: 4.1.0 + eslint-plugin-filenames: 1.3.2 + eslint-plugin-import: 2.29.0 + eslint-plugin-jest: 27.6.0 + eslint-plugin-prettier: 5.0.1 + jest: 29.7.0 + prettier: 3.1.0 + protobufjs: ^7.2.5 + ts-jest: 29.1.1 + typescript: 5.2.2 + languageName: unknown + linkType: soft + "@mdx-js/mdx@npm:^1.6.22": version: 1.6.22 resolution: "@mdx-js/mdx@npm:1.6.22" @@ -4535,13 +4620,13 @@ __metadata: languageName: node linkType: hard -"@typescript-eslint/scope-manager@npm:5.59.7": - version: 5.59.7 - resolution: "@typescript-eslint/scope-manager@npm:5.59.7" +"@typescript-eslint/scope-manager@npm:5.62.0": + version: 5.62.0 + resolution: "@typescript-eslint/scope-manager@npm:5.62.0" dependencies: - "@typescript-eslint/types": 5.59.7 - "@typescript-eslint/visitor-keys": 5.59.7 - checksum: 43f7ea93fddbe2902122a41050677fe3eff2ea468f435b981592510cfc6136e8c28ac7d3a3e05fb332c0b3078a29bd0c91c35b2b1f4e788b4eb9aaeb70e21583 + "@typescript-eslint/types": 5.62.0 + "@typescript-eslint/visitor-keys": 5.62.0 + checksum: 6062d6b797fe1ce4d275bb0d17204c827494af59b5eaf09d8a78cdd39dadddb31074dded4297aaf5d0f839016d601032857698b0e4516c86a41207de606e9573 languageName: node linkType: hard @@ -4572,10 +4657,10 @@ __metadata: languageName: node linkType: hard -"@typescript-eslint/types@npm:5.59.7": - version: 5.59.7 - resolution: "@typescript-eslint/types@npm:5.59.7" - checksum: 52eccec9e2d631eb2808e48b5dc33a837b5e242fa9eddace89fc707c9f2283b5364f1d38b33d418a08d64f45f6c22f051800898e1881a912f8aac0c3ae300d0a +"@typescript-eslint/types@npm:5.62.0": + version: 5.62.0 + resolution: "@typescript-eslint/types@npm:5.62.0" + checksum: 48c87117383d1864766486f24de34086155532b070f6264e09d0e6139449270f8a9559cfef3c56d16e3bcfb52d83d42105d61b36743626399c7c2b5e0ac3b670 languageName: node linkType: hard @@ -4586,12 +4671,12 @@ __metadata: languageName: node linkType: hard -"@typescript-eslint/typescript-estree@npm:5.59.7": - version: 5.59.7 - resolution: "@typescript-eslint/typescript-estree@npm:5.59.7" +"@typescript-eslint/typescript-estree@npm:5.62.0": + version: 5.62.0 + resolution: "@typescript-eslint/typescript-estree@npm:5.62.0" dependencies: - "@typescript-eslint/types": 5.59.7 - "@typescript-eslint/visitor-keys": 5.59.7 + "@typescript-eslint/types": 5.62.0 + "@typescript-eslint/visitor-keys": 5.62.0 debug: ^4.3.4 globby: ^11.1.0 is-glob: ^4.0.3 @@ -4600,7 +4685,7 @@ __metadata: peerDependenciesMeta: typescript: optional: true - checksum: eefe82eedf9ee2e14463c3f2b5b18df084c1328a859b245ee897a9a7075acce7cca0216a21fd7968b75aa64189daa008bfde1e2f9afbcc336f3dfe856e7f342e + checksum: 3624520abb5807ed8f57b1197e61c7b1ed770c56dfcaca66372d584ff50175225798bccb701f7ef129d62c5989070e1ee3a0aa2d84e56d9524dcf011a2bb1a52 languageName: node linkType: hard @@ -4640,30 +4725,30 @@ __metadata: linkType: hard "@typescript-eslint/utils@npm:^5.10.0": - version: 5.59.7 - resolution: "@typescript-eslint/utils@npm:5.59.7" + version: 5.62.0 + resolution: "@typescript-eslint/utils@npm:5.62.0" dependencies: "@eslint-community/eslint-utils": ^4.2.0 "@types/json-schema": ^7.0.9 "@types/semver": ^7.3.12 - "@typescript-eslint/scope-manager": 5.59.7 - "@typescript-eslint/types": 5.59.7 - "@typescript-eslint/typescript-estree": 5.59.7 + "@typescript-eslint/scope-manager": 5.62.0 + "@typescript-eslint/types": 5.62.0 + "@typescript-eslint/typescript-estree": 5.62.0 eslint-scope: ^5.1.1 semver: ^7.3.7 peerDependencies: eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 - checksum: d8682700187ca94cc6441480cb6b87d0514a9748103c15dd93206c5b1c6fefa59063662f27a4103e16abbcfb654a61d479bc55af8f23d96f342431b87f31bb4e + checksum: ee9398c8c5db6d1da09463ca7bf36ed134361e20131ea354b2da16a5fdb6df9ba70c62a388d19f6eebb421af1786dbbd79ba95ddd6ab287324fc171c3e28d931 languageName: node linkType: hard -"@typescript-eslint/visitor-keys@npm:5.59.7": - version: 5.59.7 - resolution: "@typescript-eslint/visitor-keys@npm:5.59.7" +"@typescript-eslint/visitor-keys@npm:5.62.0": + version: 5.62.0 + resolution: "@typescript-eslint/visitor-keys@npm:5.62.0" dependencies: - "@typescript-eslint/types": 5.59.7 + "@typescript-eslint/types": 5.62.0 eslint-visitor-keys: ^3.3.0 - checksum: 4367f2ea68dd96a0520485434ad11e1bd26239eeeb3a2150bee7478a0f1df3c2099a39f96486722932be0456bcb7a47a483b452876d1d30bdeb9b81d354eef3d + checksum: 976b05d103fe8335bef5c93ad3f76d781e3ce50329c0243ee0f00c0fcfb186c81df50e64bfdd34970148113f8ade90887f53e3c4938183afba830b4ba8e30a35 languageName: node linkType: hard @@ -7413,13 +7498,13 @@ __metadata: languageName: node linkType: hard -"enhanced-resolve@npm:^5.14.0": - version: 5.14.0 - resolution: "enhanced-resolve@npm:5.14.0" +"enhanced-resolve@npm:^5.12.0, enhanced-resolve@npm:^5.14.0": + version: 5.15.0 + resolution: "enhanced-resolve@npm:5.15.0" dependencies: graceful-fs: ^4.2.4 tapable: ^2.2.0 - checksum: fff1aaebbf376371e5df4502e111967f6247c37611ad3550e4e7fca657f6dcb29ef7ffe88bf14e5010b78997f1ddd984a8db97af87ee0a5477771398fd326f5b + checksum: fbd8cdc9263be71cc737aa8a7d6c57b43d6aa38f6cc75dde6fcd3598a130cc465f979d2f4d01bb3bf475acb43817749c79f8eef9be048683602ca91ab52e4f11 languageName: node linkType: hard @@ -7631,7 +7716,25 @@ __metadata: languageName: node linkType: hard -"eslint-module-utils@npm:^2.8.0": +"eslint-import-resolver-typescript@npm:3.6.1": + version: 3.6.1 + resolution: "eslint-import-resolver-typescript@npm:3.6.1" + dependencies: + debug: ^4.3.4 + enhanced-resolve: ^5.12.0 + eslint-module-utils: ^2.7.4 + fast-glob: ^3.3.1 + get-tsconfig: ^4.5.0 + is-core-module: ^2.11.0 + is-glob: ^4.0.3 + peerDependencies: + eslint: "*" + eslint-plugin-import: "*" + checksum: 454fa0646533050fb57f13d27daf8c71f51b0bb9156d6a461290ccb8576d892209fcc6702a89553f3f5ea8e5b407395ca2e5de169a952c953685f1f7c46b4496 + languageName: node + linkType: hard + +"eslint-module-utils@npm:^2.7.4, eslint-module-utils@npm:^2.8.0": version: 2.8.0 resolution: "eslint-module-utils@npm:2.8.0" dependencies: @@ -7811,14 +7914,14 @@ __metadata: languageName: node linkType: hard -"eslint@npm:8.53.0": - version: 8.53.0 - resolution: "eslint@npm:8.53.0" +"eslint@npm:8.54.0": + version: 8.54.0 + resolution: "eslint@npm:8.54.0" dependencies: "@eslint-community/eslint-utils": ^4.2.0 "@eslint-community/regexpp": ^4.6.1 "@eslint/eslintrc": ^2.1.3 - "@eslint/js": 8.53.0 + "@eslint/js": 8.54.0 "@humanwhocodes/config-array": ^0.11.13 "@humanwhocodes/module-importer": ^1.0.1 "@nodelib/fs.walk": ^1.2.8 @@ -7855,7 +7958,7 @@ __metadata: text-table: ^0.2.0 bin: eslint: bin/eslint.js - checksum: 2da808655c7aa4b33f8970ba30d96b453c3071cc4d6cd60d367163430677e32ff186b65270816b662d29139283138bff81f28dddeb2e73265495245a316ed02c + checksum: 7e876e9da2a18a017271cf3733d05a3dfbbe469272d75753408c6ea5b1646c71c6bb18cb91e10ca930144c32c1ce3701e222f1ae6784a3975a69f8f8aa68e49f languageName: node linkType: hard @@ -8101,7 +8204,7 @@ __metadata: languageName: node linkType: hard -"fast-glob@npm:^3.2.11, fast-glob@npm:^3.2.9, fast-glob@npm:^3.3.0, fast-glob@npm:^3.3.2": +"fast-glob@npm:^3.2.11, fast-glob@npm:^3.2.9, fast-glob@npm:^3.3.0, fast-glob@npm:^3.3.1, fast-glob@npm:^3.3.2": version: 3.3.2 resolution: "fast-glob@npm:3.3.2" dependencies: @@ -8327,9 +8430,9 @@ __metadata: linkType: hard "flatbuffers@npm:^23.1.21": - version: 23.1.21 - resolution: "flatbuffers@npm:23.1.21" - checksum: 6e890adb8300e5352d579abf7c674eec7d838c27e82981c3d9cd5a4043ff83fc7097a88eae68729eb7cff6ded2b1a896ee2009632027d613040bef669290fbc6 + version: 23.5.26 + resolution: "flatbuffers@npm:23.5.26" + checksum: 2528e549118d02fca9775cc9c17cf445741bb58b0a9575eb07bba823394a79dccb3bf97c48561720aa61d3dc9f42759129df0e05eb94c12f809cfb6126eaedd1 languageName: node linkType: hard @@ -8619,6 +8722,15 @@ __metadata: languageName: node linkType: hard +"get-tsconfig@npm:^4.5.0": + version: 4.7.2 + resolution: "get-tsconfig@npm:4.7.2" + dependencies: + resolve-pkg-maps: ^1.0.0 + checksum: 172358903250eff0103943f816e8a4e51d29b8e5449058bdf7266714a908a48239f6884308bd3a6ff28b09f692b9533dbebfd183ab63e4e14f073cda91f1bca9 + languageName: node + linkType: hard + "github-slugger@npm:^1.4.0": version: 1.5.0 resolution: "github-slugger@npm:1.5.0" @@ -8747,15 +8859,15 @@ __metadata: linkType: hard "globby@npm:^13.1.1": - version: 13.1.3 - resolution: "globby@npm:13.1.3" + version: 13.2.2 + resolution: "globby@npm:13.2.2" dependencies: dir-glob: ^3.0.1 - fast-glob: ^3.2.11 - ignore: ^5.2.0 + fast-glob: ^3.3.0 + ignore: ^5.2.4 merge2: ^1.4.1 slash: ^4.0.0 - checksum: 93f06e02002cdf368f7e3d55bd59e7b00784c7cc8fe92c7ee5082cc7171ff6109fda45e1c97a80bb48bc811dedaf7843c7c9186f5f84bde4883ab630e13c43df + checksum: f3d84ced58a901b4fcc29c846983108c426631fe47e94872868b65565495f7bee7b3defd68923bd480582771fd4bbe819217803a164a618ad76f1d22f666f41e languageName: node linkType: hard @@ -9575,7 +9687,7 @@ __metadata: languageName: node linkType: hard -"is-core-module@npm:^2.13.0, is-core-module@npm:^2.13.1, is-core-module@npm:^2.9.0": +"is-core-module@npm:^2.11.0, is-core-module@npm:^2.13.0, is-core-module@npm:^2.13.1, is-core-module@npm:^2.9.0": version: 2.13.1 resolution: "is-core-module@npm:2.13.1" dependencies: @@ -12776,29 +12888,9 @@ __metadata: languageName: node linkType: hard -"protobufjs@npm:7.2.2": - version: 7.2.2 - resolution: "protobufjs@npm:7.2.2" - dependencies: - "@protobufjs/aspromise": ^1.1.2 - "@protobufjs/base64": ^1.1.2 - "@protobufjs/codegen": ^2.0.4 - "@protobufjs/eventemitter": ^1.1.0 - "@protobufjs/fetch": ^1.1.0 - "@protobufjs/float": ^1.0.2 - "@protobufjs/inquire": ^1.1.0 - "@protobufjs/path": ^1.1.2 - "@protobufjs/pool": ^1.1.0 - "@protobufjs/utf8": ^1.1.0 - "@types/node": ">=13.7.0" - long: ^5.0.0 - checksum: 86166e8f3e46789fa4d117ae72c17356db36bf87c0e0710d224be32e543b1c378a94e66dc2b1de5af45edfc25f56acfc7e688c50c956426a3ae97bc474f4445c - languageName: node - linkType: hard - -"protobufjs@npm:7.2.3": - version: 7.2.3 - resolution: "protobufjs@npm:7.2.3" +"protobufjs@npm:7.2.5, protobufjs@npm:^7.2.5": + version: 7.2.5 + resolution: "protobufjs@npm:7.2.5" dependencies: "@protobufjs/aspromise": ^1.1.2 "@protobufjs/base64": ^1.1.2 @@ -12812,7 +12904,7 @@ __metadata: "@protobufjs/utf8": ^1.1.0 "@types/node": ">=13.7.0" long: ^5.0.0 - checksum: 9afa6de5fced0139a5180c063718508fac3ea734a9f1aceb99712367b15473a83327f91193f16b63540f9112b09a40912f5f0441a9b0d3f3c6a1c7f707d78249 + checksum: 3770a072114061faebbb17cfd135bc4e187b66bc6f40cd8bac624368b0270871ec0cfb43a02b9fb4f029c8335808a840f1afba3c2e7ede7063b98ae6b98a703f languageName: node linkType: hard @@ -13481,6 +13573,13 @@ __metadata: languageName: node linkType: hard +"resolve-pkg-maps@npm:^1.0.0": + version: 1.0.0 + resolution: "resolve-pkg-maps@npm:1.0.0" + checksum: 1012afc566b3fdb190a6309cc37ef3b2dcc35dff5fa6683a9d00cd25c3247edfbc4691b91078c97adc82a29b77a2660c30d791d65dab4fc78bfc473f60289977 + languageName: node + linkType: hard + "resolve.exports@npm:^2.0.0": version: 2.0.0 resolution: "resolve.exports@npm:2.0.0" @@ -13595,8 +13694,12 @@ __metadata: version: 0.0.0-use.local resolution: "root-workspace-0b6124@workspace:." dependencies: + "@types/node": 18.13.0 cspell: 8.0.0 + jest: 29.7.0 prettier: 3.1.0 + ts-jest: 29.1.1 + ts-node: 10.9.1 typedoc: 0.25.3 typescript: 5.2.2 languageName: unknown @@ -15662,7 +15765,7 @@ __metadata: "@docusaurus/core": 2.4.1 "@docusaurus/module-type-aliases": 2.4.1 "@docusaurus/preset-classic": 2.4.1 - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/rostime": 1.1.2 "@foxglove/schemas": 1.6.2 "@foxglove/tsconfig": 2.0.0 @@ -15672,7 +15775,7 @@ __metadata: "@types/promise-queue": 2.2.0 buffer: 6.0.3 classnames: 2.3.2 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -15686,7 +15789,7 @@ __metadata: prism-react-renderer: 1.3.5 process: 0.11.10 promise-queue: 2.2.5 - protobufjs: 7.2.3 + protobufjs: 7.2.5 react: 17.0.2 react-async: 10.0.1 react-dom: 17.0.2