From 38e9c9d2a513ae5f2da25d0414121b6ce9762b8d Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Tue, 28 Feb 2023 15:12:37 -0800 Subject: [PATCH 01/27] [TypeScript] @mcap/support (#838) --- .gitattributes | 1 + .github/workflows/ci.yml | 17 +- RELEASE.md | 11 +- cspell.config.yaml | 2 + package.json | 1 + typescript/core/.yarnrc | 2 +- typescript/examples/bag2mcap/package.json | 1 + .../examples/bag2mcap/scripts/bag2mcap.ts | 29 +- typescript/examples/bag2mcap/tsconfig.json | 3 +- typescript/examples/basicwriter/package.json | 1 + .../examples/basicwriter/scripts/main.ts | 24 +- typescript/examples/basicwriter/tsconfig.json | 3 +- .../examples/flatbufferswriter/package.json | 1 + .../flatbufferswriter/scripts/main.ts | 23 +- .../examples/flatbufferswriter/tsconfig.json | 3 +- typescript/examples/validate/package.json | 1 + .../examples/validate/scripts/validate.ts | 16 +- typescript/examples/validate/tsconfig.json | 3 +- .../examples/validate/typings/protobufjs.d.ts | 14 - typescript/support/.eslintrc.js | 19 + typescript/support/.yarnrc | 2 + typescript/support/LICENSE | 21 + typescript/support/jest.config.json | 16 + typescript/support/nodejs.d.ts | 1 + typescript/support/nodejs.js | 1 + typescript/support/package.json | 77 +++ typescript/support/src/decompressHandlers.ts | 32 + .../support/src/fixtures/ByteVector.bfbs | 3 + .../support/src/fixtures/byte-vector.ts | 95 +++ .../support/src/fixtures/reflection.bfbs | 3 + typescript/support/src/index.ts | 5 + .../support/src/nodejs/FileHandleWritable.ts | 27 + typescript/support/src/nodejs/index.ts | 1 + typescript/support/src/parseChannel.test.ts | 104 +++ typescript/support/src/parseChannel.ts | 184 +++++ .../support/src/parseFlatbufferSchema.test.ts | 384 +++++++++++ .../support/src/parseFlatbufferSchema.ts | 199 ++++++ .../support/src/parseJsonSchema.test.ts | 275 ++++++++ typescript/support/src/parseJsonSchema.ts | 182 +++++ .../src/protobufDefinitionsToDatatypes.ts | 81 +++ typescript/support/src/protobufDescriptors.ts | 19 + typescript/support/src/types.ts | 8 + typescript/support/tsconfig.cjs.json | 7 + typescript/support/tsconfig.json | 9 + .../typings/protobufjs.d.ts | 0 yarn.lock | 627 +++++++++--------- 46 files changed, 2128 insertions(+), 410 deletions(-) delete mode 100644 typescript/examples/validate/typings/protobufjs.d.ts create mode 100644 typescript/support/.eslintrc.js create mode 100644 typescript/support/.yarnrc create mode 100644 typescript/support/LICENSE create mode 100644 typescript/support/jest.config.json create mode 100644 typescript/support/nodejs.d.ts create mode 100644 typescript/support/nodejs.js create mode 100644 typescript/support/package.json create mode 100644 typescript/support/src/decompressHandlers.ts create mode 100644 typescript/support/src/fixtures/ByteVector.bfbs create mode 100644 typescript/support/src/fixtures/byte-vector.ts create mode 100644 typescript/support/src/fixtures/reflection.bfbs create mode 100644 typescript/support/src/index.ts create mode 100644 typescript/support/src/nodejs/FileHandleWritable.ts create mode 100644 typescript/support/src/nodejs/index.ts create mode 100644 typescript/support/src/parseChannel.test.ts create mode 100644 typescript/support/src/parseChannel.ts create mode 100644 typescript/support/src/parseFlatbufferSchema.test.ts create mode 100644 typescript/support/src/parseFlatbufferSchema.ts create mode 100644 typescript/support/src/parseJsonSchema.test.ts create mode 100644 typescript/support/src/parseJsonSchema.ts create mode 100644 typescript/support/src/protobufDefinitionsToDatatypes.ts create mode 100644 typescript/support/src/protobufDescriptors.ts create mode 100644 typescript/support/src/types.ts create mode 100644 typescript/support/tsconfig.cjs.json create mode 100644 typescript/support/tsconfig.json rename typescript/{examples/bag2mcap => support}/typings/protobufjs.d.ts (100%) diff --git a/.gitattributes b/.gitattributes index 9a4b58178b..e978c9990b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,6 @@ *.png filter=lfs diff=lfs merge=lfs -text *.mcap filter=lfs diff=lfs merge=lfs -text +*.bfbs filter=lfs diff=lfs merge=lfs -text testdata/mcap/demo.mcap filter=lfs diff=lfs merge=lfs -text testdata/ filter=lfs diff=lfs merge=lfs -text tests/conformance/data/** linguist-generated=true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7425afff02..3cc3a29288 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,7 +177,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - lfs: "true" + lfs: true - uses: actions/cache@v3 with: path: ~/.conan/data @@ -198,7 +198,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - lfs: "true" + lfs: true - uses: actions/cache@v3 with: path: ~/.conan/data @@ -214,6 +214,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + with: + lfs: true - uses: actions/setup-node@v3 with: node-version: 16.x @@ -223,13 +225,22 @@ jobs: - run: yarn workspace @mcap/core lint:ci - run: yarn workspace @mcap/core typecheck - run: yarn workspace @mcap/core test + - run: yarn workspace @mcap/support lint:ci + - run: yarn workspace @mcap/support typecheck + - run: yarn workspace @mcap/support test - - name: Publish to NPM + - name: Publish @mcap/core to NPM if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/core/v') }} run: yarn workspace @mcap/core publish --access public env: NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + - name: Publish @mcap/support to NPM + if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/support/v') }} + run: yarn workspace @mcap/support publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + typescript-examples: runs-on: ubuntu-latest steps: diff --git a/RELEASE.md b/RELEASE.md index f3389ea67f..4b0a91d425 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -2,7 +2,7 @@ How to make releases. -## NPM +## @mcap/core - Checkout the version of the code you want to release - Update package.json in typescript/core/package.json with the new version. @@ -10,3 +10,12 @@ How to make releases. - Wait for the PR to pass CI and merge - Checkout main and tag the merged commit with `releases/typescript/core/v#.#.#` (replace #.#.# with the version you used in package.json) - Push the new tag to the repo with `git push origin releases/typescript/core/v#.#.#` + +## @mcap/support + +- Checkout the version of the code you want to release +- Update package.json in typescript/support/package.json with the new version. +- Make a PR with your changes to package.json +- Wait for the PR to pass CI and merge +- Checkout main and tag the merged commit with `releases/typescript/support/v#.#.#` (replace #.#.# with the version you used in package.json) +- Push the new tag to the repo with `git push origin releases/typescript/support/v#.#.#` diff --git a/cspell.config.yaml b/cspell.config.yaml index 64410e20de..4e72909309 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -14,6 +14,7 @@ ignorePaths: - bin - dist - yarn-error.log + - "*.bfbs" - "*.csv" - "*.Dockerfile" - Dockerfile @@ -79,6 +80,7 @@ words: - rostime - schemaless - serde + - sfixed - stoull - struct - swiftformat diff --git a/package.json b/package.json index 5324b7d7a3..00832c33cb 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "typescript/core", "typescript/benchmarks", "typescript/examples/*", + "typescript/support", "tests/conformance" ] }, diff --git a/typescript/core/.yarnrc b/typescript/core/.yarnrc index 27e40fb0db..9e445bed7c 100644 --- a/typescript/core/.yarnrc +++ b/typescript/core/.yarnrc @@ -1,2 +1,2 @@ version-tag-prefix "releases/typescript/core/v" -version-git-message "TypeScript v%s" +version-git-message "TypeScript @mcap/core v%s" diff --git a/typescript/examples/bag2mcap/package.json b/typescript/examples/bag2mcap/package.json index ad39a965aa..7e8fbb8148 100644 --- a/typescript/examples/bag2mcap/package.json +++ b/typescript/examples/bag2mcap/package.json @@ -30,6 +30,7 @@ "@foxglove/wasm-lz4": "1.0.2", "@foxglove/wasm-zstd": "1.0.1", "@mcap/core": "*", + "@mcap/support": "*", "@types/jest": "29.4.0", "@types/lodash": "4.14.191", "@types/node": "18.13.0", diff --git a/typescript/examples/bag2mcap/scripts/bag2mcap.ts b/typescript/examples/bag2mcap/scripts/bag2mcap.ts index 656f0d2efe..3d911e1063 100644 --- a/typescript/examples/bag2mcap/scripts/bag2mcap.ts +++ b/typescript/examples/bag2mcap/scripts/bag2mcap.ts @@ -12,9 +12,11 @@ import { toNanoSec } from "@foxglove/rostime"; import Bzip2 from "@foxglove/wasm-bz2"; import decompressLZ4 from "@foxglove/wasm-lz4"; import zstd from "@foxglove/wasm-zstd"; -import { McapWriter, IWritable, McapTypes } from "@mcap/core"; +import { McapWriter, McapTypes } from "@mcap/core"; +import { ProtobufDescriptor, protobufToDescriptor } from "@mcap/support"; +import { FileHandleWritable } from "@mcap/support/nodejs"; import { program } from "commander"; -import { open, FileHandle } from "fs/promises"; +import { open } from "fs/promises"; import protobufjs from "protobufjs"; import descriptor from "protobufjs/ext/descriptor"; @@ -55,7 +57,7 @@ function rosMsgDefinitionToProto( msgDef: string, ): { rootType: protobufjs.Type; - descriptorSet: ReturnType; + descriptorSet: ProtobufDescriptor; schemaName: string; } { const definitionArr = parseMessageDefinition(msgDef); @@ -154,7 +156,7 @@ function rosMsgDefinitionToProto( const rootType = root.lookupType(schemaName); // create a descriptor message for the root - const descriptorSet = root.toDescriptor("proto3"); + const descriptorSet = protobufToDescriptor(root); for (const file of descriptorSet.file) { // Strip leading `.` from the package names to make them relative to the descriptor file.package = file.package?.substring(1); @@ -194,25 +196,6 @@ function convertTypedArrays(msg: Record): Record { - const written = await this.handle.write(buffer); - this.totalBytesWritten += written.bytesWritten; - } - - position(): bigint { - return BigInt(this.totalBytesWritten); - } -} - async function convert(filePath: string, options: { indexed: boolean }) { await decompressLZ4.isLoaded; await zstd.isLoaded; diff --git a/typescript/examples/bag2mcap/tsconfig.json b/typescript/examples/bag2mcap/tsconfig.json index 13b7e779c7..5d97e53644 100644 --- a/typescript/examples/bag2mcap/tsconfig.json +++ b/typescript/examples/bag2mcap/tsconfig.json @@ -7,7 +7,8 @@ "outDir": "./dist/esm", "lib": ["es2020", "dom"], "paths": { - "@mcap/core": ["../../core/src"] + "@mcap/core": ["../../core/src"], + "@mcap/support": ["../../support/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/examples/basicwriter/package.json b/typescript/examples/basicwriter/package.json index 68a769afe7..e6c007f434 100644 --- a/typescript/examples/basicwriter/package.json +++ b/typescript/examples/basicwriter/package.json @@ -22,6 +22,7 @@ "devDependencies": { "@foxglove/eslint-plugin": "0.21.0", "@mcap/core": "*", + "@mcap/support": "*", "@types/node": "18.13.0", "eslint": "8.34.0", "eslint-config-prettier": "8.6.0", diff --git a/typescript/examples/basicwriter/scripts/main.ts b/typescript/examples/basicwriter/scripts/main.ts index a1a4db6144..94aa90d910 100644 --- a/typescript/examples/basicwriter/scripts/main.ts +++ b/typescript/examples/basicwriter/scripts/main.ts @@ -1,24 +1,6 @@ -import { McapWriter, IWritable } from "@mcap/core"; -import { open, FileHandle } from "fs/promises"; - -// Mcap IWritable interface for nodejs FileHandle -class FileHandleWritable implements IWritable { - private handle: FileHandle; - private totalBytesWritten = 0; - - constructor(handle: FileHandle) { - this.handle = handle; - } - - async write(buffer: Uint8Array): Promise { - const written = await this.handle.write(buffer); - this.totalBytesWritten += written.bytesWritten; - } - - position(): bigint { - return BigInt(this.totalBytesWritten); - } -} +import { McapWriter } from "@mcap/core"; +import { open } from "fs/promises"; +import { FileHandleWritable } from "@mcap/support/nodejs"; async function main() { const mcapFilePath = "output.mcap"; diff --git a/typescript/examples/basicwriter/tsconfig.json b/typescript/examples/basicwriter/tsconfig.json index 13b7e779c7..5d97e53644 100644 --- a/typescript/examples/basicwriter/tsconfig.json +++ b/typescript/examples/basicwriter/tsconfig.json @@ -7,7 +7,8 @@ "outDir": "./dist/esm", "lib": ["es2020", "dom"], "paths": { - "@mcap/core": ["../../core/src"] + "@mcap/core": ["../../core/src"], + "@mcap/support": ["../../support/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/examples/flatbufferswriter/package.json b/typescript/examples/flatbufferswriter/package.json index 1ddf8413e2..1082e76c6c 100644 --- a/typescript/examples/flatbufferswriter/package.json +++ b/typescript/examples/flatbufferswriter/package.json @@ -23,6 +23,7 @@ "@foxglove/eslint-plugin": "0.21.0", "@foxglove/schemas": "^1.0.0", "@mcap/core": "*", + "@mcap/support": "*", "@types/node": "18.13.0", "eslint": "8.34.0", "eslint-config-prettier": "8.6.0", diff --git a/typescript/examples/flatbufferswriter/scripts/main.ts b/typescript/examples/flatbufferswriter/scripts/main.ts index 647c8374c6..3ac09ef8fc 100644 --- a/typescript/examples/flatbufferswriter/scripts/main.ts +++ b/typescript/examples/flatbufferswriter/scripts/main.ts @@ -1,31 +1,14 @@ import { Grid, NumericType } from "@foxglove/schemas"; -import { McapWriter, IWritable } from "@mcap/core"; +import { McapWriter } from "@mcap/core"; import { Builder } from "flatbuffers"; import fs from "fs"; -import { open, FileHandle } from "fs/promises"; +import { open } from "fs/promises"; +import { FileHandleWritable } from "@mcap/support/nodejs"; import { buildGridMessage, buildTfMessage } from "./flatbufferUtils"; const QUAT_IDENTITY = { x: 0, y: 0, z: 0, w: 1 }; -// Mcap IWritable interface for nodejs FileHandle -class FileHandleWritable implements IWritable { - private handle: FileHandle; - private totalBytesWritten = 0; - - constructor(handle: FileHandle) { - this.handle = handle; - } - - async write(buffer: Uint8Array): Promise { - const written = await this.handle.write(buffer); - this.totalBytesWritten += written.bytesWritten; - } - - position(): bigint { - return BigInt(this.totalBytesWritten); - } -} function nextPowerOfTwo(numToRound: number) { let nextPower = 1; while (nextPower < numToRound) { diff --git a/typescript/examples/flatbufferswriter/tsconfig.json b/typescript/examples/flatbufferswriter/tsconfig.json index 13b7e779c7..5d97e53644 100644 --- a/typescript/examples/flatbufferswriter/tsconfig.json +++ b/typescript/examples/flatbufferswriter/tsconfig.json @@ -7,7 +7,8 @@ "outDir": "./dist/esm", "lib": ["es2020", "dom"], "paths": { - "@mcap/core": ["../../core/src"] + "@mcap/core": ["../../core/src"], + "@mcap/support": ["../../support/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/examples/validate/package.json b/typescript/examples/validate/package.json index 64d4ebc57d..0b26ab700e 100644 --- a/typescript/examples/validate/package.json +++ b/typescript/examples/validate/package.json @@ -30,6 +30,7 @@ "@foxglove/wasm-lz4": "1.0.2", "@foxglove/wasm-zstd": "1.0.1", "@mcap/core": "*", + "@mcap/support": "*", "@types/jest": "29.4.0", "@types/lodash": "4.14.191", "@types/node": "18.13.0", diff --git a/typescript/examples/validate/scripts/validate.ts b/typescript/examples/validate/scripts/validate.ts index 7cd8c48df1..cd1837c2ff 100644 --- a/typescript/examples/validate/scripts/validate.ts +++ b/typescript/examples/validate/scripts/validate.ts @@ -1,8 +1,6 @@ import { parse as parseMessageDefinition } from "@foxglove/rosmsg"; import { LazyMessageReader as ROS1LazyMessageReader } from "@foxglove/rosmsg-serialization"; import { MessageReader as ROS2MessageReader } from "@foxglove/rosmsg2-serialization"; -import decompressLZ4 from "@foxglove/wasm-lz4"; -import zstd from "@foxglove/wasm-zstd"; import { hasMcapPrefix, McapConstants, @@ -10,16 +8,14 @@ import { McapStreamReader, McapTypes, } from "@mcap/core"; +import { loadDecompressHandlers, protobufFromBinaryDescriptor } from "@mcap/support"; import { program } from "commander"; import { createReadStream } from "fs"; import fs from "fs/promises"; import { isEqual } from "lodash"; import { performance } from "perf_hooks"; -import protobufjs from "protobufjs"; -import { FileDescriptorSet } from "protobufjs/ext/descriptor"; type Channel = McapTypes.Channel; -type DecompressHandlers = McapTypes.DecompressHandlers; type TypedMcapRecord = McapTypes.TypedMcapRecord; function log(...data: unknown[]) { @@ -87,13 +83,7 @@ async function validate( filePath: string, { deserialize, dump, stream }: { deserialize: boolean; dump: boolean; stream: boolean }, ) { - await decompressLZ4.isLoaded; - await zstd.isLoaded; - - const decompressHandlers: DecompressHandlers = { - lz4: (buffer, decompressedSize) => decompressLZ4(buffer, Number(decompressedSize)), - zstd: (buffer, decompressedSize) => zstd.decompress(buffer, Number(decompressedSize)), - }; + const decompressHandlers = await loadDecompressHandlers(); const recordCounts = new Map(); const schemasById = new Map(); @@ -161,7 +151,7 @@ async function validate( ); messageDeserializer = (data) => reader.readMessage(data); } else if (record.messageEncoding === "protobuf") { - const root = protobufjs.Root.fromDescriptor(FileDescriptorSet.decode(schema.data)); + const root = protobufFromBinaryDescriptor(schema.data); const type = root.lookupType(schema.name); messageDeserializer = (data) => diff --git a/typescript/examples/validate/tsconfig.json b/typescript/examples/validate/tsconfig.json index 13b7e779c7..5d97e53644 100644 --- a/typescript/examples/validate/tsconfig.json +++ b/typescript/examples/validate/tsconfig.json @@ -7,7 +7,8 @@ "outDir": "./dist/esm", "lib": ["es2020", "dom"], "paths": { - "@mcap/core": ["../../core/src"] + "@mcap/core": ["../../core/src"], + "@mcap/support": ["../../support/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/examples/validate/typings/protobufjs.d.ts b/typescript/examples/validate/typings/protobufjs.d.ts deleted file mode 100644 index 8c0445835e..0000000000 --- a/typescript/examples/validate/typings/protobufjs.d.ts +++ /dev/null @@ -1,14 +0,0 @@ -import protobufjs from "protobufjs"; -import descriptor from "protobufjs/ext/descriptor"; - -// https://github.com/protobufjs/protobuf.js/issues/1499 -declare module "protobufjs" { - interface ReflectionObject { - toDescriptor( - protoVersion: string, - ): protobufjs.Message & descriptor.IFileDescriptorSet; - } - declare namespace ReflectionObject { - export const fromDescriptor: (desc: protobufjs.Message) => protobufjs.Root; - } -} diff --git a/typescript/support/.eslintrc.js b/typescript/support/.eslintrc.js new file mode 100644 index 0000000000..073f6b2fa3 --- /dev/null +++ b/typescript/support/.eslintrc.js @@ -0,0 +1,19 @@ +/* eslint-env node */ +module.exports = { + env: { es2020: true }, + ignorePatterns: ["dist", "nodejs.d.ts", "nodejs.js"], + extends: ["plugin:@foxglove/base", "plugin:@foxglove/jest"], + overrides: [ + { + files: ["*.ts", "*.tsx"], + extends: ["plugin:@foxglove/typescript"], + parserOptions: { + project: "tsconfig.json", + tsconfigRootDir: __dirname, + }, + }, + ], + rules: { + "no-warning-comments": ["error", { terms: ["fixme"], location: "anywhere" }], + }, +}; diff --git a/typescript/support/.yarnrc b/typescript/support/.yarnrc new file mode 100644 index 0000000000..565db54d62 --- /dev/null +++ b/typescript/support/.yarnrc @@ -0,0 +1,2 @@ +version-tag-prefix "releases/typescript/support/v" +version-git-message "TypeScript @mcap/support v%s" diff --git a/typescript/support/LICENSE b/typescript/support/LICENSE new file mode 100644 index 0000000000..0aa593ea8c --- /dev/null +++ b/typescript/support/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Foxglove Technologies Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/typescript/support/jest.config.json b/typescript/support/jest.config.json new file mode 100644 index 0000000000..c5dcd8fa38 --- /dev/null +++ b/typescript/support/jest.config.json @@ -0,0 +1,16 @@ +{ + "testMatch": ["/src/**/*.test.ts"], + "transform": { + "^.+\\.ts$": "ts-jest" + }, + "globals": { + "ts-jest": { + "diagnostics": { + "//": "add 6133 (unused variables) to default ignore codes", + "ignoreCodes": [6059, 18002, 18003, 6133] + } + } + }, + "//": "Native find is slow because it does not exclude files: https://github.com/facebook/jest/pull/11264#issuecomment-825377579", + "haste": { "forceNodeFilesystemAPI": true } +} diff --git a/typescript/support/nodejs.d.ts b/typescript/support/nodejs.d.ts new file mode 100644 index 0000000000..07ef4f5705 --- /dev/null +++ b/typescript/support/nodejs.d.ts @@ -0,0 +1 @@ +export * from "./src/nodejs"; diff --git a/typescript/support/nodejs.js b/typescript/support/nodejs.js new file mode 100644 index 0000000000..557ef34e07 --- /dev/null +++ b/typescript/support/nodejs.js @@ -0,0 +1 @@ +module.exports = require("./dist/cjs/nodejs/index.js"); diff --git a/typescript/support/package.json b/typescript/support/package.json new file mode 100644 index 0000000000..15c0662dd8 --- /dev/null +++ b/typescript/support/package.json @@ -0,0 +1,77 @@ +{ + "name": "@mcap/support", + "version": "0.1.0", + "description": "Common schema and message parsing logic for use with MCAP and related protocols", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/foxglove/mcap.git" + }, + "author": { + "name": "Foxglove Technologies", + "email": "support@foxglove.dev" + }, + "homepage": "https://foxglove.dev/", + "module": "dist/esm/index.js", + "main": "dist/cjs/index.js", + "typings": "dist/esm/index.d.ts", + "exports": { + ".": { + "import": "./dist/esm/index.js", + "require": "./dist/cjs/index.js" + }, + "./nodejs": { + "import": "./dist/esm/nodejs/index.js", + "require": "./dist/cjs/nodejs/index.js" + } + }, + "typedocMain": "src/index.ts", + "files": [ + "dist", + "src", + "nodejs.d.ts", + "nodejs.js" + ], + "scripts": { + "prepack": "tsc -b tsconfig.json tsconfig.cjs.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "lint:ci": "eslint --report-unused-disable-directives .", + "lint": "eslint --report-unused-disable-directives --fix .", + "test": "jest" + }, + "devDependencies": { + "@foxglove/eslint-plugin": "0.21.0", + "@foxglove/tsconfig": "1.1.0", + "@types/jest": "29.4.0", + "@types/node": "18.13.0", + "@typescript-eslint/eslint-plugin": "5.52.0", + "@typescript-eslint/parser": "5.52.0", + "cspell": "^6.26.3", + "eslint": "8.34.0", + "eslint-config-prettier": "8.6.0", + "eslint-plugin-es": "4.1.0", + "eslint-plugin-filenames": "1.3.2", + "eslint-plugin-import": "2.27.5", + "eslint-plugin-jest": "27.2.1", + "eslint-plugin-prettier": "4.2.1", + "jest": "29.4.3", + "prettier": "2.8.4", + "ts-jest": "29.0.5", + "typedoc": "^0.23.25", + "typescript": "4.9.5" + }, + "dependencies": { + "@foxglove/message-definition": "^0.2.0", + "@foxglove/rosmsg": "^4.0.0", + "@foxglove/rosmsg-serialization": "^2.0.0", + "@foxglove/rosmsg2-serialization": "^2.0.0", + "@foxglove/schemas": "^1.1.0", + "@foxglove/wasm-bz2": "^0.1.1", + "@foxglove/wasm-lz4": "^1.0.2", + "@foxglove/wasm-zstd": "^1.0.1", + "@protobufjs/base64": "^1.1.2", + "flatbuffers": "^23.1.21", + "flatbuffers_reflection": "^0.0.3", + "protobufjs": "^7.2.2" + } +} diff --git a/typescript/support/src/decompressHandlers.ts b/typescript/support/src/decompressHandlers.ts new file mode 100644 index 0000000000..b67df875a9 --- /dev/null +++ b/typescript/support/src/decompressHandlers.ts @@ -0,0 +1,32 @@ +type DecompressHandlers = { + [compression: string]: (buffer: Uint8Array, decompressedSize: bigint) => Uint8Array; +}; + +let handlersPromise: Promise | undefined; +export async function loadDecompressHandlers(): Promise { + return await (handlersPromise ??= _loadDecompressHandlers()); +} + +// eslint-disable-next-line no-underscore-dangle +async function _loadDecompressHandlers(): Promise { + const [decompressZstd, decompressLZ4, bzip2] = await Promise.all([ + import("@foxglove/wasm-zstd").then(async (mod) => { + await mod.isLoaded; + return mod.decompress; + }), + import("@foxglove/wasm-lz4").then(async (mod) => { + await mod.default.isLoaded; + return mod.default; + }), + import("@foxglove/wasm-bz2").then(async (mod) => await mod.default.init()), + ]); + + return { + lz4: (buffer, decompressedSize) => decompressLZ4(buffer, Number(decompressedSize)), + + bz2: (buffer, decompressedSize) => + bzip2.decompress(buffer, Number(decompressedSize), { small: false }), + + zstd: (buffer, decompressedSize) => decompressZstd(buffer, Number(decompressedSize)), + }; +} diff --git a/typescript/support/src/fixtures/ByteVector.bfbs b/typescript/support/src/fixtures/ByteVector.bfbs new file mode 100644 index 0000000000..9c57006f12 --- /dev/null +++ b/typescript/support/src/fixtures/ByteVector.bfbs @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b800c9b0754f148c25ebd86ba9457923a6dd1099d55e5e9f2bd064e85001ac2 +size 296 diff --git a/typescript/support/src/fixtures/byte-vector.ts b/typescript/support/src/fixtures/byte-vector.ts new file mode 100644 index 0000000000..4b7e56030a --- /dev/null +++ b/typescript/support/src/fixtures/byte-vector.ts @@ -0,0 +1,95 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/ + +// automatically generated by the FlatBuffers compiler, do not modify +/* eslint-disable */ +import * as flatbuffers from "flatbuffers"; + +export class ByteVector { + bb: flatbuffers.ByteBuffer | null = null; + bb_pos = 0; + __init(i: number, bb: flatbuffers.ByteBuffer): ByteVector { + this.bb_pos = i; + this.bb = bb; + return this; + } + + static getRootAsByteVector(bb: flatbuffers.ByteBuffer, obj?: ByteVector): ByteVector { + return (obj || new ByteVector()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + static getSizePrefixedRootAsByteVector(bb: flatbuffers.ByteBuffer, obj?: ByteVector): ByteVector { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new ByteVector()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + data(index: number): number | null { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.readUint8(this.bb!.__vector(this.bb_pos + offset) + index) : 0; + } + + dataLength(): number { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; + } + + dataArray(): Uint8Array | null { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset + ? new Uint8Array( + this.bb!.bytes().buffer, + this.bb!.bytes().byteOffset + this.bb!.__vector(this.bb_pos + offset), + this.bb!.__vector_len(this.bb_pos + offset), + ) + : null; + } + + static startByteVector(builder: flatbuffers.Builder) { + builder.startObject(1); + } + + static addData(builder: flatbuffers.Builder, dataOffset: flatbuffers.Offset) { + builder.addFieldOffset(0, dataOffset, 0); + } + + static createDataVector( + builder: flatbuffers.Builder, + data: number[] | Uint8Array, + ): flatbuffers.Offset { + builder.startVector(1, data.length, 1); + for (let i = data.length - 1; i >= 0; i--) { + builder.addInt8(data[i]!); + } + return builder.endVector(); + } + + static startDataVector(builder: flatbuffers.Builder, numElems: number) { + builder.startVector(1, numElems, 1); + } + + static endByteVector(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static finishByteVectorBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset) { + builder.finish(offset); + } + + static finishSizePrefixedByteVectorBuffer( + builder: flatbuffers.Builder, + offset: flatbuffers.Offset, + ) { + builder.finish(offset, undefined, true); + } + + static createByteVector( + builder: flatbuffers.Builder, + dataOffset: flatbuffers.Offset, + ): flatbuffers.Offset { + ByteVector.startByteVector(builder); + ByteVector.addData(builder, dataOffset); + return ByteVector.endByteVector(builder); + } +} diff --git a/typescript/support/src/fixtures/reflection.bfbs b/typescript/support/src/fixtures/reflection.bfbs new file mode 100644 index 0000000000..f5b1e43d5a --- /dev/null +++ b/typescript/support/src/fixtures/reflection.bfbs @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a345ca0456791076519c32a8c5f3a95b8fc8dd680efddc659a7bc723c01128f +size 5608 diff --git a/typescript/support/src/index.ts b/typescript/support/src/index.ts new file mode 100644 index 0000000000..c8aee4c069 --- /dev/null +++ b/typescript/support/src/index.ts @@ -0,0 +1,5 @@ +export * from "./parseJsonSchema"; +export * from "./protobufDefinitionsToDatatypes"; +export * from "./protobufDescriptors"; +export * from "./parseChannel"; +export * from "./decompressHandlers"; diff --git a/typescript/support/src/nodejs/FileHandleWritable.ts b/typescript/support/src/nodejs/FileHandleWritable.ts new file mode 100644 index 0000000000..dc2fdbcfcd --- /dev/null +++ b/typescript/support/src/nodejs/FileHandleWritable.ts @@ -0,0 +1,27 @@ +import { FileHandle } from "fs/promises"; + +interface IWritable { + write(buffer: Uint8Array): Promise; + position(): bigint; +} + +/** + * IWritable implementation for FileHandle. + */ +export class FileHandleWritable implements IWritable { + private handle: FileHandle; + private totalBytesWritten = 0; + + constructor(handle: FileHandle) { + this.handle = handle; + } + + async write(buffer: Uint8Array): Promise { + const written = await this.handle.write(buffer); + this.totalBytesWritten += written.bytesWritten; + } + + position(): bigint { + return BigInt(this.totalBytesWritten); + } +} diff --git a/typescript/support/src/nodejs/index.ts b/typescript/support/src/nodejs/index.ts new file mode 100644 index 0000000000..7f4bb8327f --- /dev/null +++ b/typescript/support/src/nodejs/index.ts @@ -0,0 +1 @@ +export * from "./FileHandleWritable"; diff --git a/typescript/support/src/parseChannel.test.ts b/typescript/support/src/parseChannel.test.ts new file mode 100644 index 0000000000..d9d3ab65aa --- /dev/null +++ b/typescript/support/src/parseChannel.test.ts @@ -0,0 +1,104 @@ +import fs from "fs"; +import { FileDescriptorSet, IFileDescriptorSet } from "protobufjs/ext/descriptor"; + +import { parseChannel } from "./parseChannel"; +import { protobufToDescriptor } from "./protobufDescriptors"; + +describe("parseChannel", () => { + it("works with json/jsonschema", () => { + const channel = parseChannel({ + messageEncoding: "json", + schema: { + name: "X", + encoding: "jsonschema", + data: new TextEncoder().encode( + JSON.stringify({ type: "object", properties: { value: { type: "string" } } }), + ), + }, + }); + expect(channel.deserializer(new TextEncoder().encode(JSON.stringify({ value: "hi" })))).toEqual( + { value: "hi" }, + ); + }); + + it("works with flatbuffer", () => { + const reflectionSchema = fs.readFileSync(`${__dirname}/fixtures/reflection.bfbs`); + const channel = parseChannel({ + messageEncoding: "flatbuffer", + schema: { name: "reflection.Schema", encoding: "flatbuffer", data: reflectionSchema }, + }); + const deserialized = channel.deserializer(reflectionSchema) as { + objects: Record[]; + }; + expect(deserialized.objects.length).toEqual(10); + expect(deserialized.objects[0]!.name).toEqual("reflection.Enum"); + }); + + it("works with protobuf", () => { + const fds = FileDescriptorSet.encode(protobufToDescriptor(FileDescriptorSet.root)).finish(); + const channel = parseChannel({ + messageEncoding: "protobuf", + schema: { name: "google.protobuf.FileDescriptorSet", encoding: "protobuf", data: fds }, + }); + const deserialized = channel.deserializer(fds) as IFileDescriptorSet; + expect(deserialized.file[0]!.name).toEqual("google_protobuf.proto"); + }); + + it("handles protobuf repeated enum having multiple default value aliases", () => { + /* + syntax = "proto3"; + + enum ExampleEnum { + option allow_alias = true; + UNKNOWN = 0; + WHATEVER = 0; + FOO = 1; + BAR = 2; + } + + message ExampleMessage { + repeated ExampleEnum data = 1; + } + */ + const channel = parseChannel({ + messageEncoding: "protobuf", + schema: { + name: "ExampleMessage", + encoding: "protobuf", + data: Buffer.from( + "0A8D010A156578616D706C655F6D6573736167652E70726F746F222C0A0E4578616D706C654D657373616765121A0A046461746118012003280E320C2E4578616D706C65456E756D2A3E0A0B4578616D706C65456E756D120B0A07554E4B4E4F574E1000120C0A085748415445564552100012070A03464F4F100112070A0342415210021A021001620670726F746F33", + "hex", + ), + }, + }); + expect(channel.deserializer(Buffer.from("0A0101", "hex"))).toEqual({ data: [1] }); + }); + + it("works with ros1", () => { + const channel = parseChannel({ + messageEncoding: "ros1", + schema: { + name: "foo_msgs/Bar", + encoding: "ros1msg", + data: new TextEncoder().encode("string data"), + }, + }); + + const obj = channel.deserializer(new Uint8Array([4, 0, 0, 0, 65, 66, 67, 68])); + expect(obj).toEqual({ data: "ABCD" }); + }); + + it("works with ros2", () => { + const channel = parseChannel({ + messageEncoding: "cdr", + schema: { + name: "foo_msgs/Bar", + encoding: "ros2msg", + data: new TextEncoder().encode("string data"), + }, + }); + + const obj = channel.deserializer(new Uint8Array([0, 1, 0, 0, 5, 0, 0, 0, 65, 66, 67, 68, 0])); + expect(obj).toEqual({ data: "ABCD" }); + }); +}); diff --git a/typescript/support/src/parseChannel.ts b/typescript/support/src/parseChannel.ts new file mode 100644 index 0000000000..9c12f3ad77 --- /dev/null +++ b/typescript/support/src/parseChannel.ts @@ -0,0 +1,184 @@ +import { MessageDefinition } from "@foxglove/message-definition"; +import { parse as parseMessageDefinition } from "@foxglove/rosmsg"; +import { MessageReader } from "@foxglove/rosmsg-serialization"; +import { MessageReader as ROS2MessageReader } from "@foxglove/rosmsg2-serialization"; +import { FileDescriptorSet, IFileDescriptorSet } from "protobufjs/ext/descriptor"; + +import { parseFlatbufferSchema } from "./parseFlatbufferSchema"; +import { parseJsonSchema } from "./parseJsonSchema"; +import { protobufDefinitionsToDatatypes, stripLeadingDot } from "./protobufDefinitionsToDatatypes"; +import { protobufFromDescriptor } from "./protobufDescriptors"; +import { MessageDefinitionMap } from "./types"; + +type Channel = { + messageEncoding: string; + schema: { name: string; encoding: string; data: Uint8Array } | undefined; +}; + +export type ParsedChannel = { + deserializer: (data: ArrayBufferView) => unknown; + datatypes: MessageDefinitionMap; +}; + +function parsedDefinitionsToDatatypes( + parsedDefinitions: MessageDefinition[], + rootName: string, +): MessageDefinitionMap { + const datatypes: MessageDefinitionMap = new Map(); + parsedDefinitions.forEach(({ name, definitions }, index) => { + if (index === 0) { + datatypes.set(rootName, { name: rootName, definitions }); + } else if (name != undefined) { + datatypes.set(name, { name, definitions }); + } + }); + return datatypes; +} + +/** + * Process a channel/schema and extract information that can be used to deserialize messages on the + * channel, and schemas in the format expected by Studio's RosDatatypes. + * + * See: + * - https://github.com/foxglove/mcap/blob/main/docs/specification/well-known-message-encodings.md + * - https://github.com/foxglove/mcap/blob/main/docs/specification/well-known-schema-encodings.md + */ +export function parseChannel(channel: Channel): ParsedChannel { + if (channel.messageEncoding === "json") { + if (channel.schema != undefined && channel.schema.encoding !== "jsonschema") { + throw new Error( + `Message encoding ${channel.messageEncoding} with schema encoding '${channel.schema.encoding}' is not supported (expected jsonschema or no schema)`, + ); + } + const textDecoder = new TextDecoder(); + let datatypes: MessageDefinitionMap = new Map(); + let deserializer = (data: ArrayBufferView) => JSON.parse(textDecoder.decode(data)) as unknown; + if (channel.schema != undefined) { + const schema = + channel.schema.data.length > 0 + ? (JSON.parse(textDecoder.decode(channel.schema.data)) as unknown) + : undefined; + if (schema != undefined) { + if (typeof schema !== "object") { + throw new Error(`Invalid schema, expected JSON object, got ${typeof schema}`); + } + const { datatypes: parsedDatatypes, postprocessValue } = parseJsonSchema( + schema as Record, + channel.schema.name, + ); + datatypes = parsedDatatypes; + deserializer = (data) => + postprocessValue(JSON.parse(textDecoder.decode(data)) as Record); + } + } + return { deserializer, datatypes }; + } + + if (channel.messageEncoding === "flatbuffer") { + if (channel.schema?.encoding !== "flatbuffer") { + throw new Error( + `Message encoding ${channel.messageEncoding} with ${ + channel.schema == undefined + ? "no encoding" + : `schema encoding '${channel.schema.encoding}'` + } is not supported (expected flatbuffer)`, + ); + } + return parseFlatbufferSchema(channel.schema.name, channel.schema.data); + } + + if (channel.messageEncoding === "protobuf") { + if (channel.schema?.encoding !== "protobuf") { + throw new Error( + `Message encoding ${channel.messageEncoding} with ${ + channel.schema == undefined + ? "no encoding" + : `schema encoding '${channel.schema.encoding}'` + } is not supported (expected protobuf)`, + ); + } + const descriptorSet = FileDescriptorSet.decode(channel.schema.data); + + // Modify the definition of google.protobuf.Timestamp so it gets parsed as {sec, nsec}, + // compatible with the rest of Studio. + for (const file of (descriptorSet as unknown as IFileDescriptorSet).file) { + if (file.package === "google.protobuf") { + for (const message of file.messageType ?? []) { + if (message.name === "Timestamp" || message.name === "Duration") { + for (const field of message.field ?? []) { + if (field.name === "seconds") { + field.name = "sec"; + } else if (field.name === "nanos") { + field.name = "nsec"; + } + } + } + } + } + } + + const root = protobufFromDescriptor(descriptorSet); + root.resolveAll(); + const type = root.lookupType(channel.schema.name); + + const deserializer = (data: ArrayBufferView) => { + return type.toObject( + type.decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength)), + { defaults: true }, + ); + }; + + const datatypes: MessageDefinitionMap = new Map(); + protobufDefinitionsToDatatypes(datatypes, type); + + if (!datatypes.has(channel.schema.name)) { + throw new Error( + `Protobuf schema does not contain an entry for '${ + channel.schema.name + }'. The schema name should be fully-qualified, e.g. '${stripLeadingDot(type.fullName)}'.`, + ); + } + + return { deserializer, datatypes }; + } + + if (channel.messageEncoding === "ros1") { + if (channel.schema?.encoding !== "ros1msg") { + throw new Error( + `Message encoding ${channel.messageEncoding} with ${ + channel.schema == undefined + ? "no encoding" + : `schema encoding '${channel.schema.encoding}'` + } is not supported (expected ros1msg)`, + ); + } + const schema = new TextDecoder().decode(channel.schema.data); + const parsedDefinitions = parseMessageDefinition(schema); + const reader = new MessageReader(parsedDefinitions); + return { + datatypes: parsedDefinitionsToDatatypes(parsedDefinitions, channel.schema.name), + deserializer: (data) => reader.readMessage(data), + }; + } + + if (channel.messageEncoding === "cdr") { + if (channel.schema?.encoding !== "ros2msg") { + throw new Error( + `Message encoding ${channel.messageEncoding} with ${ + channel.schema == undefined + ? "no encoding" + : `schema encoding '${channel.schema.encoding}'` + } is not supported (expected ros2msg)`, + ); + } + const schema = new TextDecoder().decode(channel.schema.data); + const parsedDefinitions = parseMessageDefinition(schema, { ros2: true }); + const reader = new ROS2MessageReader(parsedDefinitions); + return { + datatypes: parsedDefinitionsToDatatypes(parsedDefinitions, channel.schema.name), + deserializer: (data) => reader.readMessage(data), + }; + } + + throw new Error(`Unsupported encoding ${channel.messageEncoding}`); +} diff --git a/typescript/support/src/parseFlatbufferSchema.test.ts b/typescript/support/src/parseFlatbufferSchema.test.ts new file mode 100644 index 0000000000..9787e6ab74 --- /dev/null +++ b/typescript/support/src/parseFlatbufferSchema.test.ts @@ -0,0 +1,384 @@ +/* eslint @typescript-eslint/no-explicit-any: 0 */ +/* eslint @typescript-eslint/no-unsafe-member-access: 0 */ +/* eslint @typescript-eslint/no-unsafe-call: 0 */ + +import { ByteBuffer, Builder } from "flatbuffers"; +import { Schema, BaseType, Type } from "flatbuffers_reflection"; +import fs from "fs"; + +import { ByteVector } from "./fixtures/byte-vector"; +import { parseFlatbufferSchema } from "./parseFlatbufferSchema"; + +const enumSchema = { + definitions: [ + { + isArray: true, + isComplex: true, + name: "attributes", + type: "reflection.KeyValue", + }, + { + name: "declaration_file", + type: "string", + }, + { + isArray: true, + name: "documentation", + type: "string", + }, + { + name: "is_union", + type: "bool", + }, + { + name: "name", + type: "string", + }, + { + isComplex: true, + name: "underlying_type", + type: "reflection.Type", + }, + { + isArray: true, + isComplex: true, + name: "values", + type: "reflection.EnumVal", + }, + ], +}; +const typeSchema = { + definitions: [ + { + name: "base_size", + type: "uint32", + }, + { + isConstant: true, + name: "None", + type: "int8", + value: 0n, + }, + { + isConstant: true, + name: "UType", + type: "int8", + value: 1n, + }, + { + isConstant: true, + name: "Bool", + type: "int8", + value: 2n, + }, + { + isConstant: true, + name: "Byte", + type: "int8", + value: 3n, + }, + { + isConstant: true, + name: "UByte", + type: "int8", + value: 4n, + }, + { + isConstant: true, + name: "Short", + type: "int8", + value: 5n, + }, + { + isConstant: true, + name: "UShort", + type: "int8", + value: 6n, + }, + { + isConstant: true, + name: "Int", + type: "int8", + value: 7n, + }, + { + isConstant: true, + name: "UInt", + type: "int8", + value: 8n, + }, + { + isConstant: true, + name: "Long", + type: "int8", + value: 9n, + }, + { + isConstant: true, + name: "ULong", + type: "int8", + value: 10n, + }, + { + isConstant: true, + name: "Float", + type: "int8", + value: 11n, + }, + { + isConstant: true, + name: "Double", + type: "int8", + value: 12n, + }, + { + isConstant: true, + name: "String", + type: "int8", + value: 13n, + }, + { + isConstant: true, + name: "Vector", + type: "int8", + value: 14n, + }, + { + isConstant: true, + name: "Obj", + type: "int8", + value: 15n, + }, + { + isConstant: true, + name: "Union", + type: "int8", + value: 16n, + }, + { + isConstant: true, + name: "Array", + type: "int8", + value: 17n, + }, + { + isConstant: true, + name: "MaxBaseType", + type: "int8", + value: 18n, + }, + { + name: "base_type", + type: "int8", + }, + { + isConstant: true, + name: "None", + type: "int8", + value: 0n, + }, + { + isConstant: true, + name: "UType", + type: "int8", + value: 1n, + }, + { + isConstant: true, + name: "Bool", + type: "int8", + value: 2n, + }, + { + isConstant: true, + name: "Byte", + type: "int8", + value: 3n, + }, + { + isConstant: true, + name: "UByte", + type: "int8", + value: 4n, + }, + { + isConstant: true, + name: "Short", + type: "int8", + value: 5n, + }, + { + isConstant: true, + name: "UShort", + type: "int8", + value: 6n, + }, + { + isConstant: true, + name: "Int", + type: "int8", + value: 7n, + }, + { + isConstant: true, + name: "UInt", + type: "int8", + value: 8n, + }, + { + isConstant: true, + name: "Long", + type: "int8", + value: 9n, + }, + { + isConstant: true, + name: "ULong", + type: "int8", + value: 10n, + }, + { + isConstant: true, + name: "Float", + type: "int8", + value: 11n, + }, + { + isConstant: true, + name: "Double", + type: "int8", + value: 12n, + }, + { + isConstant: true, + name: "String", + type: "int8", + value: 13n, + }, + { + isConstant: true, + name: "Vector", + type: "int8", + value: 14n, + }, + { + isConstant: true, + name: "Obj", + type: "int8", + value: 15n, + }, + { + isConstant: true, + name: "Union", + type: "int8", + value: 16n, + }, + { + isConstant: true, + name: "Array", + type: "int8", + value: 17n, + }, + { + isConstant: true, + name: "MaxBaseType", + type: "int8", + value: 18n, + }, + { + name: "element", + type: "int8", + }, + { + name: "element_size", + type: "uint32", + }, + { + name: "fixed_length", + type: "uint16", + }, + { + name: "index", + type: "int32", + }, + ], +}; + +describe("parseFlatbufferSchema", () => { + it("rejects invalid schema", () => { + expect(() => parseFlatbufferSchema("test", new Uint8Array([1]))).toThrow(); + }); + it("parses root table schema", () => { + // Use the reflection Schema itself to read the reflection Schema (this is + // actually a pretty good test case for coverage, since the Schema message + // includes almost all the various flatbuffer features). + // The .bfbs file in question is generated from running + // $ flatc -b --schema reflection/reflection.fbs + // In https://github.com/google/flatbuffers + const reflectionSchemaBuffer: Buffer = fs.readFileSync(`${__dirname}/fixtures/reflection.bfbs`); + const { datatypes, deserializer } = parseFlatbufferSchema( + "reflection.Schema", + reflectionSchemaBuffer, + ); + const deserialized: any = deserializer(reflectionSchemaBuffer); + const reflectionSchemaByteBuffer: ByteBuffer = new ByteBuffer(reflectionSchemaBuffer); + const schema = Schema.getRootAsSchema(reflectionSchemaByteBuffer); + // Spot check individual components to ensure that they got deserialized correctly. + expect(deserialized.objects.length).toEqual(schema.objectsLength()); + expect(deserialized.objects.length).toEqual(10); + expect(deserialized.objects[0].name).toEqual("reflection.Enum"); + expect(deserialized.file_ident).toEqual("BFBS"); + expect(deserialized.file_ext).toEqual("bfbs"); + expect(deserialized.fbs_files[0].filename.substr(-14)).toEqual("reflection.fbs"); + // Spot check the datatypes list. + expect(datatypes.keys()).toContain("reflection.Enum"); + expect(datatypes.keys()).toContain("reflection.Object"); + expect(datatypes.get("reflection.Enum")).toEqual(enumSchema); + }); + it("parses non-root table schema", () => { + const reflectionSchemaBuffer: Buffer = fs.readFileSync(`${__dirname}/fixtures/reflection.bfbs`); + const { datatypes, deserializer } = parseFlatbufferSchema( + "reflection.Type", + reflectionSchemaBuffer, + ); + expect(datatypes.keys()).toContain("reflection.Type"); + expect(datatypes.get("reflection.Type")).toEqual(typeSchema); + + // Construct a reflection.Type object from scratch and confirm that we get + // exactly the correct result. + const builder = new Builder(); + Type.startType(builder); + Type.addBaseType(builder, BaseType.Int); + Type.addIndex(builder, 123); + builder.finish(Type.endType(builder)); + + expect(deserializer(builder.asUint8Array())).toEqual({ base_type: 7, index: 123 }); + }); + it("converts uint8 vectors to uint8arrays", () => { + const builder = new Builder(); + + /** + * Byte Vector Schema (.fbs file not included in this repo) + * table ByteVector { + * data:[uint8]; + * } + * root_type ByteVector; + */ + const data = ByteVector.createDataVector(builder, [1, 2, 3]); + ByteVector.startByteVector(builder); + ByteVector.addData(builder, data); + const byteVector = ByteVector.endByteVector(builder); + builder.finish(byteVector); + /** the underlying buffer for the builder is larger than the uint8array of the data + * this needs to be cleared so that the reading from the buffer by the parser doesn't use the wrong offsets + * normally when this is written to a file, only the contents of the Uint8Array are written, not the underlying buffer + * so this replicates that + * essentially need to make sure byteVectorBin.buffer !== builder.asUint8Array().buffer + */ + const byteVectorBin = Uint8Array.from(builder.asUint8Array()); + + const byteVectorSchemaArray = fs.readFileSync(`${__dirname}/fixtures/ByteVector.bfbs`); + const { deserializer } = parseFlatbufferSchema("ByteVector", byteVectorSchemaArray); + expect(deserializer(byteVectorBin)).toEqual({ data: new Uint8Array([1, 2, 3]) }); + }); +}); diff --git a/typescript/support/src/parseFlatbufferSchema.ts b/typescript/support/src/parseFlatbufferSchema.ts new file mode 100644 index 0000000000..935d9a7ef2 --- /dev/null +++ b/typescript/support/src/parseFlatbufferSchema.ts @@ -0,0 +1,199 @@ +import { MessageDefinitionField } from "@foxglove/message-definition"; +import { ByteBuffer } from "flatbuffers"; +import { BaseType, Schema, SchemaT, FieldT, Parser, Table } from "flatbuffers_reflection"; + +import { MessageDefinitionMap } from "./types"; + +function typeForSimpleField(type: BaseType): string { + switch (type) { + case BaseType.Bool: + return "bool"; + case BaseType.Byte: + return "int8"; + case BaseType.UType: + case BaseType.UByte: + return "uint8"; + case BaseType.Short: + return "int16"; + case BaseType.UShort: + return "uint16"; + case BaseType.Int: + return "int32"; + case BaseType.UInt: + return "uint32"; + case BaseType.Long: + return "int64"; + case BaseType.ULong: + return "uint64"; + case BaseType.Float: + return "float32"; + case BaseType.Double: + return "float64"; + case BaseType.String: + return "string"; + case BaseType.Vector: + case BaseType.Obj: + case BaseType.Union: + case BaseType.Array: + throw new Error(`${type} is not a simple type.`); + case BaseType.None: + case BaseType.MaxBaseType: + throw new Error("None is not a valid type."); + } +} + +function flatbufferString(unchecked: string | Uint8Array | null | undefined): string { + if (typeof unchecked === "string") { + return unchecked; + } + throw new Error(`Expected string, found ${typeof unchecked}`); +} + +function typeForField(schema: SchemaT, field: FieldT): MessageDefinitionField[] { + const fields: MessageDefinitionField[] = []; + switch (field.type?.baseType) { + case BaseType.UType: + case BaseType.Bool: + case BaseType.Byte: + case BaseType.UByte: + case BaseType.Short: + case BaseType.UShort: + case BaseType.Int: + case BaseType.UInt: + case BaseType.Long: + case BaseType.ULong: + case BaseType.Float: + case BaseType.Double: + case BaseType.String: + case BaseType.None: { + const simpleType = typeForSimpleField(field.type.baseType); + // Enums have magic logic--the constants definitions for the enum values + // have to go right before the enum itself. + if (field.type.index !== -1) { + const enums = schema.enums[field.type.index]?.values; + if (enums == undefined) { + throw new Error( + `Invalid schema, missing enum values for field type ${ + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + schema.enums[field.type.index]?.name + }`, + ); + } + for (const enumVal of enums) { + fields.push({ + name: flatbufferString(enumVal.name), + type: simpleType, + isConstant: true, + value: enumVal.value, + }); + } + } + fields.push({ name: flatbufferString(field.name), type: simpleType }); + break; + } + case BaseType.Vector: + switch (field.type.element) { + case BaseType.Vector: + case BaseType.Union: + case BaseType.Array: + case BaseType.None: + throw new Error("Vectors of vectors, unions, arrays, and None's are unsupported."); + case BaseType.Obj: + fields.push({ + name: flatbufferString(field.name), + type: flatbufferString(schema.objects[field.type.index]?.name), + isComplex: true, + isArray: true, + }); + break; + default: { + const type = typeForSimpleField(field.type.element); + // Enums have magic logic--the constants definitions for the enum + // values have to go right before the enum itself. + if (field.type.index !== -1) { + const enums = schema.enums[field.type.index]?.values; + if (enums == undefined) { + throw new Error("Invalid schema"); + } + for (const enumVal of enums) { + fields.push({ + name: flatbufferString(enumVal.name), + type, + isConstant: true, + value: enumVal.value, + }); + } + } + fields.push({ name: flatbufferString(field.name), type, isArray: true }); + break; + } + } + break; + case BaseType.Obj: + fields.push({ + name: flatbufferString(field.name), + type: flatbufferString(schema.objects[field.type.index]?.name), + isComplex: true, + }); + break; + case BaseType.Union: + case BaseType.Array: + case BaseType.MaxBaseType: + case undefined: + throw new Error("Unions and Arrays are not supported in @mcap/support currently."); + } + return fields; +} + +// Note: Currently this does not support "lazy" message reading in the style of the ros1 message +// reader, and so will relatively inefficiently deserialize the entire flatbuffer message. +export function parseFlatbufferSchema( + schemaName: string, + schemaArray: Uint8Array, +): { + datatypes: MessageDefinitionMap; + deserializer: (buffer: ArrayBufferView) => unknown; +} { + const datatypes: MessageDefinitionMap = new Map(); + const schemaBuffer = new ByteBuffer(schemaArray); + const rawSchema = Schema.getRootAsSchema(schemaBuffer); + const schema = rawSchema.unpack(); + + let typeIndex = -1; + for (let schemaIndex = 0; schemaIndex < schema.objects.length; ++schemaIndex) { + const object = schema.objects[schemaIndex]; + if (object?.name === schemaName) { + typeIndex = schemaIndex; + } + let fields: MessageDefinitionField[] = []; + if (object?.fields == undefined) { + continue; + } + for (const field of object.fields) { + fields = fields.concat(typeForField(schema, field)); + } + datatypes.set(flatbufferString(object.name), { definitions: fields }); + } + if (typeIndex === -1) { + if (schema.rootTable?.name !== schemaName) { + throw new Error( + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + `Type "${schemaName}" is not available in the schema for "${schema.rootTable?.name}".`, + ); + } + } + const parser = new Parser(rawSchema); + const deserializer = (buffer: ArrayBufferView) => { + const byteBuffer = new ByteBuffer( + new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength), + ); + const table = new Table( + byteBuffer, + typeIndex, + byteBuffer.readInt32(byteBuffer.position()) + byteBuffer.position(), + ); + const obj = parser.toObject(table); + return obj; + }; + return { datatypes, deserializer }; +} diff --git a/typescript/support/src/parseJsonSchema.test.ts b/typescript/support/src/parseJsonSchema.test.ts new file mode 100644 index 0000000000..105fcc01eb --- /dev/null +++ b/typescript/support/src/parseJsonSchema.test.ts @@ -0,0 +1,275 @@ +import { foxgloveMessageSchemas, generateJsonSchema } from "@foxglove/schemas/internal"; +import * as protobufjs from "protobufjs"; + +import { parseJsonSchema } from "./parseJsonSchema"; + +describe("parseJsonSchema", () => { + it("rejects invalid schema", () => { + expect(() => parseJsonSchema({}, "X")).toThrow(`Expected "type": "object"`); + expect(() => parseJsonSchema({ type: 3 }, "X")).toThrow(`Expected "type": "object"`); + expect(() => parseJsonSchema({ type: "string" }, "X")).toThrow(`Expected "type": "object"`); + expect(() => + parseJsonSchema({ type: "object", properties: { x: { type: "null" } } }, "X"), + ).toThrow(`Unsupported type "null" for x`); + expect(() => + parseJsonSchema( + { type: "object", properties: { x: { type: "string", contentEncoding: "X" } } }, + "X", + ), + ).toThrow(`Unsupported contentEncoding "X"`); + expect(() => + parseJsonSchema( + { + type: "object", + properties: { + x: { type: "array", items: { type: "string", contentEncoding: "base64" } }, + }, + }, + "X", + ), + ).toThrow(`Unsupported contentEncoding "base64" for array item`); + }); + + it.each([ + { + name: "Foo", + schema: { type: "object", properties: {} }, + expectedDatatypes: new Map([["Foo", { definitions: [] }]]), + value: {}, + expectedValue: {}, + }, + + { + name: "Foo", + schema: { + type: "object", + properties: { + str: { type: "string" }, + bin: { type: "string", contentEncoding: "base64" }, + num: { type: "number" }, + int: { type: "integer" }, + bool: { type: "boolean" }, + }, + }, + expectedDatatypes: new Map([ + [ + "Foo", + { + definitions: [ + { name: "str", type: "string" }, + { name: "bin", type: "uint8", isArray: true }, + { name: "num", type: "float64" }, + { name: "int", type: "float64" }, + { name: "bool", type: "bool" }, + ], + }, + ], + ]), + value: { + str: "str", + bin: protobufjs.util.base64.encode(new Uint8Array([0, 1, 0xfe, 0xff]), 0, 4), + num: 1.5, + int: 1.5, + bool: true, + }, + expectedValue: { + str: "str", + bin: new Uint8Array([0, 1, 0xfe, 0xff]), + num: 1.5, + int: 1.5, + bool: true, + }, + }, + + { + name: "Foo", + schema: { + type: "object", + properties: { + str: { type: "array", items: { type: "string" } }, + num: { type: "array", items: { type: "number" } }, + int: { type: "array", items: { type: "integer" } }, + bool: { type: "array", items: { type: "boolean" } }, + }, + }, + expectedDatatypes: new Map([ + [ + "Foo", + { + definitions: [ + { name: "str", type: "string", isArray: true }, + { name: "num", type: "float64", isArray: true }, + { name: "int", type: "float64", isArray: true }, + { name: "bool", type: "bool", isArray: true }, + ], + }, + ], + ]), + value: { str: ["str"], num: [1.5], int: [1.5], bool: [true] }, + expectedValue: { str: ["str"], num: [1.5], int: [1.5], bool: [true] }, + }, + + { + name: "Foo", + schema: { + type: "object", + properties: { + bar: { type: "object", properties: { str: { type: "string" } } }, + }, + }, + expectedDatatypes: new Map([ + ["Foo", { definitions: [{ name: "bar", type: "Foo.bar", isComplex: true }] }], + ["Foo.bar", { definitions: [{ name: "str", type: "string" }] }], + ]), + value: { bar: { str: "str" } }, + expectedValue: { bar: { str: "str" } }, + }, + + { + name: "Foo", + schema: { + type: "object", + properties: { + bin: { type: "string", contentEncoding: "base64" }, + nested: { + type: "object", + properties: { + bin2: { type: "string", contentEncoding: "base64" }, + bar: { + type: "array", + items: { + type: "object", + properties: { bin3: { type: "string", contentEncoding: "base64" } }, + }, + }, + }, + }, + }, + }, + expectedDatatypes: new Map([ + [ + "Foo", + { + definitions: [ + { name: "bin", type: "uint8", isArray: true }, + { name: "nested", type: "Foo.nested", isComplex: true }, + ], + }, + ], + [ + "Foo.nested", + { + definitions: [ + { name: "bin2", type: "uint8", isArray: true }, + { name: "bar", type: "Foo.nested.bar", isComplex: true, isArray: true }, + ], + }, + ], + ["Foo.nested.bar", { definitions: [{ name: "bin3", type: "uint8", isArray: true }] }], + ]), + value: { + bin: protobufjs.util.base64.encode(new Uint8Array([0xa1, 0xb2, 0xc3]), 0, 3), + nested: { + bin2: protobufjs.util.base64.encode(new Uint8Array([0xd4, 0xe5, 0xf6]), 0, 3), + bar: [ + { bin3: protobufjs.util.base64.encode(new Uint8Array([0, 1, 0xfe, 0xff]), 0, 4) }, + { bin3: protobufjs.util.base64.encode(new Uint8Array([2, 3, 0xfe, 0xff]), 0, 4) }, + ], + }, + }, + expectedValue: { + bin: new Uint8Array([0xa1, 0xb2, 0xc3]), + nested: { + bin2: new Uint8Array([0xd4, 0xe5, 0xf6]), + bar: [ + { bin3: new Uint8Array([0, 1, 0xfe, 0xff]) }, + { bin3: new Uint8Array([2, 3, 0xfe, 0xff]) }, + ], + }, + }, + }, + ])( + "converts schema to datatypes and decodes base64", + ({ name, schema, expectedDatatypes, value, expectedValue }) => { + const { datatypes, postprocessValue } = parseJsonSchema(schema, name); + expect(datatypes).toEqual(expectedDatatypes); + expect(postprocessValue(value)).toEqual(expectedValue); + }, + ); + + it("allows missing sub-properties", () => { + const { postprocessValue } = parseJsonSchema( + { + type: "object", + properties: { + foo: { type: "number" }, + bar: { + type: "object", + properties: { baz: { type: "object", properties: { quux: { type: "number" } } } }, + }, + }, + }, + "Root", + ); + expect(postprocessValue({ foo: 3 })).toEqual({ foo: 3 }); + }); + it("allows missing sub-properties in arrays", () => { + const { postprocessValue } = parseJsonSchema( + { + type: "object", + properties: { + arr: { + type: "array", + items: { + type: "object", + properties: { + foo: { type: "number" }, + bar: { + type: "object", + properties: { baz: { type: "object", properties: { quux: { type: "number" } } } }, + }, + }, + }, + }, + }, + }, + "Root", + ); + expect(postprocessValue({ arr: [{ foo: 3 }] })).toEqual({ arr: [{ foo: 3 }] }); + }); + + it("converts oneOf to enum", () => { + const { datatypes } = parseJsonSchema( + { + type: "object", + properties: { + level: { + oneOf: [ + { title: "X", const: 1 }, + { title: "Y", const: 2 }, + ], + }, + }, + }, + "Log", + ); + expect(datatypes).toEqual( + new Map([ + [ + "Log", + { + definitions: [ + { name: "X", type: "uint32", isConstant: true, value: 1 }, + { name: "Y", type: "uint32", isConstant: true, value: 2 }, + { name: "level", type: "uint32" }, + ], + }, + ], + ]), + ); + }); + + it.each(Object.values(foxgloveMessageSchemas))("handles Foxglove schema '$name'", (schema) => { + expect(() => parseJsonSchema(generateJsonSchema(schema), schema.name)).not.toThrow(); + }); +}); diff --git a/typescript/support/src/parseJsonSchema.ts b/typescript/support/src/parseJsonSchema.ts new file mode 100644 index 0000000000..ca4215bae8 --- /dev/null +++ b/typescript/support/src/parseJsonSchema.ts @@ -0,0 +1,182 @@ +/* eslint @typescript-eslint/no-unsafe-member-access: 0 */ +/* eslint @typescript-eslint/no-unsafe-assignment: 0 */ + +import { MessageDefinitionField } from "@foxglove/message-definition"; +import * as base64 from "@protobufjs/base64"; + +import { MessageDefinitionMap } from "./types"; + +export function parseJsonSchema( + rootJsonSchema: Record, + rootTypeName: string, +): { + datatypes: MessageDefinitionMap; + + /** + * A function that should be called after parsing a value from a JSON string to do any necessary + * post-processing (e.g. base64 decoding) + */ + postprocessValue: (value: Record) => unknown; +} { + const datatypes: MessageDefinitionMap = new Map(); + + function addFieldsRecursive( + schema: Record, + typeName: string, + keyPath: string[], + ): (value: Record) => unknown { + let postprocessObject: (value: Record) => unknown = (value) => value; + const fields: MessageDefinitionField[] = []; + if (schema.type !== "object") { + throw new Error( + `Expected "type": "object" for schema ${typeName}, got ${JSON.stringify(schema.type)}`, + ); + } + for (const [fieldName, fieldSchema] of Object.entries( + schema.properties as Record>, + )) { + if (Array.isArray(fieldSchema.oneOf)) { + if (fieldSchema.oneOf.every((alternative) => typeof alternative.const === "number")) { + for (const alternative of fieldSchema.oneOf) { + fields.push({ + name: alternative.title, + type: "uint32", + isConstant: true, + value: alternative.const, + }); + } + fields.push({ name: fieldName, type: "uint32" }); + continue; + } else { + throw new Error( + `Unsupported type for ${keyPath + .concat(fieldName) + .join(".")}: oneOf alternatives must have number values`, + ); + } + } + switch (fieldSchema.type) { + case "boolean": + fields.push({ name: fieldName, type: "bool" }); + break; + case "string": + switch (fieldSchema.contentEncoding) { + case undefined: + fields.push({ name: fieldName, type: "string" }); + break; + case "base64": { + fields.push({ name: fieldName, type: "uint8", isArray: true }); + const prevPostprocess = postprocessObject; + postprocessObject = (value) => { + const str = value[fieldName]; + if (typeof str === "string") { + const decoded = new Uint8Array(base64.length(str)); + if (base64.decode(str, decoded, 0) !== decoded.byteLength) { + throw new Error( + `Failed to decode base64 data for ${keyPath.concat(fieldName).join(".")}`, + ); + } + value[fieldName] = decoded; + } + return prevPostprocess(value); + }; + break; + } + default: + throw new Error( + `Unsupported contentEncoding ${JSON.stringify( + fieldSchema.contentEncoding, + )} in ${keyPath.concat(fieldName).join(".")}`, + ); + } + break; + case "number": + case "integer": + fields.push({ name: fieldName, type: "float64" }); + break; + case "object": { + const nestedTypeName = `${typeName}.${fieldName}`; + const postprocessNestedObject = addFieldsRecursive( + fieldSchema, + nestedTypeName, + keyPath.concat(fieldName), + ); + const prevPostprocess = postprocessObject; + postprocessObject = (value) => { + const fieldValue = value[fieldName]; + if (fieldValue != undefined && typeof fieldValue === "object") { + value[fieldName] = postprocessNestedObject(fieldValue as Record); + } + return prevPostprocess(value); + }; + fields.push({ name: fieldName, type: nestedTypeName, isComplex: true }); + break; + } + case "array": { + const itemSchema = fieldSchema.items as Record; + switch (itemSchema.type) { + case "boolean": + fields.push({ name: fieldName, type: "bool", isArray: true }); + break; + case "string": + if (itemSchema.contentEncoding != undefined) { + throw new Error( + `Unsupported contentEncoding ${JSON.stringify( + itemSchema.contentEncoding, + )} for array item ${keyPath.concat(fieldName).join(".")}`, + ); + } + fields.push({ name: fieldName, type: "string", isArray: true }); + break; + case "number": + case "integer": + fields.push({ name: fieldName, type: "float64", isArray: true }); + break; + case "object": { + const nestedTypeName = `${typeName}.${fieldName}`; + const postprocessArrayItem = addFieldsRecursive( + fieldSchema.items as Record, + nestedTypeName, + keyPath.concat(fieldName), + ); + const prevPostprocess = postprocessObject; + postprocessObject = (value) => { + const arr = value[fieldName]; + if (Array.isArray(arr)) { + value[fieldName] = arr.map(postprocessArrayItem); + } + return prevPostprocess(value); + }; + fields.push({ + name: fieldName, + type: nestedTypeName, + isComplex: true, + isArray: true, + }); + break; + } + default: + throw new Error( + `Unsupported type ${JSON.stringify(itemSchema.type)} for array item ${keyPath + .concat(fieldName) + .join(".")}`, + ); + } + break; + } + case "null": + default: + throw new Error( + `Unsupported type ${JSON.stringify(fieldSchema.type)} for ${keyPath + .concat(fieldName) + .join(".")}`, + ); + } + } + datatypes.set(typeName, { definitions: fields }); + return postprocessObject; + } + + const postprocessValue = addFieldsRecursive(rootJsonSchema, rootTypeName, []); + return { datatypes, postprocessValue }; +} diff --git a/typescript/support/src/protobufDefinitionsToDatatypes.ts b/typescript/support/src/protobufDefinitionsToDatatypes.ts new file mode 100644 index 0000000000..2265f291e1 --- /dev/null +++ b/typescript/support/src/protobufDefinitionsToDatatypes.ts @@ -0,0 +1,81 @@ +import { MessageDefinitionField } from "@foxglove/message-definition"; +import protobufjs from "protobufjs"; + +import { MessageDefinitionMap } from "./types"; + +function protobufScalarToRosPrimitive(type: string): string { + switch (type) { + case "double": + return "float64"; + case "float": + return "float32"; + case "int32": + case "sint32": + case "sfixed32": + return "int32"; + case "uint32": + case "fixed32": + return "uint32"; + case "int64": + case "sint64": + case "sfixed64": + return "int64"; + case "uint64": + case "fixed64": + return "uint64"; + case "bool": + return "bool"; + case "string": + return "string"; + } + throw new Error(`Expected protobuf scalar type, got ${type}`); +} + +export function stripLeadingDot(typeName: string): string { + return typeName.replace(/^\./, ""); +} + +export function protobufDefinitionsToDatatypes( + datatypes: MessageDefinitionMap, + type: protobufjs.Type, +): void { + const definitions: MessageDefinitionField[] = []; + // The empty list reference is added to the map so a `.has` lookup below can prevent infinite recursion on cyclical types + datatypes.set(stripLeadingDot(type.fullName), { definitions }); + for (const field of type.fieldsArray) { + if (field.resolvedType instanceof protobufjs.Enum) { + for (const [name, value] of Object.entries(field.resolvedType.values)) { + // Note: names from different enums might conflict. The player API will need to be updated + // to associate fields with enums (similar to the __foxglove_enum annotation hack). + // https://github.com/foxglove/studio/issues/2214 + definitions.push({ name, type: "int32", isConstant: true, value }); + } + definitions.push({ type: "int32", name: field.name }); + } else if (field.resolvedType) { + const fullName = stripLeadingDot(field.resolvedType.fullName); + definitions.push({ + type: fullName, + name: field.name, + isComplex: true, + isArray: field.repeated, + }); + + // If we've already processed this datatype we should skip it. + // This avoid infinite recursion with datatypes that reference themselves. + if (!datatypes.has(fullName)) { + protobufDefinitionsToDatatypes(datatypes, field.resolvedType); + } + } else if (field.type === "bytes") { + if (field.repeated) { + throw new Error("Repeated bytes are not currently supported"); + } + definitions.push({ type: "uint8", name: field.name, isArray: true }); + } else { + definitions.push({ + type: protobufScalarToRosPrimitive(field.type), + name: field.name, + isArray: field.repeated, + }); + } + } +} diff --git a/typescript/support/src/protobufDescriptors.ts b/typescript/support/src/protobufDescriptors.ts new file mode 100644 index 0000000000..fb572a7a5b --- /dev/null +++ b/typescript/support/src/protobufDescriptors.ts @@ -0,0 +1,19 @@ +// eslint-disable-next-line @typescript-eslint/triple-slash-reference +/// + +import protobufjs from "protobufjs"; +import { FileDescriptorSet } from "protobufjs/ext/descriptor"; + +export type ProtobufDescriptor = ReturnType; + +export function protobufToDescriptor(root: protobufjs.Root): ProtobufDescriptor { + return root.toDescriptor("proto3"); +} + +export function protobufFromDescriptor(descriptorSet: protobufjs.Message): protobufjs.Root { + return protobufjs.Root.fromDescriptor(descriptorSet); +} + +export function protobufFromBinaryDescriptor(schemaData: Uint8Array): protobufjs.Root { + return protobufFromDescriptor(FileDescriptorSet.decode(schemaData)); +} diff --git a/typescript/support/src/types.ts b/typescript/support/src/types.ts new file mode 100644 index 0000000000..8067d641bf --- /dev/null +++ b/typescript/support/src/types.ts @@ -0,0 +1,8 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/ + +import { MessageDefinition } from "@foxglove/message-definition"; + +/** A map of schema name to the schema message definition */ +export type MessageDefinitionMap = Map; diff --git a/typescript/support/tsconfig.cjs.json b/typescript/support/tsconfig.cjs.json new file mode 100644 index 0000000000..ac31cb3343 --- /dev/null +++ b/typescript/support/tsconfig.cjs.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist/cjs", + "module": "commonjs" + } +} diff --git a/typescript/support/tsconfig.json b/typescript/support/tsconfig.json new file mode 100644 index 0000000000..69d8b14bed --- /dev/null +++ b/typescript/support/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "@foxglove/tsconfig/base", + "include": ["./src/**/*"], + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist/esm", + "lib": ["es2020", "dom"] + } +} diff --git a/typescript/examples/bag2mcap/typings/protobufjs.d.ts b/typescript/support/typings/protobufjs.d.ts similarity index 100% rename from typescript/examples/bag2mcap/typings/protobufjs.d.ts rename to typescript/support/typings/protobufjs.d.ts diff --git a/yarn.lock b/yarn.lock index 4c5b5264df..403ca2954d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -44,12 +44,12 @@ "@arrows/error" "^1.0.2" fast-deep-equal "^3.1.3" -"@babel/code-frame@^7.0.0": - version "7.16.7" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.16.7.tgz#44416b6bd7624b998f5b1af5d470856c40138789" - integrity sha512-iAXqUn8IIeBTNd72xsFlgaXHkMBMt6y4HJp1tIaK465CWLT/fG1aqB7ykr95gHHmlBdGbFeWWfyB4NJJ0nmeIg== +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.18.6": + version "7.18.6" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.18.6.tgz#3b25d38c89600baa2dcc219edfa88a74eb2c427a" + integrity sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q== dependencies: - "@babel/highlight" "^7.16.7" + "@babel/highlight" "^7.18.6" "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.16.0": version "7.16.0" @@ -58,13 +58,6 @@ dependencies: "@babel/highlight" "^7.16.0" -"@babel/code-frame@^7.18.6": - version "7.18.6" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.18.6.tgz#3b25d38c89600baa2dcc219edfa88a74eb2c427a" - integrity sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q== - dependencies: - "@babel/highlight" "^7.18.6" - "@babel/compat-data@^7.16.0": version "7.16.4" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.16.4.tgz#081d6bbc336ec5c2435c6346b2ae1fb98b5ac68e" @@ -313,11 +306,6 @@ resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.15.7.tgz#220df993bfe904a4a6b02ab4f3385a5ebf6e2389" integrity sha512-K4JvCtQqad9OY2+yTU8w+E82ywk/fe+ELNlt1G8z3bVGlZfn/hOcQQsUhGhW/N+tb3fxK800wLtKOE/aM0m72w== -"@babel/helper-validator-identifier@^7.16.7": - version "7.16.7" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.16.7.tgz#e8c602438c4a8195751243da9031d1607d247cad" - integrity sha512-hsEnFemeiW4D08A5gUAZxLBTXpZ39P+a+DGDsHw1yxqyQ/jzFEnxf5uTEGp+3bzAbNOxU1paTgYS4ECU/IgfDw== - "@babel/helper-validator-identifier@^7.18.6", "@babel/helper-validator-identifier@^7.19.1": version "7.19.1" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz#7eea834cf32901ffdc1a7ee555e2f9c27e249ca2" @@ -360,15 +348,6 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/highlight@^7.16.7": - version "7.16.10" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.16.10.tgz#744f2eb81579d6eea753c227b0f570ad785aba88" - integrity sha512-5FnTQLSLswEj6IkgVw5KusNUUFY9ZGqe/TRFnP/BKYHYgfh7tc+C7mwiy95/yNP7Dh9x580Vv8r7u7ZfTBFxdw== - dependencies: - "@babel/helper-validator-identifier" "^7.16.7" - chalk "^2.0.0" - js-tokens "^4.0.0" - "@babel/highlight@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.18.6.tgz#81158601e93e2563795adcbfbdf5d64be3f2ecdf" @@ -557,72 +536,72 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@cspell/cspell-bundled-dicts@6.26.3": - version "6.26.3" - resolved "https://registry.yarnpkg.com/@cspell/cspell-bundled-dicts/-/cspell-bundled-dicts-6.26.3.tgz#4cf7f9e228c26e2a0f6a4ae82221a6513a0503ab" - integrity sha512-ZOQI5XSJiLJi9GEbdjKJvMDbgzevsmoQzvAHZ2ujwzoWfhxCeEET0+6fs88/5QvHgXwl0CDsFspXZr1OFfZLHA== +"@cspell/cspell-bundled-dicts@6.27.0": + version "6.27.0" + resolved "https://registry.yarnpkg.com/@cspell/cspell-bundled-dicts/-/cspell-bundled-dicts-6.27.0.tgz#64c0a89bc37630eeba83bdec96a904fef32840b9" + integrity sha512-udRkEUz0QIAecCUECVac8IBCu7f5siCrxUZmsoZINgqLlSh0ppmn4/CiiA+sefj99rOq/vB4VEEQCnMShfl5qw== dependencies: "@cspell/dict-ada" "^4.0.1" "@cspell/dict-aws" "^3.0.0" "@cspell/dict-bash" "^4.1.1" - "@cspell/dict-companies" "^3.0.6" - "@cspell/dict-cpp" "^4.0.2" + "@cspell/dict-companies" "^3.0.8" + "@cspell/dict-cpp" "^4.0.3" "@cspell/dict-cryptocurrencies" "^3.0.1" "@cspell/dict-csharp" "^4.0.2" - "@cspell/dict-css" "^4.0.3" - "@cspell/dict-dart" "^2.0.1" - "@cspell/dict-django" "^4.0.1" - "@cspell/dict-docker" "^1.1.5" - "@cspell/dict-dotnet" "^4.0.1" - "@cspell/dict-elixir" "^4.0.1" + "@cspell/dict-css" "^4.0.5" + "@cspell/dict-dart" "^2.0.2" + "@cspell/dict-django" "^4.0.2" + "@cspell/dict-docker" "^1.1.6" + "@cspell/dict-dotnet" "^4.0.2" + "@cspell/dict-elixir" "^4.0.2" "@cspell/dict-en-common-misspellings" "^1.0.2" "@cspell/dict-en-gb" "1.1.33" - "@cspell/dict-en_us" "^4.2.2" + "@cspell/dict-en_us" "^4.3.0" "@cspell/dict-filetypes" "^3.0.0" - "@cspell/dict-fonts" "^3.0.0" - "@cspell/dict-fullstack" "^3.1.1" + "@cspell/dict-fonts" "^3.0.1" + "@cspell/dict-fullstack" "^3.1.4" "@cspell/dict-gaming-terms" "^1.0.4" "@cspell/dict-git" "^2.0.0" - "@cspell/dict-golang" "^5.0.1" + "@cspell/dict-golang" "^5.0.2" "@cspell/dict-haskell" "^4.0.1" - "@cspell/dict-html" "^4.0.2" + "@cspell/dict-html" "^4.0.3" "@cspell/dict-html-symbol-entities" "^4.0.0" - "@cspell/dict-java" "^5.0.4" - "@cspell/dict-k8s" "^1.0.0" + "@cspell/dict-java" "^5.0.5" + "@cspell/dict-k8s" "^1.0.1" "@cspell/dict-latex" "^3.1.0" "@cspell/dict-lorem-ipsum" "^3.0.0" - "@cspell/dict-lua" "^4.0.0" + "@cspell/dict-lua" "^4.0.1" "@cspell/dict-node" "^4.0.2" - "@cspell/dict-npm" "^5.0.3" + "@cspell/dict-npm" "^5.0.5" "@cspell/dict-php" "^3.0.4" - "@cspell/dict-powershell" "^4.0.0" + "@cspell/dict-powershell" "^4.0.2" "@cspell/dict-public-licenses" "^2.0.1" "@cspell/dict-python" "^4.0.1" "@cspell/dict-r" "^2.0.1" - "@cspell/dict-ruby" "^4.0.1" - "@cspell/dict-rust" "^4.0.0" - "@cspell/dict-scala" "^4.0.0" - "@cspell/dict-software-terms" "^3.1.3" - "@cspell/dict-sql" "^2.0.1" + "@cspell/dict-ruby" "^4.0.2" + "@cspell/dict-rust" "^4.0.1" + "@cspell/dict-scala" "^4.0.1" + "@cspell/dict-software-terms" "^3.1.5" + "@cspell/dict-sql" "^2.0.2" "@cspell/dict-svelte" "^1.0.2" "@cspell/dict-swift" "^2.0.1" - "@cspell/dict-typescript" "^3.1.0" + "@cspell/dict-typescript" "^3.1.1" "@cspell/dict-vue" "^3.0.0" -"@cspell/cspell-pipe@6.26.3": - version "6.26.3" - resolved "https://registry.yarnpkg.com/@cspell/cspell-pipe/-/cspell-pipe-6.26.3.tgz#347fe00773e88189cc429e9aa3e2149f3806f43b" - integrity sha512-e4LKHgXnYj8lO2qFaPaGUjgS2Vps464sc8lRt65MJ3iHR3/AzQO1mB+MDLCqItaLmcyA/llrEY1D8m9dGiBFxA== +"@cspell/cspell-pipe@6.27.0": + version "6.27.0" + resolved "https://registry.yarnpkg.com/@cspell/cspell-pipe/-/cspell-pipe-6.27.0.tgz#a5e6cd961a7c694b17dee27dbf3a438eb6a29107" + integrity sha512-lS3kIew5+ETExORdPcJkEA03ylrEL9CHaJ1xE7SSezU3mqoUgbVW5/f2+PCIWPu1mVvUpWef2X1ODddiPPp9NQ== -"@cspell/cspell-service-bus@6.26.3": - version "6.26.3" - resolved "https://registry.yarnpkg.com/@cspell/cspell-service-bus/-/cspell-service-bus-6.26.3.tgz#b3f1027295a118516caa5517d4e35d97898290ba" - integrity sha512-dbhsB8d4dEd8adyA+/KpNYERyOt8y3VSvOdgusjweEKjezCNxIwLR3GFQHi4QWCevDzrqS+mt9hAvO5RlYP7Bg== +"@cspell/cspell-service-bus@6.27.0": + version "6.27.0" + resolved "https://registry.yarnpkg.com/@cspell/cspell-service-bus/-/cspell-service-bus-6.27.0.tgz#e83457e3682588db25b441b6f78f80186ee5dacc" + integrity sha512-GC3WseMeBE1xtIftBgfTOQxgAMHmLF5HhjHo2rVIfCKv3uyQXuNsGYmtqY7dm4GDrBx1vIYjOoT2xtEaSLoUPw== -"@cspell/cspell-types@6.26.3": - version "6.26.3" - resolved "https://registry.yarnpkg.com/@cspell/cspell-types/-/cspell-types-6.26.3.tgz#559b6e0fc5c43905472acffb2e2dddff2435b892" - integrity sha512-s5SjHbpCP/MBTCCwgADzmZvsxpygIiH/2JytVUBrk8TWr4U8/EE3gXPdJ8EUAW3Ndgls/OpGn9c31F6sFjsLjg== +"@cspell/cspell-types@6.27.0": + version "6.27.0" + resolved "https://registry.yarnpkg.com/@cspell/cspell-types/-/cspell-types-6.27.0.tgz#ccf6b1ca03df3760f8838da844057c4f86f811ea" + integrity sha512-eHKpAhIUYIjssqF20IrihzNAPDTWJsFpBzg7UznX03Z4Y77JrGAgdRZ6h69uaIafQb7uotNYWnDazpVSkMpikQ== "@cspell/dict-ada@^4.0.1": version "4.0.1" @@ -639,15 +618,15 @@ resolved "https://registry.yarnpkg.com/@cspell/dict-bash/-/dict-bash-4.1.1.tgz#fe28016096f44d4a09fe4c5bcaf6fa40f33d98c6" integrity sha512-8czAa/Mh96wu2xr0RXQEGMTBUGkTvYn/Pb0o+gqOO1YW+poXGQc3gx0YPqILDryP/KCERrNvkWUJz3iGbvwC2A== -"@cspell/dict-companies@^3.0.6": - version "3.0.6" - resolved "https://registry.yarnpkg.com/@cspell/dict-companies/-/dict-companies-3.0.6.tgz#59d016afcdb27a4a88749a56475bf790ec175dc0" - integrity sha512-6rWuwZxPisn/MP41DzBtChVgbz9b6HSjBH3X0s3k7zlBaxrw6xFAZGKH9KGFSPTiV+WD9j+IIn2/ITXERGjNLA== +"@cspell/dict-companies@^3.0.8": + version "3.0.8" + resolved "https://registry.yarnpkg.com/@cspell/dict-companies/-/dict-companies-3.0.8.tgz#3fcbdb0dc9026195abd47a14135338f5547f17d0" + integrity sha512-tQPpkxgog+7xGN3dA9p2Hd4O95+hFYfJuHeY9GgxNahBQyq3bv0REAc6xlqdtkIpfV2ga93B0l37mQr1p107Iw== -"@cspell/dict-cpp@^4.0.2": - version "4.0.2" - resolved "https://registry.yarnpkg.com/@cspell/dict-cpp/-/dict-cpp-4.0.2.tgz#ca41ccba8cb7399abae2c7a7ba0c2dd432b9d23a" - integrity sha512-Wi/ZUPoGaqvxeyaAdMtXFtfc3mZ4bw5nKYZrVHCsMzyGTC6IXL9Xp6KbwU0zEJXyNvvmRP5zd+Q4RnMf4abgCg== +"@cspell/dict-cpp@^4.0.3": + version "4.0.3" + resolved "https://registry.yarnpkg.com/@cspell/dict-cpp/-/dict-cpp-4.0.3.tgz#5479e0a6477af9b1f6d0930e3f2f6025daf62cac" + integrity sha512-gbXY9cUgRpb5mpw19VBy+YNUqNMlT5Dj70d8V1yIFbqPVHxccmxwdU4rlNaRyYrC41kDZwxmG7QQwcng6FdGcg== "@cspell/dict-cryptocurrencies@^3.0.1": version "3.0.1" @@ -659,35 +638,35 @@ resolved "https://registry.yarnpkg.com/@cspell/dict-csharp/-/dict-csharp-4.0.2.tgz#e55659dbe594e744d86b1baf0f3397fe57b1e283" integrity sha512-1JMofhLK+4p4KairF75D3A924m5ERMgd1GvzhwK2geuYgd2ZKuGW72gvXpIV7aGf52E3Uu1kDXxxGAiZ5uVG7g== -"@cspell/dict-css@^4.0.3": - version "4.0.3" - resolved "https://registry.yarnpkg.com/@cspell/dict-css/-/dict-css-4.0.3.tgz#186c1f841b1ff882bb6b51599644a6e0882f15f9" - integrity sha512-sLhutH5hlhPGuOtObR3Q0qezywuwREIcyTeERQw14kizYA3jA/J8YxSPcX2r9TsNFPHY85NAMyrH6Q38iyBGbw== +"@cspell/dict-css@^4.0.5": + version "4.0.5" + resolved "https://registry.yarnpkg.com/@cspell/dict-css/-/dict-css-4.0.5.tgz#2233138a03c163f82b0f6fbe0cdd2aada3ca4afc" + integrity sha512-z5vw8nJSyKd6d3i5UmMNoVcAp0wxvs9OHWOmAeJKT9fO3tok02gK24VZhcJ0NJtiKdHQ2zRuzdfWl51wdAiY6A== -"@cspell/dict-dart@^2.0.1": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@cspell/dict-dart/-/dict-dart-2.0.1.tgz#7edec0250f814eff7bf4a40b6fb4d2b4ce41af8c" - integrity sha512-YRuDX9k2qPSWDEsM26j8o7KMvaZ0DXc74ijK/VRwaksm1CBRPBW289pe2TE2K7y4SJjTKXgQ9urOVlozeQDpuA== +"@cspell/dict-dart@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@cspell/dict-dart/-/dict-dart-2.0.2.tgz#714285f4f8bd304c1c477779ccbbfae5949819d7" + integrity sha512-jigcODm7Z4IFZ4vParwwP3IT0fIgRq/9VoxkXfrxBMsLBGGM2QltHBj7pl+joX+c4cOHxfyZktGJK1B1wFtR4Q== -"@cspell/dict-django@^4.0.1": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@cspell/dict-django/-/dict-django-4.0.1.tgz#97009b98bb1bef56a022416b08da8f523d1a4ab6" - integrity sha512-q3l7OH39qzeN2Y64jpY39SEAqki5BUzPTypnhzM40yT+LOGSWqSh9Ix5UecejtXPDVrD8vML+m7Bp5070h52HQ== +"@cspell/dict-django@^4.0.2": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@cspell/dict-django/-/dict-django-4.0.2.tgz#08d21ee3ce7e323e4d7634abf6d69a96a6d4930c" + integrity sha512-L0Yw6+Yh2bE9/FAMG4gy9m752G4V8HEBjEAGeRIQ9qvxDLR9yD6dPOtgEFTjv7SWlKSrLb9wA/W3Q2GKCOusSg== -"@cspell/dict-docker@^1.1.5": - version "1.1.5" - resolved "https://registry.yarnpkg.com/@cspell/dict-docker/-/dict-docker-1.1.5.tgz#925a0c5c3c888038acfc0785e363d32aee05d524" - integrity sha512-SNEohOScQ+0+y9dp/jKTx60OOJQrf5es5BJ32gh5Ck3jKXNo4wd9KLgPOmQMUpencb5SGjrBsC4rr1fyfCwytg== +"@cspell/dict-docker@^1.1.6": + version "1.1.6" + resolved "https://registry.yarnpkg.com/@cspell/dict-docker/-/dict-docker-1.1.6.tgz#f84faed121e2093e3b212d19542fd27eda751c80" + integrity sha512-zCCiRTZ6EOQpBnSOm0/3rnKW1kCcAUDUA7SxJG3SuH6iZvKi3I8FEg8+O83WQUeXg0SyPNerD9F40JLnnJjJig== -"@cspell/dict-dotnet@^4.0.1": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@cspell/dict-dotnet/-/dict-dotnet-4.0.1.tgz#41fa41c75c157d0097b35767fa92234e381549b6" - integrity sha512-l11TqlUX8cDgsE/1Zrea1PqLn63s20MY3jKWMbQVB5DMDPDO2f8Pukckkwxq5p/cxDABEjuGzfF1kTX3pAakBw== +"@cspell/dict-dotnet@^4.0.2": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@cspell/dict-dotnet/-/dict-dotnet-4.0.2.tgz#f066040c88efc3a4419fb8844427430d44f970b3" + integrity sha512-Cu+Ob142tBQ2cYrpK/d3tjm/FvNXQXwdUShRIPKx03HbtUk9BoTdeFY5bX+Zz7GeV66OJCMrmpFANrtKpB8NTg== -"@cspell/dict-elixir@^4.0.1": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@cspell/dict-elixir/-/dict-elixir-4.0.1.tgz#dca91412eb0026cb76cc56f9677602558f6caa78" - integrity sha512-IejBqiTTWSXpvBm6yg4qUfnJR0LwbUUCJcK5wXOMKEJitu3yDfrT9GPc6NQJXgokbg9nBjEyxVIzNcLgx2x3/Q== +"@cspell/dict-elixir@^4.0.2": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@cspell/dict-elixir/-/dict-elixir-4.0.2.tgz#1a37e92b45d744e1b78714c64811ca3dbc600a5c" + integrity sha512-/YeHlpZ1pE9VAyxp3V0xyUPapNyC61WwFuw2RByeoMqqYaIfS3Hw+JxtimOsAKVhUvgUH58zyKl5K5Q6FqgCpw== "@cspell/dict-en-common-misspellings@^1.0.2": version "1.0.2" @@ -699,25 +678,25 @@ resolved "https://registry.yarnpkg.com/@cspell/dict-en-gb/-/dict-en-gb-1.1.33.tgz#7f1fd90fc364a5cb77111b5438fc9fcf9cc6da0e" integrity sha512-tKSSUf9BJEV+GJQAYGw5e+ouhEe2ZXE620S7BLKe3ZmpnjlNG9JqlnaBhkIMxKnNFkLY2BP/EARzw31AZnOv4g== -"@cspell/dict-en_us@^4.2.2": - version "4.2.2" - resolved "https://registry.yarnpkg.com/@cspell/dict-en_us/-/dict-en_us-4.2.2.tgz#3500bfea44b64a7c7ac4b24d4a695f4fa3fce668" - integrity sha512-NSlvE6bIkgRRlBkfltiwREu2NYT4PrLmpdi9zSeWuUMlGB+0wUGAal3B7zKC1pirhueH20W6to0lPdnEWaqa8Q== +"@cspell/dict-en_us@^4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@cspell/dict-en_us/-/dict-en_us-4.3.0.tgz#84476374edeabc46d41d8b7e117ce2f341ae8de8" + integrity sha512-lQ4r8tBiylNjmYwrWz4xUgBtVC0CciKpddMUVosdusHonFE0KjlvkZK6PFtROBupmRLMBBMjxvtpbq8SdFBqCw== "@cspell/dict-filetypes@^3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@cspell/dict-filetypes/-/dict-filetypes-3.0.0.tgz#3bb1ede3e28449f0d76024a7b918a556f210973a" integrity sha512-Fiyp0z5uWaK0d2TfR9GMUGDKmUMAsOhGD5A0kHoqnNGswL2iw0KB0mFBONEquxU65fEnQv4R+jdM2d9oucujuA== -"@cspell/dict-fonts@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-fonts/-/dict-fonts-3.0.0.tgz#af2755305fbd62fb55a8515989a29f6e58aff9c9" - integrity sha512-zTZni0AbwBVG1MKA0WpwPyIJPVF+gp6neXDQzHcu4RUnuQ4uDu0PVEuZjGHCJWwwFoR5JmkqZxVSg1y3ufJODA== +"@cspell/dict-fonts@^3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@cspell/dict-fonts/-/dict-fonts-3.0.1.tgz#0e0b875d463a9bd65e78145c9b6649ecad017df5" + integrity sha512-o2zVFKT3KcIBo88xlWhG4yOD0XQDjP7guc7C30ZZcSN8YCwaNc1nGoxU3QRea8iKcwk3cXH0G53nrQur7g9DjQ== -"@cspell/dict-fullstack@^3.1.1": - version "3.1.2" - resolved "https://registry.yarnpkg.com/@cspell/dict-fullstack/-/dict-fullstack-3.1.2.tgz#6b9867779a527f000c7e97837e391699f7606998" - integrity sha512-Xw9DkEGGxI/6Wce1sThnuiO2JDjRxMQle1EOxVPf7mpsLjVHermKhSysK70ujxB2jc93rhY9ar2+5xlizapxig== +"@cspell/dict-fullstack@^3.1.4": + version "3.1.4" + resolved "https://registry.yarnpkg.com/@cspell/dict-fullstack/-/dict-fullstack-3.1.4.tgz#930a66a1397f463c807e54dd01b0c79ec3f7fc21" + integrity sha512-OnCIn3GgAhdhsU6xMYes7/WXnbV6R/5k/zRAu/d+WZP4Ltf48z7oFfNFjHXH6b8ZwnMhpekLAnCeIfT5dcxRqw== "@cspell/dict-gaming-terms@^1.0.4": version "1.0.4" @@ -729,10 +708,10 @@ resolved "https://registry.yarnpkg.com/@cspell/dict-git/-/dict-git-2.0.0.tgz#fa5cb298845da9c69efc01c6af07a99097718dc9" integrity sha512-n1AxyX5Kgxij/sZFkxFJlzn3K9y/sCcgVPg/vz4WNJ4K9YeTsUmyGLA2OQI7d10GJeiuAo2AP1iZf2A8j9aj2w== -"@cspell/dict-golang@^5.0.1": - version "5.0.1" - resolved "https://registry.yarnpkg.com/@cspell/dict-golang/-/dict-golang-5.0.1.tgz#1da6d02500fea7d404f087147d962a7e92deb7ec" - integrity sha512-djsJC7OVKUpFdRm/aqBJEUSGP3kw/MDhAt7udYegnyQt2WjL3ZnVoG7r5eOEhPEEKzWVBYoi6UKSNpdQEodlbg== +"@cspell/dict-golang@^5.0.2": + version "5.0.2" + resolved "https://registry.yarnpkg.com/@cspell/dict-golang/-/dict-golang-5.0.2.tgz#3c2c36fbe84f8b83424e06809f0dff76bdf07aaa" + integrity sha512-TNOQzsiLv4I56w5188OnJW+2ttjekoBl8IyPpI25GeV3dky4d+TX5pujayvcKQ+SM8vV8u2lpQpvyr4YePhiQg== "@cspell/dict-haskell@^4.0.1": version "4.0.1" @@ -744,20 +723,20 @@ resolved "https://registry.yarnpkg.com/@cspell/dict-html-symbol-entities/-/dict-html-symbol-entities-4.0.0.tgz#4d86ac18a4a11fdb61dfb6f5929acd768a52564f" integrity sha512-HGRu+48ErJjoweR5IbcixxETRewrBb0uxQBd6xFGcxbEYCX8CnQFTAmKI5xNaIt2PKaZiJH3ijodGSqbKdsxhw== -"@cspell/dict-html@^4.0.2": - version "4.0.2" - resolved "https://registry.yarnpkg.com/@cspell/dict-html/-/dict-html-4.0.2.tgz#36bf81aea239fc92e277875f249b3d784e7fdeb7" - integrity sha512-BskOE2K3AtGLkcjdJmo+H6/fjdfDP4XYAsEGXpB26rvdnXAnGEstE/Q8Do6UfJCvgOVYCpdUZLcMIEpoTy7QhQ== +"@cspell/dict-html@^4.0.3": + version "4.0.3" + resolved "https://registry.yarnpkg.com/@cspell/dict-html/-/dict-html-4.0.3.tgz#155450cb57750774583fce463d01d6323ab41701" + integrity sha512-Gae8i8rrArT0UyG1I6DHDK62b7Be6QEcBSIeWOm4VIIW1CASkN9B0qFgSVnkmfvnu1Y3H7SSaaEynKjdj3cs8w== -"@cspell/dict-java@^5.0.4": - version "5.0.4" - resolved "https://registry.yarnpkg.com/@cspell/dict-java/-/dict-java-5.0.4.tgz#a5bf80474bbe10966c13701c0eab751ae959d5e7" - integrity sha512-43VrLOLcBxavv6eyL4BpsnHrhVOgyYYeJqQRJG5XKObcpWy3+Lpadj58CfTVOr7M/Je3pUpd4tvsUhf/lWXMVA== +"@cspell/dict-java@^5.0.5": + version "5.0.5" + resolved "https://registry.yarnpkg.com/@cspell/dict-java/-/dict-java-5.0.5.tgz#c673f27ce7a5d96e205f42e8be540aeda0beef11" + integrity sha512-X19AoJgWIBwJBSWGFqSgHaBR/FEykBHTMjL6EqOnhIGEyE9nvuo32tsSHjXNJ230fQxQptEvRZoaldNLtKxsRg== -"@cspell/dict-k8s@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-k8s/-/dict-k8s-1.0.0.tgz#30353cef5a5310b5a6b8549e0a4fa6faf4956b55" - integrity sha512-XqIql+nd2DiuPuL+qPc24bN/L1mZY75kAYcuMBMW5iYgBoivkiVOg7br/aofX3ApajvHDln6tNkPZhmhsOg6Ww== +"@cspell/dict-k8s@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@cspell/dict-k8s/-/dict-k8s-1.0.1.tgz#6c0cc521dd42fee2c807368ebfef77137686f3a1" + integrity sha512-gc5y4Nm3hVdMZNBZfU2M1AsAmObZsRWjCUk01NFPfGhFBXyVne41T7E62rpnzu5330FV/6b/TnFcPgRmak9lLw== "@cspell/dict-latex@^3.1.0": version "3.1.0" @@ -769,30 +748,30 @@ resolved "https://registry.yarnpkg.com/@cspell/dict-lorem-ipsum/-/dict-lorem-ipsum-3.0.0.tgz#c6347660fcab480b47bdcaec3b57e8c3abc4af68" integrity sha512-msEV24qEpzWZs2kcEicqYlhyBpR0amfDkJOs+iffC07si9ftqtQ+yP3lf1VFLpgqw3SQh1M1vtU7RD4sPrNlcQ== -"@cspell/dict-lua@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-lua/-/dict-lua-4.0.0.tgz#5c56f3543ace8951c72e7ff3982917d120a6cd23" - integrity sha512-aQPyc/nP67tOlW6ACpio9Q5mZ/Z1hqwXC5rt5tkoQ2GsnCqeyIXDrX0CN+RGK53Lj4P02Jz/dPxs/nX8eDUFsw== +"@cspell/dict-lua@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@cspell/dict-lua/-/dict-lua-4.0.1.tgz#4c31975646cb2d71f1216c7aeaa0c5ab6994ea25" + integrity sha512-j0MFmeCouSoC6EdZTbvGe1sJ9V+ruwKSeF+zRkNNNload7R72Co5kX1haW2xLHGdlq0kqSy1ODRZKdVl0e+7hg== "@cspell/dict-node@^4.0.2": version "4.0.2" resolved "https://registry.yarnpkg.com/@cspell/dict-node/-/dict-node-4.0.2.tgz#9e5f64d882568fdd2a2243542d1263dbbb87c53a" integrity sha512-FEQJ4TnMcXEFslqBQkXa5HposMoCGsiBv2ux4IZuIXgadXeHKHUHk60iarWpjhzNzQLyN2GD7NoRMd12bK3Llw== -"@cspell/dict-npm@^5.0.3": - version "5.0.3" - resolved "https://registry.yarnpkg.com/@cspell/dict-npm/-/dict-npm-5.0.3.tgz#f1118168fb20bf1a3a2e02187dfffb90cd15def5" - integrity sha512-fEX67zIJISbS3gXVk/y/ZUvDIVtjc/CYJK7Mz0iTVrmlCKnLiD41lApe8v4g/12eE7hLfx/sfCXDrUWyzXVq1A== +"@cspell/dict-npm@^5.0.5": + version "5.0.5" + resolved "https://registry.yarnpkg.com/@cspell/dict-npm/-/dict-npm-5.0.5.tgz#fa6c1bc983e34ddc6d97094c758a4e166afd6214" + integrity sha512-eirZm4XpJNEcbmLGIwI2qXdRRlCKwEsH9mT3qCUytmbj6S6yn63F+8bShMW/yQBedV7+GXq9Td+cJdqiVutOiA== "@cspell/dict-php@^3.0.4": version "3.0.4" resolved "https://registry.yarnpkg.com/@cspell/dict-php/-/dict-php-3.0.4.tgz#32f25137a41c264886256a5b296e92224b0170b9" integrity sha512-QX6zE/ZfnT3O5lSwV8EPVh8Va39ds34gSNNR8I4GWiuDpKcTkZPFi4OLoP3Tlhbl/3G0Ha35OkSDLvZfu8mnkA== -"@cspell/dict-powershell@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-powershell/-/dict-powershell-4.0.0.tgz#7d89e7afca056734925c2c847c0095f15bd1dd4e" - integrity sha512-1Lbm+3+Sx63atl4MM3lPeCUc90JjRyKP9+exmy2cQimXNju9ngtuDWwapHUnhQ47qnzrsBY4ydm36KCfJarXJA== +"@cspell/dict-powershell@^4.0.2": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@cspell/dict-powershell/-/dict-powershell-4.0.2.tgz#d544e7ad9466b96bae49f6dd145eda61c67098a2" + integrity sha512-3Wk2Z0fxpewML0zq4a9W5IsPZ0YwvzA8c6ykFdwQ0xcBQc/xRfdb9Z5drYXf9bobck1+MacGrprSeQXrmeByNQ== "@cspell/dict-public-licenses@^2.0.1": version "2.0.1" @@ -809,30 +788,30 @@ resolved "https://registry.yarnpkg.com/@cspell/dict-r/-/dict-r-2.0.1.tgz#73474fb7cce45deb9094ebf61083fbf5913f440a" integrity sha512-KCmKaeYMLm2Ip79mlYPc8p+B2uzwBp4KMkzeLd5E6jUlCL93Y5Nvq68wV5fRLDRTf7N1LvofkVFWfDcednFOgA== -"@cspell/dict-ruby@^4.0.1": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@cspell/dict-ruby/-/dict-ruby-4.0.1.tgz#31adc6cb6f0ca91e7a045a4c1279715b67ff1265" - integrity sha512-p9nLDsffPadPLLwdLQj4Gk0IsZ64iCSxnSCaeFXslFiD17FtJVh1YMHP7KE9M73u22Hprq+a1Yw25/xp6Tkt3g== +"@cspell/dict-ruby@^4.0.2": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@cspell/dict-ruby/-/dict-ruby-4.0.2.tgz#f94dd18ce21825756f96b4510e1e6dc8cbe45f84" + integrity sha512-fCoQHvLhTAetzXCUZMpyoCUPFMiyLHuECIPOiuYW6TGnP2eGV9y4j2J8HAOVtkyxOKUoyK+zZgtrma64yTUMkg== -"@cspell/dict-rust@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-rust/-/dict-rust-4.0.0.tgz#23fb18bb6c97c61b6ba382c8a7c76fae9d12fe92" - integrity sha512-nzJsgLR6/JXtM41Cr5FG89r8sBKW6aFjvCqPxeaBJYLAL0JuvsVUcd16rW2lTsdbx5J8yUQDD7mgCZFk6merJQ== +"@cspell/dict-rust@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@cspell/dict-rust/-/dict-rust-4.0.1.tgz#ef0b88cb3a45265824e2c9ce31b0baa4e1050351" + integrity sha512-xJSSzHDK2z6lSVaOmMxl3PTOtfoffaxMo7fTcbZUF+SCJzfKbO6vnN9TCGX2sx1RHFDz66Js6goz6SAZQdOwaw== -"@cspell/dict-scala@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-scala/-/dict-scala-4.0.0.tgz#8901f40a00f801000245e932176e4718cedfb552" - integrity sha512-ugdjt66/Ah34yF3u3DUNjCHXnBqIuxUUfdeBobbGxfm29CNgidrISV1NUh+xi8tPynMzSTpGbBiArFBH6on5XQ== +"@cspell/dict-scala@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@cspell/dict-scala/-/dict-scala-4.0.1.tgz#ed7156ecdfa78b64c8e2477b5b8df075b7a3cef8" + integrity sha512-UvdQpAugrCqRC+2wfqJ4FFKpJr+spLrrrAmqdWEgAyZNMz8ib9FkO+yoIQnNFeodzI9xVPN9Hror+MjXbb2soQ== -"@cspell/dict-software-terms@^3.1.3": - version "3.1.3" - resolved "https://registry.yarnpkg.com/@cspell/dict-software-terms/-/dict-software-terms-3.1.3.tgz#c5a8f5b3e60abad934ca40070c435917ef39d035" - integrity sha512-gmVR+VgjLGPvUPaF4m1JEv9/mWvKE0bkj5w8nwtijZtAToZ6SKpWM/rkcZOsa691fMyhU6LbIfy4tAZbSSLVYA== +"@cspell/dict-software-terms@^3.1.5": + version "3.1.5" + resolved "https://registry.yarnpkg.com/@cspell/dict-software-terms/-/dict-software-terms-3.1.5.tgz#9000ba07df6d868df257ca2438df36a0eb74acc6" + integrity sha512-wmkWHHkp2AN9EDWNBLB0VASB5OtsC3KnhoAHxCJzC6AB3xjYoBfKsvgI/o50gfbsCVQceHpqXjOEYSw/xxTKNw== -"@cspell/dict-sql@^2.0.1": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@cspell/dict-sql/-/dict-sql-2.0.1.tgz#60177b82fb0ddfc0d63b0f6268e33f69627a9347" - integrity sha512-7fvVcvy751cl31KMD5j04yMGq2UKj018/1hx3FNtdUI9UuUTMvhBrTAqHEEemR3ZeIC9i/5p5SQjwQ13bn04qw== +"@cspell/dict-sql@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@cspell/dict-sql/-/dict-sql-2.0.2.tgz#1b23ace554bafb2acd3024bb69aadd0c855ab53e" + integrity sha512-XxUoamMFU9OGcDHLY6+pTlQDsqq9wcY7Oc4C55hqmotxFeFaaqinoD1UIAm1yDngRP7fKK4mVPPFmJI6bmspHg== "@cspell/dict-svelte@^1.0.2": version "1.0.2" @@ -844,27 +823,27 @@ resolved "https://registry.yarnpkg.com/@cspell/dict-swift/-/dict-swift-2.0.1.tgz#06ec86e52e9630c441d3c19605657457e33d7bb6" integrity sha512-gxrCMUOndOk7xZFmXNtkCEeroZRnS2VbeaIPiymGRHj5H+qfTAzAKxtv7jJbVA3YYvEzWcVE2oKDP4wcbhIERw== -"@cspell/dict-typescript@^3.1.0": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-typescript/-/dict-typescript-3.1.0.tgz#4b90c1cc01cf64ad1a87d1a38ebbeb23706c5820" - integrity sha512-4hdLlQMOYrUbGfJg2cWnbsBUevObwgL76TLVC0rwnrkSwzOxAxiGaG39VtRMvgAAe2lX6L+jka3fy0MmxzFOHw== +"@cspell/dict-typescript@^3.1.1": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@cspell/dict-typescript/-/dict-typescript-3.1.1.tgz#25a9c241fa79c032f907db21b0aaf7c7baee6cc3" + integrity sha512-N9vNJZoOXmmrFPR4ir3rGvnqqwmQGgOYoL1+y6D4oIhyr7FhaYiyF/d7QT61RmjZQcATMa6PSL+ZisCeRLx9+A== "@cspell/dict-vue@^3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@cspell/dict-vue/-/dict-vue-3.0.0.tgz#68ccb432ad93fcb0fd665352d075ae9a64ea9250" integrity sha512-niiEMPWPV9IeRBRzZ0TBZmNnkK3olkOPYxC1Ny2AX4TGlYRajcW0WUtoSHmvvjZNfWLSg2L6ruiBeuPSbjnG6A== -"@cspell/dynamic-import@6.26.3": - version "6.26.3" - resolved "https://registry.yarnpkg.com/@cspell/dynamic-import/-/dynamic-import-6.26.3.tgz#4bf9c0b5627287546d3a385a60d3a551b431c214" - integrity sha512-Ic5uNy49mDg/6Qtbuc51zq2sDd0lXiFVN2QKSueNjk5hA5Zh/ZLQhrB70q7qaQwQg7FTiRxvJjpRtNoVqbY/sg== +"@cspell/dynamic-import@6.27.0": + version "6.27.0" + resolved "https://registry.yarnpkg.com/@cspell/dynamic-import/-/dynamic-import-6.27.0.tgz#2a6ab57993ee581a83a6514259aedf6cb5546ff5" + integrity sha512-PrBAH0+6OERWeY57PUH+9WmxYtUBnBjTYmm+3Zpg3SUd/ToD0cNf83OJCrcDF7KNQYE9rjCPUfVdsOe3aJlIHw== dependencies: import-meta-resolve "^2.2.1" -"@cspell/strong-weak-map@6.26.3": - version "6.26.3" - resolved "https://registry.yarnpkg.com/@cspell/strong-weak-map/-/strong-weak-map-6.26.3.tgz#707f3af0fdc2049415848cc5b314e18ffc5e6096" - integrity sha512-PC+I5obQY6f/l4/Z4TiE6HJhDiuR8wCPYqezPtBuD1Fw7Op0Nni77gUPKajTxhy1WHpks/PTTSjnV/cX9Mgt1Q== +"@cspell/strong-weak-map@6.27.0": + version "6.27.0" + resolved "https://registry.yarnpkg.com/@cspell/strong-weak-map/-/strong-weak-map-6.27.0.tgz#4065eab7b4ca45db0a08b697be7c64f59abb122a" + integrity sha512-4hel0Dik7GTX8yVOjOXlumIOzYD8mWzDeyMEhEJb3qwDfsTPeeYo1EMTSgtS+oJXR6kQ09qBrF1lok44v6NOAQ== "@cspotcode/source-map-support@^0.8.0": version "0.8.1" @@ -907,6 +886,11 @@ tsutils "^3" typescript "^4" +"@foxglove/message-definition@^0.2.0": + version "0.2.0" + resolved "https://registry.yarnpkg.com/@foxglove/message-definition/-/message-definition-0.2.0.tgz#e31922e58e57d224717bcfbd20d3cbaad709dc6b" + integrity sha512-IQHIGCvBZR8GIua9nEpS+hsMF3gm1bfbrrnjG0rgtcFBWiNuKbzx4vIP8OIwDC+8wtwcFdfJhf4Vp5TPFiUUcQ== + "@foxglove/rosbag@0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@foxglove/rosbag/-/rosbag-0.2.3.tgz#bcba9c099dfdbb6a8fd27535d3c322805ea68d78" @@ -917,12 +901,13 @@ "@foxglove/rostime" "^1.1.2" heap "^0.2.7" -"@foxglove/rosmsg-msgs-common@^2.0.0": - version "2.1.0" - resolved "https://registry.yarnpkg.com/@foxglove/rosmsg-msgs-common/-/rosmsg-msgs-common-2.1.0.tgz#8d348db60dc69670916ca9dc56999abb6de4134f" - integrity sha512-Lxu3CEb+Ko1hrLu1bIqvvoc8DvGW1FAA21sWW0MoTPTgoQ1Rnpke2Z4H6O78C8DwXPzXAqZIisgxfFM9NYiHPA== +"@foxglove/rosmsg-msgs-common@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@foxglove/rosmsg-msgs-common/-/rosmsg-msgs-common-3.0.0.tgz#d5f09b9e1feb43eb21aedc12ab23bc47787558e4" + integrity sha512-MSruqgOtT+PrQO2wqprvB0dA7ZbPKM9AqDcdOQFo/e8RIjXfgMsio64kmYHYXF8iHcutqoAaTHxi74XMwRzwzg== dependencies: - "@foxglove/rosmsg" "^3.1.0" + "@foxglove/message-definition" "^0.2.0" + "@foxglove/rosmsg" "^4.0.0" "@foxglove/rosmsg-serialization@1.5.3", "@foxglove/rosmsg-serialization@^1.5.2": version "1.5.3" @@ -931,6 +916,13 @@ dependencies: "@foxglove/rosmsg" "^3.1.0" +"@foxglove/rosmsg-serialization@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@foxglove/rosmsg-serialization/-/rosmsg-serialization-2.0.0.tgz#40e6babdcb8e02e1b70d3bcce049ac996b7201be" + integrity sha512-SvhpGm5v/I8ieNOc/H9XR/UxLlhKqDBOdLfhGkufzSEBpJxR86mT9xYzZam9AYT0gEvxOwGDIaRvIuvkyxU0RA== + dependencies: + "@foxglove/message-definition" "^0.2.0" + "@foxglove/rosmsg2-serialization@1.1.1": version "1.1.1" resolved "https://registry.yarnpkg.com/@foxglove/rosmsg2-serialization/-/rosmsg2-serialization-1.1.1.tgz#89ca2017d2cdf6bbbf70aeb6726966d13ba1ac6c" @@ -940,6 +932,15 @@ "@foxglove/rosmsg" "^3.1.0" "@foxglove/rostime" "^1.1.2" +"@foxglove/rosmsg2-serialization@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@foxglove/rosmsg2-serialization/-/rosmsg2-serialization-2.0.0.tgz#4d5f54108690914d5ea5387ce93fce9c5681d7ed" + integrity sha512-EJppfp/VeQyyV/UsvAkuXULIbMQn1u2fLU49Mx4yYWX1owsgSKMCOliCmoTgwAjGWL3CzwhUV5iR8tptVroJSA== + dependencies: + "@foxglove/cdr" "^2.0.0" + "@foxglove/message-definition" "^0.2.0" + "@foxglove/rostime" "^1.1.2" + "@foxglove/rosmsg@3.1.0", "@foxglove/rosmsg@^3.1.0": version "3.1.0" resolved "https://registry.yarnpkg.com/@foxglove/rosmsg/-/rosmsg-3.1.0.tgz#edbe3e2e743a5cc89df8195889a054e639230916" @@ -947,37 +948,45 @@ dependencies: md5-typescript "^1.0.5" +"@foxglove/rosmsg@^4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@foxglove/rosmsg/-/rosmsg-4.0.0.tgz#b0f1b57db4bfe2ea00f1d5904caeedb2d13bccb0" + integrity sha512-MuTAcA4U+wzlhUIHYgfBrjTDE/kzukWmhyI8suKgmjlnVjrpyzDNiMYsV5lU0KTW6noGKtqu46ZtxWSEWsOsEg== + dependencies: + "@foxglove/message-definition" "^0.2.0" + md5-typescript "^1.0.5" + "@foxglove/rostime@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@foxglove/rostime/-/rostime-1.1.2.tgz#f8e1f10bff115c22be8a3e06791c2ac800449ee8" integrity sha512-vWuTJCuGv0xvgwOlrZ1y2MevmNMVxWcUU/HwlmYXi/jUq/kRaACStU18uyuZ3LzdNKaffkti0rTcXWESaQjwQw== -"@foxglove/schemas@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@foxglove/schemas/-/schemas-1.0.0.tgz#8c1af195dc3a78b6f64d8f21fbdd6ba180a5920d" - integrity sha512-qJxJ74VKYwY+sdW1GZNmYK51whKuvQ+NMuE93PDl/FzRvEOqJ6Phxds9LapaRbv0ITuMplFT2YldPRaveIy/bQ== +"@foxglove/schemas@^1.0.0", "@foxglove/schemas@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@foxglove/schemas/-/schemas-1.1.0.tgz#d3080995a8ba596630d33572c746a19fe4513319" + integrity sha512-ojkDY1C/jTCvuh7ctm0jyhdIRlOL0R0jCsISeCw2/U9VzuF/AQtZfLZKDEOfAIa/MVnt3EQVHhOb3oaWetXMzA== dependencies: - "@foxglove/rosmsg-msgs-common" "^2.0.0" - tslib "^2" + "@foxglove/rosmsg-msgs-common" "^3.0.0" + tslib "^2.5.0" "@foxglove/tsconfig@1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@foxglove/tsconfig/-/tsconfig-1.1.0.tgz#48c37fffd6f349c3ee08a60fc62ccf636f3b59a6" integrity sha512-qZU4MtXVgPhDBFazSEx7yDEuEg8cPHXFQVhBaUABZkCBdcnEE9sxlgEt0gSikF4fRtY6COGIJPVRflnPJXjJKA== -"@foxglove/wasm-bz2@0.1.1": +"@foxglove/wasm-bz2@0.1.1", "@foxglove/wasm-bz2@^0.1.1": version "0.1.1" resolved "https://registry.yarnpkg.com/@foxglove/wasm-bz2/-/wasm-bz2-0.1.1.tgz#d3bf629b7caf747a0278ed2efb9c38644b6d3439" integrity sha512-huMVZ//J9S1TAh689pj7U5tstbmtGhH1g9/HCj9jE3UPaLF83dTeJIOLE0+pe16ha1iH4QRvYgv35aykikHkvA== dependencies: tslib "^2" -"@foxglove/wasm-lz4@1.0.2": +"@foxglove/wasm-lz4@1.0.2", "@foxglove/wasm-lz4@^1.0.2": version "1.0.2" resolved "https://registry.yarnpkg.com/@foxglove/wasm-lz4/-/wasm-lz4-1.0.2.tgz#ce8e27ae9c039058a033e288043f38acee3d1e2b" integrity sha512-qwGOVLITlMBwuI+Vf09xWkTtbQldaCKchGEiFNZzsY0Q6ZXnHFhFgl4UAew7Xaq0wCv5YB7hqYsHmSrFIVSjCg== -"@foxglove/wasm-zstd@1.0.1": +"@foxglove/wasm-zstd@1.0.1", "@foxglove/wasm-zstd@^1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@foxglove/wasm-zstd/-/wasm-zstd-1.0.1.tgz#41f97f9c1ed5d10790603376fbecda7ce2145fe5" integrity sha512-hqKTO45Sl+kSq9SDqM9xi5L+GncjFEZIJgYEkGTlpJXTfhy1Gf+a+3epWMcE0tLUKmzM77dtu8snape/J+7ueQ== @@ -1862,7 +1871,7 @@ brace-expansion@^2.0.1: dependencies: balanced-match "^1.0.0" -braces@^3.0.1, braces@^3.0.2, braces@~3.0.2: +braces@^3.0.2, braces@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== @@ -2045,7 +2054,7 @@ color-convert@^2.0.1: color-name@1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" - integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= + integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== color-name@^1.1.4, color-name@~1.1.4: version "1.1.4" @@ -2086,7 +2095,7 @@ common-tags@^1.8.0: concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" - integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= + integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== configstore@^5.0.1: version "5.0.1" @@ -2117,7 +2126,7 @@ core-util-is@^1.0.3: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== -cosmiconfig@^8.0.0: +cosmiconfig@8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-8.0.0.tgz#e9feae014eab580f858f8a0288f38997a7bebe97" integrity sha512-da1EafcpH6b/TD8vDRaWV7xFINlHlF6zKsGwS1TsuVJTZRkquaS5HTMq7uq6h31619QjbsYl21gVDOm32KM1vQ== @@ -2146,66 +2155,66 @@ crypto-random-string@^2.0.0: resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5" integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA== -cspell-dictionary@6.26.3: - version "6.26.3" - resolved "https://registry.yarnpkg.com/cspell-dictionary/-/cspell-dictionary-6.26.3.tgz#969572f91ad434203360c31aca2f67b12ce05671" - integrity sha512-wUiTHe7OWZuptEROJm3gzSk12ABAozArFnKVNfsfVR/tgBIjLTgX+9RIOuJL0g+vDxIsZu8dpOuty3MPmI3vBg== +cspell-dictionary@6.27.0: + version "6.27.0" + resolved "https://registry.yarnpkg.com/cspell-dictionary/-/cspell-dictionary-6.27.0.tgz#8cd3ed900f915d32f3f0d48de28857b37107e4d1" + integrity sha512-u2HZ6Yl0tIBzflJ9Rt5i15kP1KN41XtyKfqCCntugX3gYtfbQ1t/HvAPcq4g7EDZV2tzqSKGcNvzuJgHJyxxFw== dependencies: - "@cspell/cspell-pipe" "6.26.3" - "@cspell/cspell-types" "6.26.3" - cspell-trie-lib "6.26.3" + "@cspell/cspell-pipe" "6.27.0" + "@cspell/cspell-types" "6.27.0" + cspell-trie-lib "6.27.0" fast-equals "^4.0.3" gensequence "^4.0.3" -cspell-gitignore@6.26.3: - version "6.26.3" - resolved "https://registry.yarnpkg.com/cspell-gitignore/-/cspell-gitignore-6.26.3.tgz#bf716d58908552aa399438fc23581dde959b0f84" - integrity sha512-K6Gl3I71UZOU9077xXhJmG2Bwzhj6ay64pnry6+KBHJDAxuSWnXaWg1/KpSf6ciwwvggyRxhKOSykzLI8Ivweg== +cspell-gitignore@6.27.0: + version "6.27.0" + resolved "https://registry.yarnpkg.com/cspell-gitignore/-/cspell-gitignore-6.27.0.tgz#eb5d700c357fc1318732f6bf8b58d0240e9edc24" + integrity sha512-aXIuSMtm2pW8Z0y4SHBqvXwY4Hk3lz4b3PD50IU1gn2mjk8GioEYAnWlN/FANeOuMjOXiRfLaX0McvNErnZxdw== dependencies: - cspell-glob "6.26.3" + cspell-glob "6.27.0" find-up "^5.0.0" -cspell-glob@6.26.3: - version "6.26.3" - resolved "https://registry.yarnpkg.com/cspell-glob/-/cspell-glob-6.26.3.tgz#73bb82fdc82681f2bbe3f879400cd0e4bcec2eb2" - integrity sha512-6f6waZGHZ1Vt9HVOqQrkYfq5EMJ+UvJGgiq1tVO8jDGdayupNIaivh9XT6ReWHJVLbKypJddQzrw7eMMEd0Mmg== +cspell-glob@6.27.0: + version "6.27.0" + resolved "https://registry.yarnpkg.com/cspell-glob/-/cspell-glob-6.27.0.tgz#fd9ac0828c12e2aa67da94de53c60f1a11da4575" + integrity sha512-YzS9SiNU5iFIdEMCeVOsGlCvffCM7M51io9ylkkY6138NLCnqWw/DSePzIAAsuLqh9nsJt8xiXrHPerKiPnN3g== dependencies: micromatch "^4.0.5" -cspell-grammar@6.26.3: - version "6.26.3" - resolved "https://registry.yarnpkg.com/cspell-grammar/-/cspell-grammar-6.26.3.tgz#f277da8830971865bba9f8d78d7d86da345f072e" - integrity sha512-eoqMETuGH6bjsSnK5UGtfLKLkW+VKOQBGRQBVBfI+2KKaZyfvm7/q8ScRYdAsoQg67Ws7/2Dplej7vRltyfCQQ== +cspell-grammar@6.27.0: + version "6.27.0" + resolved "https://registry.yarnpkg.com/cspell-grammar/-/cspell-grammar-6.27.0.tgz#3c0e9561c36c0fd8795667d7215945a4f3a193db" + integrity sha512-03SH0+bWazhkzUUK+t6ywUZvWuDcqj4J171oIdA3fvdG7nBpTqyFc1/vU1mReZK7CDde16BUaqCkWgf+El+N1w== dependencies: - "@cspell/cspell-pipe" "6.26.3" - "@cspell/cspell-types" "6.26.3" + "@cspell/cspell-pipe" "6.27.0" + "@cspell/cspell-types" "6.27.0" -cspell-io@6.26.3: - version "6.26.3" - resolved "https://registry.yarnpkg.com/cspell-io/-/cspell-io-6.26.3.tgz#c144ae8bde6b28210897ef7e66c080e01aec7f18" - integrity sha512-bUzsHM+A+jfMEYuwBnC/w2KIgf4TPEx3E5AIfg+qtRuP2paTYOFulNBWgxzWovSkXH08R4yNgDQIN1dO3Fhzjw== +cspell-io@6.27.0: + version "6.27.0" + resolved "https://registry.yarnpkg.com/cspell-io/-/cspell-io-6.27.0.tgz#6ace9ddb190354d5aca6fe9de766c48edb4ac787" + integrity sha512-WsvXjbbWwIQVembEtlNuC8cJrGtyUuk8GvZzL9bBpSofU2UXvFjaQ9bAZjIeHibGQrIIuOM0ra0CqOcLb5mShA== dependencies: - "@cspell/cspell-service-bus" "6.26.3" + "@cspell/cspell-service-bus" "6.27.0" node-fetch "^2.6.9" -cspell-lib@6.26.3: - version "6.26.3" - resolved "https://registry.yarnpkg.com/cspell-lib/-/cspell-lib-6.26.3.tgz#2462c61aad8cd01007e2b605f441f7c608b5c844" - integrity sha512-UwtrGSHoZxQmTm78yB55KLIz46THG1neZ87mYHdoYgc5EOc2gKTWRPfYsioUs8fH31L+4CwHNbdxvTRg+Vpg/Q== +cspell-lib@6.27.0: + version "6.27.0" + resolved "https://registry.yarnpkg.com/cspell-lib/-/cspell-lib-6.27.0.tgz#c3f073e56e6a4ffb36e03ab4f439fe24e9215ff3" + integrity sha512-wwi3VCFiWaVFrV/ycm4yNBjjPY3pyhWud4lVJs51p2IkKqJbmR7hZ+bSIyw6d9MVzPKRD67eMaNBYXFGDaNHGQ== dependencies: - "@cspell/cspell-bundled-dicts" "6.26.3" - "@cspell/cspell-pipe" "6.26.3" - "@cspell/cspell-types" "6.26.3" - "@cspell/strong-weak-map" "6.26.3" + "@cspell/cspell-bundled-dicts" "6.27.0" + "@cspell/cspell-pipe" "6.27.0" + "@cspell/cspell-types" "6.27.0" + "@cspell/strong-weak-map" "6.27.0" clear-module "^4.1.2" comment-json "^4.2.3" configstore "^5.0.1" - cosmiconfig "^8.0.0" - cspell-dictionary "6.26.3" - cspell-glob "6.26.3" - cspell-grammar "6.26.3" - cspell-io "6.26.3" - cspell-trie-lib "6.26.3" + cosmiconfig "8.0.0" + cspell-dictionary "6.27.0" + cspell-glob "6.27.0" + cspell-grammar "6.27.0" + cspell-io "6.27.0" + cspell-trie-lib "6.27.0" fast-equals "^4.0.3" find-up "^5.0.0" gensequence "^4.0.3" @@ -2215,27 +2224,28 @@ cspell-lib@6.26.3: vscode-languageserver-textdocument "^1.0.8" vscode-uri "^3.0.7" -cspell-trie-lib@6.26.3: - version "6.26.3" - resolved "https://registry.yarnpkg.com/cspell-trie-lib/-/cspell-trie-lib-6.26.3.tgz#cfdba2e87483dabf037b2b4b8a1f268e6de6020b" - integrity sha512-pda7iXr74SC9eD5ksAEDDR2M/ervnGaHXugTjn+TVXXBH16lnmqz/Ns5Zlp351lwb3BhqjVU+XqZ0tn28ISvAw== +cspell-trie-lib@6.27.0: + version "6.27.0" + resolved "https://registry.yarnpkg.com/cspell-trie-lib/-/cspell-trie-lib-6.27.0.tgz#724d0895dbc11dd8f0ae3792ab4498e186253070" + integrity sha512-kelDXszZKzlWbk7hV3cTtWYd2Gs3MXqTNSL3udtN4Oow74ik+h1bWsOOmXKKjtKvRctx4omWC1JdriQXfhBMBA== dependencies: - "@cspell/cspell-pipe" "6.26.3" - "@cspell/cspell-types" "6.26.3" + "@cspell/cspell-pipe" "6.27.0" + "@cspell/cspell-types" "6.27.0" gensequence "^4.0.3" cspell@^6.26.3: - version "6.26.3" - resolved "https://registry.yarnpkg.com/cspell/-/cspell-6.26.3.tgz#f778de51a01e77c0c5aa060e0c0b197d1aa14619" - integrity sha512-h7p8JpWSFhgNbsJLlpjzMCQ0k6TuhX/M5JcrED14x17CuZR7ad29lQDRF0Un82Wxhd8hJNxZubV9IBdWZA7Qig== + version "6.27.0" + resolved "https://registry.yarnpkg.com/cspell/-/cspell-6.27.0.tgz#8e3b2265d4be712060a6ed09bbfb72215519a8f0" + integrity sha512-BhDzXJIRFcpswOy32tgnoOw2OdSM91pQuUpRxfZvRxdM4HBxUrodDYJ5HbBz7D0vNBrXFidFX/157KNX4m1gmg== dependencies: - "@cspell/cspell-pipe" "6.26.3" - "@cspell/dynamic-import" "6.26.3" + "@cspell/cspell-pipe" "6.27.0" + "@cspell/dynamic-import" "6.27.0" chalk "^4.1.2" commander "^10.0.0" - cspell-gitignore "6.26.3" - cspell-glob "6.26.3" - cspell-lib "6.26.3" + cspell-gitignore "6.27.0" + cspell-glob "6.27.0" + cspell-io "6.27.0" + cspell-lib "6.27.0" fast-glob "^3.2.12" fast-json-stable-stringify "^2.1.0" file-entry-cache "^6.0.1" @@ -2498,7 +2508,7 @@ escalade@^3.1.1: escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" - integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= + integrity sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg== escape-string-regexp@^2.0.0: version "2.0.0" @@ -2785,9 +2795,9 @@ fast-levenshtein@^2.0.6: integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= fastq@^1.6.0: - version "1.13.0" - resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c" - integrity sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw== + version "1.15.0" + resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.15.0.tgz#d04d07c6a2a68fe4599fea8d2e103a937fae6b3a" + integrity sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw== dependencies: reusify "^1.0.4" @@ -2841,10 +2851,15 @@ flatbuffers@^23.1.21: resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-23.1.21.tgz#a677f0e79484ce70070e22994c12e2bdeb005133" integrity sha512-7ztssjhUCQKCd7MwQNlQDwNam8iL4reBAMHglf024amlVRn/aIo7B8+ee/igKxxK5B7Wp4nWe4sTYg8lpK2CAQ== +flatbuffers_reflection@^0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/flatbuffers_reflection/-/flatbuffers_reflection-0.0.3.tgz#16a4e83c354e5cc871f05f763f20e161c0db2816" + integrity sha512-qP4yT0U2ouqp+kgootXdLaRl9O5tj8YM1Dpcucs3E14u6NvyWqHZ7/JdfhhWnSI9qnZVWVIrn0bnBME2l2nXBg== + flatted@^3.1.0: - version "3.2.4" - resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.4.tgz#28d9969ea90661b5134259f312ab6aa7929ac5e2" - integrity sha512-8/sOawo8tJ4QOBX8YlQBMxL8+RLZfxMQOif9o0KUKTNTjMYElWPE0r/m5VNFxTRd0NSw8qSy8dajrwX4RYI1Hw== + version "3.2.7" + resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.7.tgz#609f39207cb614b89d0765b477cb2d437fbf9787" + integrity sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ== for-each@^0.3.3: version "0.3.3" @@ -2865,7 +2880,7 @@ fs-extra@^10.0.0: fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" - integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= + integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== fsevents@^2.3.2, fsevents@~2.3.2: version "2.3.2" @@ -2962,7 +2977,19 @@ glob-parent@^6.0.2: dependencies: is-glob "^4.0.3" -glob@^7.1.3, glob@^7.1.4: +glob@^7.1.3: + version "7.2.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" + integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.1.1" + once "^1.3.0" + path-is-absolute "^1.0.0" + +glob@^7.1.4: version "7.2.0" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== @@ -2977,7 +3004,7 @@ glob@^7.1.3, glob@^7.1.4: global-dirs@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-0.1.1.tgz#b319c0dd4607f353f3be9cca4c72fc148c49f445" - integrity sha1-sxnA3UYH81PzvpzKTHL8FIxJ9EU= + integrity sha512-NknMLn7F2J7aflwFOlGdNIuCDpN3VGoSoB+aap3KABFWbHVn1TCgFC+np23J8W2BiZbjfEw3BFBycSMv1AFblg== dependencies: ini "^1.3.4" @@ -3019,16 +3046,16 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" -graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0: - version "4.2.9" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.9.tgz#041b05df45755e587a24942279b9d113146e1c96" - integrity sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ== - -graceful-fs@^4.2.9: +graceful-fs@^4.1.2, graceful-fs@^4.2.9: version "4.2.10" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== +graceful-fs@^4.1.6, graceful-fs@^4.2.0: + version "4.2.9" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.9.tgz#041b05df45755e587a24942279b9d113146e1c96" + integrity sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ== + grapheme-splitter@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz#9cf3a665c6247479896834af35cf1dbb4400767e" @@ -3047,7 +3074,7 @@ has-bigints@^1.0.2: has-flag@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" - integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= + integrity sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw== has-flag@^4.0.0: version "4.0.0" @@ -3144,12 +3171,12 @@ import-meta-resolve@^2.2.1: imurmurhash@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" - integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= + integrity sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA== inflight@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" - integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk= + integrity sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA== dependencies: once "^1.3.0" wrappy "1" @@ -3194,7 +3221,7 @@ is-array-buffer@^3.0.1: is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" - integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0= + integrity sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg== is-bigint@^1.0.1: version "1.0.4" @@ -3252,7 +3279,7 @@ is-date-object@^1.0.1: is-extglob@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" - integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI= + integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ== is-fullwidth-code-point@^3.0.0: version "3.0.0" @@ -3356,7 +3383,7 @@ is-typed-array@^1.1.10, is-typed-array@^1.1.9: is-typedarray@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" - integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= + integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== is-weakref@^1.0.1: version "1.0.1" @@ -4053,7 +4080,7 @@ marked@^4.2.12: md5-typescript@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/md5-typescript/-/md5-typescript-1.0.5.tgz#68c0b24dff8e5d3162e498fa9893b63be72e038f" - integrity sha1-aMCyTf+OXTFi5Jj6mJO2O+cuA48= + integrity sha512-ovAc4EtiNt2dY8JPhPr/wkC9h4U5k/nuClNVcG0Ga3V1rMlYpAY24ZaaymFXJlz+ccJ6UMPo3FSaVKe7czBsXw== merge-stream@^2.0.0: version "2.0.0" @@ -4065,15 +4092,7 @@ merge2@^1.3.0, merge2@^1.4.1: resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== -micromatch@^4.0.4: - version "4.0.4" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.4.tgz#896d519dfe9db25fce94ceb7a500919bf881ebf9" - integrity sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg== - dependencies: - braces "^3.0.1" - picomatch "^2.2.3" - -micromatch@^4.0.5: +micromatch@^4.0.4, micromatch@^4.0.5: version "4.0.5" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== @@ -4086,24 +4105,17 @@ mimic-fn@^2.1.0: resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== -minimatch@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" - integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== - dependencies: - brace-expansion "^1.1.7" - -minimatch@^3.0.5, minimatch@^3.1.2: +minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== dependencies: brace-expansion "^1.1.7" -minimatch@^6.1.6: - version "6.2.0" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-6.2.0.tgz#2b70fd13294178c69c04dfc05aebdb97a4e79e42" - integrity sha512-sauLxniAmvnhhRjFwPNnJKaPFYyddAgbYdeUpHULtCT/GhzdCx/MDNy+Y40lBxTQUrMzDE8e0S43Z5uqfO0REg== +minimatch@^7.1.3: + version "7.3.0" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-7.3.0.tgz#cfb7337e7460308e7147c58250fa0dee3da7929c" + integrity sha512-WaMDuhKa7a6zKiwplR1AOz+zGvJba24k5VU1Cy6NhEguavT2YRlHxuINUgTas4wiS6fwBpYq4TcA1XIECSntyw== dependencies: brace-expansion "^2.0.1" @@ -4228,7 +4240,7 @@ object.values@^1.1.6: once@^1.3.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" - integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= + integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== dependencies: wrappy "1" @@ -4316,7 +4328,7 @@ path-exists@^4.0.0: path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" - integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= + integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== path-key@^3.0.0, path-key@^3.1.0: version "3.1.1" @@ -4338,12 +4350,12 @@ picocolors@^1.0.0: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== -picomatch@^2.0.4, picomatch@^2.2.3: +picomatch@^2.0.4: version "2.3.0" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw== -picomatch@^2.2.1, picomatch@^2.3.1: +picomatch@^2.2.1, picomatch@^2.2.3, picomatch@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== @@ -4457,7 +4469,7 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.5" -protobufjs@7.2.2: +protobufjs@7.2.2, protobufjs@^7.2.2: version "7.2.2" resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.2.2.tgz#2af401d8c547b9476fb37ffc65782cf302342ca3" integrity sha512-++PrQIjrom+bFDPpfmqXfAGSQs40116JRrqqyf53dymUMvvb5d/LMRyicRoF1AUKoXVS1/IgJXlEgcpr4gTF3Q== @@ -4526,7 +4538,7 @@ regexpp@^3.0.0, regexpp@^3.2.0: repeat-string@^1.6.1: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" - integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= + integrity sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w== require-directory@^2.1.1: version "2.1.1" @@ -4670,16 +4682,16 @@ side-channel@^1.0.4: get-intrinsic "^1.0.2" object-inspect "^1.9.0" -signal-exit@^3.0.2, signal-exit@^3.0.3: - version "3.0.6" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.6.tgz#24e630c4b0f03fea446a2bd299e62b4a6ca8d0af" - integrity sha512-sDl4qMFpijcGw22U5w63KmD3cZJfBuFlVNbVMKje2keoKML7X2UzWbc4XrmEbDwg0NXJc3yv4/ox7b+JWb57kQ== - -signal-exit@^3.0.7: +signal-exit@^3.0.2, signal-exit@^3.0.7: version "3.0.7" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== +signal-exit@^3.0.3: + version "3.0.6" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.6.tgz#24e630c4b0f03fea446a2bd299e62b4a6ca8d0af" + integrity sha512-sDl4qMFpijcGw22U5w63KmD3cZJfBuFlVNbVMKje2keoKML7X2UzWbc4XrmEbDwg0NXJc3yv4/ox7b+JWb57kQ== + sisteransi@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed" @@ -4960,12 +4972,7 @@ tslib@^1.8.1: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== -tslib@^2: - version "2.3.1" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01" - integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw== - -tslib@^2.5.0: +tslib@^2, tslib@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.0.tgz#42bfed86f5787aeb41d031866c8f402429e0fddf" integrity sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg== @@ -5016,13 +5023,13 @@ typedarray-to-buffer@^3.1.5: is-typedarray "^1.0.0" typedoc@^0.23.25: - version "0.23.25" - resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.23.25.tgz#5f8f1850fd044c4d15d453117affddf11a265610" - integrity sha512-O1he153qVyoCgJYSvIyY3bPP1wAJTegZfa6tL3APinSZhJOf8CSd8F/21M6ex8pUY/fuY6n0jAsT4fIuMGA6sA== + version "0.23.26" + resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.23.26.tgz#ae082683698bad68757d8fe619242a56d6b5bf36" + integrity sha512-5m4KwR5tOLnk0OtMaRn9IdbeRM32uPemN9kur7YK9wFqx8U0CYrvO9aVq6ysdZSV1c824BTm+BuQl2Ze/k1HtA== dependencies: lunr "^2.3.9" marked "^4.2.12" - minimatch "^6.1.6" + minimatch "^7.1.3" shiki "^0.14.1" typescript@4.9.5, typescript@^4: @@ -5192,7 +5199,7 @@ wrap-ansi@^7.0.0: wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" - integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= + integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== write-file-atomic@^3.0.0: version "3.0.3" From 6f49dfad3e6d4d00162c3b8b836447ce79e8ff3d Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Tue, 28 Mar 2023 14:15:41 -0700 Subject: [PATCH 02/27] WIP --- .vscode/settings.json | 7 +- package.json | 4 +- tests/conformance/package.json | 5 +- .../TypescriptIndexedReaderTestRunner.ts | 24 +- tests/conformance/tsconfig.cjs.json | 6 +- tests/conformance/tsconfig.json | 6 +- typescript/CONTRIBUTING.md | 33 + typescript/README.md | 33 +- typescript/core/README.md | 8 + typescript/core/jest.config.json | 16 +- typescript/core/package.json | 7 +- typescript/core/tsconfig.json | 4 +- typescript/examples/bag2mcap/package.json | 6 +- typescript/examples/basicwriter/package.json | 2 +- .../examples/flatbufferswriter/package.json | 2 +- typescript/examples/validate/package.json | 6 +- .../examples/validate/scripts/validate.ts | 24 +- typescript/support/README.md | 53 ++ typescript/support/jest.config.json | 16 +- typescript/support/package.json | 22 +- .../support/src/nodejs/FileHandleReadable.ts | 37 + .../support/src/nodejs/FileHandleWritable.ts | 6 +- typescript/support/src/nodejs/index.ts | 1 + typescript/support/tsconfig.json | 8 +- typescript/typedoc.json | 5 + yarn.lock | 788 ++---------------- 26 files changed, 281 insertions(+), 848 deletions(-) create mode 100644 typescript/CONTRIBUTING.md create mode 100644 typescript/support/README.md create mode 100644 typescript/support/src/nodejs/FileHandleReadable.ts create mode 100644 typescript/typedoc.json diff --git a/.vscode/settings.json b/.vscode/settings.json index 4e56bcc568..a9338fda74 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,16 +17,15 @@ "tests/conformance/data": true, "python/**/build": true, "python/docs/*-apidoc": true, + ".yarn/**": true, + "yarn.lock": true }, "python.formatting.provider": "black", "python.analysis.typeCheckingMode": "strict", "python.linting.flake8Enabled": true, "python.linting.enabled": true, - "python.linting.flake8Args": [ - "--config", - "python/.flake8" - ], + "python.linting.flake8Args": ["--config", "python/.flake8"], "python.analysis.extraPaths": [ "./python/mcap", "./python/mcap-protobuf-support", diff --git a/package.json b/package.json index 9fc877e987..53b14bd9b8 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,10 @@ "docs:lint": "prettier '**/*.md' --check", "docs:build": "tailwindcss --config docs/website/tailwind.config.js -i docs/website/style.css -o docs/website/dist/bundle.css", "docs:swift:start": "swift package --disable-sandbox preview-documentation --target MCAP", - "typedoc": "typedoc --out __docs__/typescript typescript/core/src/index.ts --tsconfig typescript/core/tsconfig.json", + "typedoc": "typedoc --out __docs__/typescript --options typescript/typedoc.json", "spellcheck": "cspell --relative '**'", + "typescript:build": "yarn workspace @mcap/core build && yarn workspace @mcap/support build", + "typescript:clean": "yarn workspace @mcap/core build --clean && yarn workspace @mcap/support build --clean", "test:conformance:generate-inputs": "yarn workspace @foxglove/mcap-conformance generate-inputs --data-dir \"$(pwd)/tests/conformance/data\"", "test:conformance": "yarn workspace @foxglove/mcap-conformance run-tests --data-dir \"$(pwd)/tests/conformance/data\"" }, diff --git a/tests/conformance/package.json b/tests/conformance/package.json index 54089a3ed5..00647c793a 100644 --- a/tests/conformance/package.json +++ b/tests/conformance/package.json @@ -7,13 +7,14 @@ "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", "generate-inputs": "ts-node --files --project tsconfig.cjs.json scripts/generate-inputs", - "run-tests": "ts-node --files --project tsconfig.cjs.json scripts/run-tests" + "run-tests": "tsc -b tsconfig.json tsconfig.cjs.json && ts-node --files --project tsconfig.cjs.json scripts/run-tests" }, "devDependencies": { "@foxglove/crc": "^0.0.3", "@foxglove/eslint-plugin": "0.21.0", "@foxglove/tsconfig": "1.1.0", - "@mcap/core": "*", + "@mcap/core": "workspace:*", + "@mcap/support": "workspace:*", "@types/diff": "^5.0.2", "@types/js-yaml": "^4.0.5", "@types/node": "18.13.0", diff --git a/tests/conformance/scripts/run-tests/runners/TypescriptIndexedReaderTestRunner.ts b/tests/conformance/scripts/run-tests/runners/TypescriptIndexedReaderTestRunner.ts index d3b5ba16c5..0af798935b 100644 --- a/tests/conformance/scripts/run-tests/runners/TypescriptIndexedReaderTestRunner.ts +++ b/tests/conformance/scripts/run-tests/runners/TypescriptIndexedReaderTestRunner.ts @@ -1,4 +1,5 @@ import { McapIndexedReader } from "@mcap/core"; +import { FileHandleReadable } from "@mcap/support/nodejs"; import fs from "fs/promises"; import { TestFeatures, TestVariant } from "variants/types"; @@ -41,28 +42,7 @@ export default class TypescriptIndexedReaderTestRunner extends IndexedReadTestRu } private async _run(fileHandle: fs.FileHandle): Promise { - let buffer = new ArrayBuffer(4096); - const readable = { - size: async () => BigInt((await fileHandle.stat()).size), - read: async (offset: bigint, length: bigint) => { - if (offset > Number.MAX_SAFE_INTEGER || length > Number.MAX_SAFE_INTEGER) { - throw new Error(`Read too large: offset ${offset}, length ${length}`); - } - if (length > buffer.byteLength) { - buffer = new ArrayBuffer(Number(length * 2n)); - } - const result = await fileHandle.read({ - buffer: new DataView(buffer, 0, Number(length)), - position: Number(offset), - }); - if (result.bytesRead !== Number(length)) { - throw new Error( - `Read only ${result.bytesRead} bytes from offset ${offset}, expected ${length}`, - ); - } - return new Uint8Array(result.buffer.buffer, result.buffer.byteOffset, result.bytesRead); - }, - }; + const readable = new FileHandleReadable(fileHandle); const reader = await McapIndexedReader.Initialize({ readable }); if (reader.chunkIndexes.length === 0) { diff --git a/tests/conformance/tsconfig.cjs.json b/tests/conformance/tsconfig.cjs.json index ac31cb3343..916ffa5a73 100644 --- a/tests/conformance/tsconfig.cjs.json +++ b/tests/conformance/tsconfig.cjs.json @@ -3,5 +3,9 @@ "compilerOptions": { "outDir": "./dist/cjs", "module": "commonjs" - } + }, + "references": [ + { "path": "../../typescript/core/tsconfig.cjs.json" }, + { "path": "../../typescript/support/tsconfig.cjs.json" } + ] } diff --git a/tests/conformance/tsconfig.json b/tests/conformance/tsconfig.json index a8c9f12ab0..05b1a76f34 100644 --- a/tests/conformance/tsconfig.json +++ b/tests/conformance/tsconfig.json @@ -6,13 +6,17 @@ "noEmit": true, "lib": ["es2020", "dom"], "paths": { - "@mcap/core": ["../../typescript/core/src"] + "@mcap/core": ["../../typescript/core/src"], + "@mcap/support/nodejs": ["../../typescript/src/nodejs"], + "@mcap/support": ["../../typescript/support/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 "baseUrl": "." }, + "references": [{ "path": "../../typescript/core" }, { "path": "../../typescript/support" }], "ts-node": { "require": ["tsconfig-paths/register"] + // "files": true } } diff --git a/typescript/CONTRIBUTING.md b/typescript/CONTRIBUTING.md new file mode 100644 index 0000000000..276f4d904a --- /dev/null +++ b/typescript/CONTRIBUTING.md @@ -0,0 +1,33 @@ +## Development guide + +Install dependencies: + +``` +corepack enable +yarn install +``` + +Run lint/tests: + +``` +yarn workspace @mcap/core lint +yarn workspace @mcap/core test +``` + +Read and validate an MCAP file: + +``` +yarn workspace @foxglove/mcap-example-validate validate file.mcap +``` + +Run benchmarks: + +``` +yarn workspace @foxglove/mcap-benchmarks bench +``` + +Run benchmarks with Chrome debugger attached to use profiling tools: + +``` +yarn workspace @foxglove/mcap-benchmarks bench:debug +``` diff --git a/typescript/README.md b/typescript/README.md index f05e71c0aa..b90b762053 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -1,33 +1,8 @@ # TypeScript libraries for MCAP -Install dependencies: +[MCAP](https://github.com/foxglove/mcap) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. -``` -corepack enable -yarn install -``` +The following NPM packages are provided for use with JavaScript and TypeScript: -Run lint/tests: - -``` -yarn workspace @mcap/core lint -yarn workspace @mcap/core test -``` - -Read and validate an MCAP file: - -``` -yarn workspace @foxglove/mcap-example-validate validate file.mcap -``` - -Run benchmarks: - -``` -yarn workspace @foxglove/mcap-benchmarks bench -``` - -Run benchmarks with Chrome debugger attached to use profiling tools: - -``` -yarn workspace @foxglove/mcap-benchmarks bench:debug -``` +- **@mcap/core** – low-level readers and writers +- **@mcap/support** – support for well-known compression formats and encodings, and Node.js and browser environments diff --git a/typescript/core/README.md b/typescript/core/README.md index 160403fd35..852080048c 100644 --- a/typescript/core/README.md +++ b/typescript/core/README.md @@ -7,3 +7,11 @@ The `@mcap/core` package provides low-level readers and writers for the MCAP for ## Examples Examples of how to use the `@mcap/core` APIs can be found in the [TypeScript examples folder](https://github.com/foxglove/mcap/tree/main/typescript/examples) in the MCAP repo. + +## License + +`@mcap/core` is licensed under the [MIT License](https://opensource.org/licenses/MIT). + +## Stay in touch + +Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/typescript/core/jest.config.json b/typescript/core/jest.config.json index c5dcd8fa38..57e9583c8d 100644 --- a/typescript/core/jest.config.json +++ b/typescript/core/jest.config.json @@ -1,15 +1,15 @@ { "testMatch": ["/src/**/*.test.ts"], "transform": { - "^.+\\.ts$": "ts-jest" - }, - "globals": { - "ts-jest": { - "diagnostics": { - "//": "add 6133 (unused variables) to default ignore codes", - "ignoreCodes": [6059, 18002, 18003, 6133] + "^.+\\.ts$": [ + "ts-jest", + { + "diagnostics": { + "//": "add 6133 (unused variables) to default ignore codes", + "ignoreCodes": [6059, 18002, 18003, 6133] + } } - } + ] }, "//": "Native find is slow because it does not exclude files: https://github.com/facebook/jest/pull/11264#issuecomment-825377579", "haste": { "forceNodeFilesystemAPI": true } diff --git a/typescript/core/package.json b/typescript/core/package.json index 477e3e4a0c..6ef2d7c6d7 100644 --- a/typescript/core/package.json +++ b/typescript/core/package.json @@ -15,13 +15,16 @@ "module": "dist/esm/src/index.js", "main": "dist/cjs/src/index.js", "typings": "dist/esm/src/index.d.ts", - "typedocMain": "src/index.ts", + "typedoc": { + "entryPoint": "src/index.ts" + }, "files": [ "dist", "src" ], "scripts": { - "prepack": "tsc -b tsconfig.json tsconfig.cjs.json", + "build": "tsc -b tsconfig.json tsconfig.cjs.json", + "prepack": "yarn build", "typecheck": "tsc -p tsconfig.json --noEmit", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", diff --git a/typescript/core/tsconfig.json b/typescript/core/tsconfig.json index c9f6292197..5dd61a6d17 100644 --- a/typescript/core/tsconfig.json +++ b/typescript/core/tsconfig.json @@ -4,6 +4,8 @@ "compilerOptions": { "rootDir": ".", "outDir": "./dist/esm", - "lib": ["es2020", "dom"] + "lib": ["es2020", "dom"], + "composite": true, + "incremental": true } } diff --git a/typescript/examples/bag2mcap/package.json b/typescript/examples/bag2mcap/package.json index 0d91d2b72d..7891900b50 100644 --- a/typescript/examples/bag2mcap/package.json +++ b/typescript/examples/bag2mcap/package.json @@ -17,7 +17,7 @@ "typecheck": "tsc -p tsconfig.json --noEmit", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", - "bag2mcap": "ts-node --files --project tsconfig.cjs.json scripts/bag2mcap.ts" + "bag2mcap": "yarn typescript:build && ts-node --files --project tsconfig.cjs.json scripts/bag2mcap.ts" }, "devDependencies": { "@foxglove/eslint-plugin": "0.21.0", @@ -31,7 +31,6 @@ "@foxglove/wasm-zstd": "1.0.1", "@mcap/core": "*", "@mcap/support": "*", - "@types/jest": "29.4.0", "@types/lodash": "4.14.191", "@types/node": "18.13.0", "@typescript-eslint/eslint-plugin": "5.52.0", @@ -42,13 +41,10 @@ "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", "eslint-plugin-import": "2.27.5", - "eslint-plugin-jest": "27.2.1", "eslint-plugin-prettier": "4.2.1", - "jest": "29.4.3", "lodash": "4.17.21", "prettier": "2.8.4", "protobufjs": "7.2.2", - "ts-jest": "29.0.5", "ts-node": "10.9.1", "tsconfig-paths": "4.1.2", "typescript": "4.9.5" diff --git a/typescript/examples/basicwriter/package.json b/typescript/examples/basicwriter/package.json index e6c007f434..bf1a291e54 100644 --- a/typescript/examples/basicwriter/package.json +++ b/typescript/examples/basicwriter/package.json @@ -17,7 +17,7 @@ "typecheck": "tsc -p tsconfig.json --noEmit", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", - "main": "ts-node --files --project tsconfig.cjs.json scripts/main.ts" + "main": "yarn typescript:build && ts-node --files --project tsconfig.cjs.json scripts/main.ts" }, "devDependencies": { "@foxglove/eslint-plugin": "0.21.0", diff --git a/typescript/examples/flatbufferswriter/package.json b/typescript/examples/flatbufferswriter/package.json index 1082e76c6c..966c881c26 100644 --- a/typescript/examples/flatbufferswriter/package.json +++ b/typescript/examples/flatbufferswriter/package.json @@ -17,7 +17,7 @@ "typecheck": "tsc -p tsconfig.json --noEmit", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", - "main": "ts-node --files --project tsconfig.cjs.json scripts/main.ts" + "main": "yarn typescript:build && ts-node --files --project tsconfig.cjs.json scripts/main.ts" }, "devDependencies": { "@foxglove/eslint-plugin": "0.21.0", diff --git a/typescript/examples/validate/package.json b/typescript/examples/validate/package.json index 001573322f..da3343f970 100644 --- a/typescript/examples/validate/package.json +++ b/typescript/examples/validate/package.json @@ -17,7 +17,7 @@ "typecheck": "tsc -p tsconfig.json --noEmit", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", - "validate": "ts-node --files --project tsconfig.cjs.json scripts/validate.ts" + "validate": "yarn typescript:build && ts-node --files --project tsconfig.cjs.json scripts/validate.ts" }, "devDependencies": { "@foxglove/eslint-plugin": "0.21.0", @@ -31,7 +31,6 @@ "@foxglove/wasm-zstd": "1.0.1", "@mcap/core": "*", "@mcap/support": "*", - "@types/jest": "29.4.0", "@types/lodash": "4.14.191", "@types/node": "18.13.0", "@typescript-eslint/eslint-plugin": "5.52.0", @@ -42,13 +41,10 @@ "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", "eslint-plugin-import": "2.27.5", - "eslint-plugin-jest": "27.2.1", "eslint-plugin-prettier": "4.2.1", - "jest": "29.4.3", "lodash": "4.17.21", "prettier": "2.8.4", "protobufjs": "7.2.2", - "ts-jest": "29.0.5", "ts-node": "10.9.1", "tsconfig-paths": "4.1.2", "typescript": "4.9.5" diff --git a/typescript/examples/validate/scripts/validate.ts b/typescript/examples/validate/scripts/validate.ts index cd1837c2ff..c6740449ef 100644 --- a/typescript/examples/validate/scripts/validate.ts +++ b/typescript/examples/validate/scripts/validate.ts @@ -9,6 +9,7 @@ import { McapTypes, } from "@mcap/core"; import { loadDecompressHandlers, protobufFromBinaryDescriptor } from "@mcap/support"; +import { FileHandleReadable } from "@mcap/support/nodejs"; import { program } from "commander"; import { createReadStream } from "fs"; import fs from "fs/promises"; @@ -221,29 +222,8 @@ async function validate( if (!stream) { const handle = await fs.open(filePath, "r"); try { - let buffer = new ArrayBuffer(4096); const reader = await McapIndexedReader.Initialize({ - readable: { - size: async () => BigInt((await handle.stat()).size), - read: async (offset, length) => { - if (offset > Number.MAX_SAFE_INTEGER || length > Number.MAX_SAFE_INTEGER) { - throw new Error(`Read too large: offset ${offset}, length ${length}`); - } - if (length > buffer.byteLength) { - buffer = new ArrayBuffer(Number(length * 2n)); - } - const result = await handle.read({ - buffer: new DataView(buffer, 0, Number(length)), - position: Number(offset), - }); - if (result.bytesRead !== Number(length)) { - throw new Error( - `Read only ${result.bytesRead} bytes from offset ${offset}, expected ${length}`, - ); - } - return new Uint8Array(result.buffer.buffer, result.buffer.byteOffset, result.bytesRead); - }, - }, + readable: new FileHandleReadable(handle), decompressHandlers, }); for (const record of reader.schemasById.values()) { diff --git a/typescript/support/README.md b/typescript/support/README.md new file mode 100644 index 0000000000..7524dcbc24 --- /dev/null +++ b/typescript/support/README.md @@ -0,0 +1,53 @@ +# @mcap/support + +[MCAP](https://github.com/foxglove/mcap) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. + +The `@mcap/support` package provides utilities for working with MCAP files that use [well-known compression formats and encodings](https://mcap.dev/specification/appendix.html), from Node.js and browsers. + +## Usage examples + +### Reading MCAP files in a browser + +TODO + +### Reading MCAP files in Node.js + +```ts +import { loadDecompressHandlers } from "@mcap/support"; +import { FileHandleReadable } from "@mcap/support/nodejs"; +const decompressHandlers = await loadDecompressHandlers(); + +const fileHandle = await open("file.mcap", "r"); + +const reader = await McapIndexedReader.Initialize({ + readable: new FileHandleReadable(fileHandle), + decompressHandlers, +}); +``` + +### Writing MCAP files with Node.js + +```ts +import zstd from "@foxglove/wasm-zstd"; +import { FileHandleWritable } from "@mcap/support/nodejs"; +import { open } from "fs/promises"; + +const fileHandle = await open("file.mcap", "w"); + +await zstd.isLoaded; +const writer = new McapWriter({ + writable: new FileHandleWritable(fileHandle), + compressChunk: (data) => ({ + compression: "zstd", + compressedData: zstd.compress(data), + }), +}); +``` + +## License + +`@mcap/support` is licensed under the [MIT License](https://opensource.org/licenses/MIT). + +## Stay in touch + +Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/typescript/support/jest.config.json b/typescript/support/jest.config.json index c5dcd8fa38..57e9583c8d 100644 --- a/typescript/support/jest.config.json +++ b/typescript/support/jest.config.json @@ -1,15 +1,15 @@ { "testMatch": ["/src/**/*.test.ts"], "transform": { - "^.+\\.ts$": "ts-jest" - }, - "globals": { - "ts-jest": { - "diagnostics": { - "//": "add 6133 (unused variables) to default ignore codes", - "ignoreCodes": [6059, 18002, 18003, 6133] + "^.+\\.ts$": [ + "ts-jest", + { + "diagnostics": { + "//": "add 6133 (unused variables) to default ignore codes", + "ignoreCodes": [6059, 18002, 18003, 6133] + } } - } + ] }, "//": "Native find is slow because it does not exclude files: https://github.com/facebook/jest/pull/11264#issuecomment-825377579", "haste": { "forceNodeFilesystemAPI": true } diff --git a/typescript/support/package.json b/typescript/support/package.json index 15c0662dd8..2aa98b022b 100644 --- a/typescript/support/package.json +++ b/typescript/support/package.json @@ -12,20 +12,22 @@ "email": "support@foxglove.dev" }, "homepage": "https://foxglove.dev/", - "module": "dist/esm/index.js", - "main": "dist/cjs/index.js", - "typings": "dist/esm/index.d.ts", + "module": "dist/esm/src/index.js", + "main": "dist/cjs/src/index.js", + "typings": "dist/esm/src/index.d.ts", "exports": { ".": { - "import": "./dist/esm/index.js", - "require": "./dist/cjs/index.js" + "import": "./dist/esm/src/index.js", + "require": "./dist/cjs/src/index.js" }, "./nodejs": { - "import": "./dist/esm/nodejs/index.js", - "require": "./dist/cjs/nodejs/index.js" + "import": "./dist/esm/src/nodejs/index.js", + "require": "./dist/cjs/src/nodejs/index.js" } }, - "typedocMain": "src/index.ts", + "typedoc": { + "entryPoint": "src/index.ts" + }, "files": [ "dist", "src", @@ -33,7 +35,8 @@ "nodejs.js" ], "scripts": { - "prepack": "tsc -b tsconfig.json tsconfig.cjs.json", + "build": "tsc -b tsconfig.json tsconfig.cjs.json", + "prepack": "yarn build", "typecheck": "tsc -p tsconfig.json --noEmit", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", @@ -42,6 +45,7 @@ "devDependencies": { "@foxglove/eslint-plugin": "0.21.0", "@foxglove/tsconfig": "1.1.0", + "@mcap/core": "workspace:*", "@types/jest": "29.4.0", "@types/node": "18.13.0", "@typescript-eslint/eslint-plugin": "5.52.0", diff --git a/typescript/support/src/nodejs/FileHandleReadable.ts b/typescript/support/src/nodejs/FileHandleReadable.ts new file mode 100644 index 0000000000..6196335042 --- /dev/null +++ b/typescript/support/src/nodejs/FileHandleReadable.ts @@ -0,0 +1,37 @@ +import { FileHandle } from "fs/promises"; +import type { McapTypes } from "@mcap/core"; + +/** + * IReadable implementation for FileHandle. + */ +export class FileHandleReadable implements McapTypes.IReadable { + private handle: FileHandle; + private buffer = new ArrayBuffer(4096); + + constructor(handle: FileHandle) { + this.handle = handle; + } + + async size(): Promise { + return BigInt((await this.handle.stat()).size); + } + + async read(offset: bigint, length: bigint): Promise { + if (offset > Number.MAX_SAFE_INTEGER || length > Number.MAX_SAFE_INTEGER) { + throw new Error(`Read too large: offset ${offset}, length ${length}`); + } + if (length > this.buffer.byteLength) { + this.buffer = new ArrayBuffer(Number(length * 2n)); + } + const result = await this.handle.read({ + buffer: new DataView(this.buffer, 0, Number(length)), + position: Number(offset), + }); + if (result.bytesRead !== Number(length)) { + throw new Error( + `Read only ${result.bytesRead} bytes from offset ${offset}, expected ${length}`, + ); + } + return new Uint8Array(result.buffer.buffer, result.buffer.byteOffset, result.bytesRead); + } +} diff --git a/typescript/support/src/nodejs/FileHandleWritable.ts b/typescript/support/src/nodejs/FileHandleWritable.ts index dc2fdbcfcd..4efb7a9588 100644 --- a/typescript/support/src/nodejs/FileHandleWritable.ts +++ b/typescript/support/src/nodejs/FileHandleWritable.ts @@ -1,10 +1,6 @@ +import type { IWritable } from "@mcap/core"; import { FileHandle } from "fs/promises"; -interface IWritable { - write(buffer: Uint8Array): Promise; - position(): bigint; -} - /** * IWritable implementation for FileHandle. */ diff --git a/typescript/support/src/nodejs/index.ts b/typescript/support/src/nodejs/index.ts index 7f4bb8327f..cec1cee26c 100644 --- a/typescript/support/src/nodejs/index.ts +++ b/typescript/support/src/nodejs/index.ts @@ -1 +1,2 @@ +export * from "./FileHandleReadable"; export * from "./FileHandleWritable"; diff --git a/typescript/support/tsconfig.json b/typescript/support/tsconfig.json index 69d8b14bed..6a6b398a2e 100644 --- a/typescript/support/tsconfig.json +++ b/typescript/support/tsconfig.json @@ -2,8 +2,10 @@ "extends": "@foxglove/tsconfig/base", "include": ["./src/**/*"], "compilerOptions": { - "rootDir": "./src", "outDir": "./dist/esm", - "lib": ["es2020", "dom"] - } + "lib": ["es2020", "dom"], + "composite": true, + "incremental": true + }, + "references": [{ "path": "../core" }] } diff --git a/typescript/typedoc.json b/typescript/typedoc.json new file mode 100644 index 0000000000..6969e3d175 --- /dev/null +++ b/typescript/typedoc.json @@ -0,0 +1,5 @@ +{ + "name": "MCAP TypeScript SDK", + "entryPointStrategy": "packages", + "entryPoints": ["./core", "./support"] +} diff --git a/yarn.lock b/yarn.lock index 901b39c200..2b45ede496 100644 --- a/yarn.lock +++ b/yarn.lock @@ -59,25 +59,7 @@ __metadata: languageName: node linkType: hard -"@babel/code-frame@npm:^7.0.0": - version: 7.16.7 - resolution: "@babel/code-frame@npm:7.16.7" - dependencies: - "@babel/highlight": ^7.16.7 - checksum: db2f7faa31bc2c9cf63197b481b30ea57147a5fc1a6fab60e5d6c02cdfbf6de8e17b5121f99917b3dabb5eeb572da078312e70697415940383efc140d4e0808b - languageName: node - linkType: hard - -"@babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.16.0": - version: 7.16.0 - resolution: "@babel/code-frame@npm:7.16.0" - dependencies: - "@babel/highlight": ^7.16.0 - checksum: 8961d0302ec6b8c2e9751a11e06a17617425359fd1645e4dae56a90a03464c68a0916115100fbcd030961870313f21865d0b85858360a2c68aabdda744393607 - languageName: node - linkType: hard - -"@babel/code-frame@npm:^7.18.6": +"@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.18.6": version: 7.18.6 resolution: "@babel/code-frame@npm:7.18.6" dependencies: @@ -86,13 +68,6 @@ __metadata: languageName: node linkType: hard -"@babel/compat-data@npm:^7.16.0": - version: 7.16.4 - resolution: "@babel/compat-data@npm:7.16.4" - checksum: 4949ce54eafc4b38d5623696a872acaaced1a523605708d81c2c483253941917d90dae0de40fc01e152ae56075dadd89c23014da5a632b09c001a716fa689cae - languageName: node - linkType: hard - "@babel/compat-data@npm:^7.20.5": version: 7.20.14 resolution: "@babel/compat-data@npm:7.20.14" @@ -100,7 +75,7 @@ __metadata: languageName: node linkType: hard -"@babel/core@npm:^7.11.6": +"@babel/core@npm:^7.11.6, @babel/core@npm:^7.12.3": version: 7.20.12 resolution: "@babel/core@npm:7.20.12" dependencies: @@ -123,41 +98,7 @@ __metadata: languageName: node linkType: hard -"@babel/core@npm:^7.12.3": - version: 7.16.0 - resolution: "@babel/core@npm:7.16.0" - dependencies: - "@babel/code-frame": ^7.16.0 - "@babel/generator": ^7.16.0 - "@babel/helper-compilation-targets": ^7.16.0 - "@babel/helper-module-transforms": ^7.16.0 - "@babel/helpers": ^7.16.0 - "@babel/parser": ^7.16.0 - "@babel/template": ^7.16.0 - "@babel/traverse": ^7.16.0 - "@babel/types": ^7.16.0 - convert-source-map: ^1.7.0 - debug: ^4.1.0 - gensync: ^1.0.0-beta.2 - json5: ^2.1.2 - semver: ^6.3.0 - source-map: ^0.5.0 - checksum: a140f669daa90c774016a76b1f85641975333c1c219ae0a8e65d8b4c316836e918276e0dfd55613b14f8e578406a92393d4368a63bdd5d0708122976ee2ee8e3 - languageName: node - linkType: hard - -"@babel/generator@npm:^7.16.0, @babel/generator@npm:^7.7.2": - version: 7.16.0 - resolution: "@babel/generator@npm:7.16.0" - dependencies: - "@babel/types": ^7.16.0 - jsesc: ^2.5.1 - source-map: ^0.5.0 - checksum: 9ff53e0db72a225c8783c4a277698b4efcead750542ebb9cff31732ba62d092090715a772df10a323446924712f6928ad60c03db4e7051bed3a9701b552d51fb - languageName: node - linkType: hard - -"@babel/generator@npm:^7.20.7": +"@babel/generator@npm:^7.20.7, @babel/generator@npm:^7.7.2": version: 7.20.14 resolution: "@babel/generator@npm:7.20.14" dependencies: @@ -168,20 +109,6 @@ __metadata: languageName: node linkType: hard -"@babel/helper-compilation-targets@npm:^7.16.0": - version: 7.16.3 - resolution: "@babel/helper-compilation-targets@npm:7.16.3" - dependencies: - "@babel/compat-data": ^7.16.0 - "@babel/helper-validator-option": ^7.14.5 - browserslist: ^4.17.5 - semver: ^6.3.0 - peerDependencies: - "@babel/core": ^7.0.0 - checksum: 038bcd43ac914371c51bf6e72b5cedcae432f0d359285d74a9133c6a839bd625a7d5412d7471d50aa78a3e1c79b0a692b50a8d6a1299ebf69733b512ff199323 - languageName: node - linkType: hard - "@babel/helper-compilation-targets@npm:^7.20.7": version: 7.20.7 resolution: "@babel/helper-compilation-targets@npm:7.20.7" @@ -204,17 +131,6 @@ __metadata: languageName: node linkType: hard -"@babel/helper-function-name@npm:^7.16.0": - version: 7.16.0 - resolution: "@babel/helper-function-name@npm:7.16.0" - dependencies: - "@babel/helper-get-function-arity": ^7.16.0 - "@babel/template": ^7.16.0 - "@babel/types": ^7.16.0 - checksum: 8c02371d28678f3bb492e69d4635b2fe6b1c5a93ce129bf883f1fafde2005f4dbc0e643f52103ca558b698c0774bfb84a93f188d71db1c077f754b6220629b92 - languageName: node - linkType: hard - "@babel/helper-function-name@npm:^7.19.0": version: 7.19.0 resolution: "@babel/helper-function-name@npm:7.19.0" @@ -225,24 +141,6 @@ __metadata: languageName: node linkType: hard -"@babel/helper-get-function-arity@npm:^7.16.0": - version: 7.16.0 - resolution: "@babel/helper-get-function-arity@npm:7.16.0" - dependencies: - "@babel/types": ^7.16.0 - checksum: 1a68322c7b5fdffb1b51df32f7a53b1ff2268b5b99d698f0a1a426dcb355482a44ef3dae982a507907ba975314638dabb6d77ac1778098bdbe99707e6c29cae8 - languageName: node - linkType: hard - -"@babel/helper-hoist-variables@npm:^7.16.0": - version: 7.16.0 - resolution: "@babel/helper-hoist-variables@npm:7.16.0" - dependencies: - "@babel/types": ^7.16.0 - checksum: 2ee5b400c267c209a53c90eea406a8f09c30d4d7a2b13e304289d858a2e34a99272c062cfad6dad63705662943951c42ff20042ef539b2d3c4f8743183a28954 - languageName: node - linkType: hard - "@babel/helper-hoist-variables@npm:^7.18.6": version: 7.18.6 resolution: "@babel/helper-hoist-variables@npm:7.18.6" @@ -252,24 +150,6 @@ __metadata: languageName: node linkType: hard -"@babel/helper-member-expression-to-functions@npm:^7.16.0": - version: 7.16.0 - resolution: "@babel/helper-member-expression-to-functions@npm:7.16.0" - dependencies: - "@babel/types": ^7.16.0 - checksum: 58ef8e3a4af0c1dc43a2011f43f25502877ac1c5aa9a4a6586f0265ab857b65831f60560044bc9380df43c91ac21cad39a84095b91764b433d1acf18d27e38d6 - languageName: node - linkType: hard - -"@babel/helper-module-imports@npm:^7.16.0": - version: 7.16.0 - resolution: "@babel/helper-module-imports@npm:7.16.0" - dependencies: - "@babel/types": ^7.16.0 - checksum: 8e1eb9ac39440e52080b87c78d8d318e7c93658bdd0f3ce0019c908de88cbddafdc241f392898c0b0ba81fc52c8c6d2f9cc1b163ac5ed2a474d49b11646b7516 - languageName: node - linkType: hard - "@babel/helper-module-imports@npm:^7.18.6": version: 7.18.6 resolution: "@babel/helper-module-imports@npm:7.18.6" @@ -279,22 +159,6 @@ __metadata: languageName: node linkType: hard -"@babel/helper-module-transforms@npm:^7.16.0": - version: 7.16.0 - resolution: "@babel/helper-module-transforms@npm:7.16.0" - dependencies: - "@babel/helper-module-imports": ^7.16.0 - "@babel/helper-replace-supers": ^7.16.0 - "@babel/helper-simple-access": ^7.16.0 - "@babel/helper-split-export-declaration": ^7.16.0 - "@babel/helper-validator-identifier": ^7.15.7 - "@babel/template": ^7.16.0 - "@babel/traverse": ^7.16.0 - "@babel/types": ^7.16.0 - checksum: a3d0e5556f26ebdf2ae422af3b9a1ba1848fead891f46bcd1c6a4be88ad8e9f348140f81d1843a3481574be1643a9c79b01469231f5b5801f5d5e691efdd11f3 - languageName: node - linkType: hard - "@babel/helper-module-transforms@npm:^7.20.11": version: 7.20.11 resolution: "@babel/helper-module-transforms@npm:7.20.11" @@ -311,50 +175,13 @@ __metadata: languageName: node linkType: hard -"@babel/helper-optimise-call-expression@npm:^7.16.0": - version: 7.16.0 - resolution: "@babel/helper-optimise-call-expression@npm:7.16.0" - dependencies: - "@babel/types": ^7.16.0 - checksum: 121ae6054fcec76ed2c4dd83f0281b901c1e3cfac1bbff79adc3667983903ad1030a0ad9a8bea58e52b225e13881cf316f371c65276976e7a6762758a98be8f6 - languageName: node - linkType: hard - -"@babel/helper-plugin-utils@npm:^7.0.0, @babel/helper-plugin-utils@npm:^7.10.4, @babel/helper-plugin-utils@npm:^7.12.13, @babel/helper-plugin-utils@npm:^7.14.5, @babel/helper-plugin-utils@npm:^7.8.0": - version: 7.14.5 - resolution: "@babel/helper-plugin-utils@npm:7.14.5" - checksum: fe20e90a24d02770a60ebe80ab9f0dfd7258503cea8006c71709ac9af1aa3e47b0de569499673f11ea6c99597f8c0e4880ae1d505986e61101b69716820972fe - languageName: node - linkType: hard - -"@babel/helper-plugin-utils@npm:^7.18.6": +"@babel/helper-plugin-utils@npm:^7.0.0, @babel/helper-plugin-utils@npm:^7.10.4, @babel/helper-plugin-utils@npm:^7.12.13, @babel/helper-plugin-utils@npm:^7.14.5, @babel/helper-plugin-utils@npm:^7.18.6, @babel/helper-plugin-utils@npm:^7.8.0": version: 7.20.2 resolution: "@babel/helper-plugin-utils@npm:7.20.2" checksum: f6cae53b7fdb1bf3abd50fa61b10b4470985b400cc794d92635da1e7077bb19729f626adc0741b69403d9b6e411cddddb9c0157a709cc7c4eeb41e663be5d74b languageName: node linkType: hard -"@babel/helper-replace-supers@npm:^7.16.0": - version: 7.16.0 - resolution: "@babel/helper-replace-supers@npm:7.16.0" - dependencies: - "@babel/helper-member-expression-to-functions": ^7.16.0 - "@babel/helper-optimise-call-expression": ^7.16.0 - "@babel/traverse": ^7.16.0 - "@babel/types": ^7.16.0 - checksum: 61f04bbe05ff0987d5a8d5253cb101d47004a27951d6c5cd95457e30fcb3adaca85f0bcaa7f31f4d934f22386b935ac7281398c68982d4a4768769d95c028460 - languageName: node - linkType: hard - -"@babel/helper-simple-access@npm:^7.16.0": - version: 7.16.0 - resolution: "@babel/helper-simple-access@npm:7.16.0" - dependencies: - "@babel/types": ^7.16.0 - checksum: 2d7155f318411788b42d2f4a3d406de12952ad620d0bd411a0f3b5803389692ad61d9e7fab5f93b23ad3d8a09db4a75ca9722b9873a606470f468bc301944af6 - languageName: node - linkType: hard - "@babel/helper-simple-access@npm:^7.20.2": version: 7.20.2 resolution: "@babel/helper-simple-access@npm:7.20.2" @@ -364,15 +191,6 @@ __metadata: languageName: node linkType: hard -"@babel/helper-split-export-declaration@npm:^7.16.0": - version: 7.16.0 - resolution: "@babel/helper-split-export-declaration@npm:7.16.0" - dependencies: - "@babel/types": ^7.16.0 - checksum: 8bd87b5ea2046b145f0f55bc75cbdb6df69eaeb32919ee3c1c758757025aebca03e567a4d48389eb4f16a55021adb6ed8fa58aa771e164b15fa5e0a0722f771d - languageName: node - linkType: hard - "@babel/helper-split-export-declaration@npm:^7.18.6": version: 7.18.6 resolution: "@babel/helper-split-export-declaration@npm:7.18.6" @@ -389,20 +207,6 @@ __metadata: languageName: node linkType: hard -"@babel/helper-validator-identifier@npm:^7.15.7": - version: 7.15.7 - resolution: "@babel/helper-validator-identifier@npm:7.15.7" - checksum: f041c28c531d1add5cc345b25d5df3c29c62bce3205b4d4a93dcd164ccf630350acba252d374fad8f5d8ea526995a215829f27183ba7ce7ce141843bf23068a6 - languageName: node - linkType: hard - -"@babel/helper-validator-identifier@npm:^7.16.7": - version: 7.16.7 - resolution: "@babel/helper-validator-identifier@npm:7.16.7" - checksum: dbb3db9d184343152520a209b5684f5e0ed416109cde82b428ca9c759c29b10c7450657785a8b5c5256aa74acc6da491c1f0cf6b784939f7931ef82982051b69 - languageName: node - linkType: hard - "@babel/helper-validator-identifier@npm:^7.18.6, @babel/helper-validator-identifier@npm:^7.19.1": version: 7.19.1 resolution: "@babel/helper-validator-identifier@npm:7.19.1" @@ -410,13 +214,6 @@ __metadata: languageName: node linkType: hard -"@babel/helper-validator-option@npm:^7.14.5": - version: 7.14.5 - resolution: "@babel/helper-validator-option@npm:7.14.5" - checksum: 1b25c34a5cb3d8602280f33b9ab687d2a77895e3616458d0f70ddc450ada9b05e342c44f322bc741d51b252e84cff6ec44ae93d622a3354828579a643556b523 - languageName: node - linkType: hard - "@babel/helper-validator-option@npm:^7.18.6": version: 7.18.6 resolution: "@babel/helper-validator-option@npm:7.18.6" @@ -424,17 +221,6 @@ __metadata: languageName: node linkType: hard -"@babel/helpers@npm:^7.16.0": - version: 7.16.3 - resolution: "@babel/helpers@npm:7.16.3" - dependencies: - "@babel/template": ^7.16.0 - "@babel/traverse": ^7.16.3 - "@babel/types": ^7.16.0 - checksum: b725b1aab734e9e1407247ee499880583855843fa2855377a2c26277bd9fbd7080219109189bc69b18d71cc30759666bfe66d534729b41452097866d1f5a66ef - languageName: node - linkType: hard - "@babel/helpers@npm:^7.20.7": version: 7.20.13 resolution: "@babel/helpers@npm:7.20.13" @@ -446,28 +232,6 @@ __metadata: languageName: node linkType: hard -"@babel/highlight@npm:^7.16.0": - version: 7.16.0 - resolution: "@babel/highlight@npm:7.16.0" - dependencies: - "@babel/helper-validator-identifier": ^7.15.7 - chalk: ^2.0.0 - js-tokens: ^4.0.0 - checksum: abf244c48fcff20ec87830e8b99c776f4dcdd9138e63decc195719a94148da35339639e0d8045eb9d1f3e67a39ab90a9c3f5ce2d579fb1a0368d911ddf29b4e5 - languageName: node - linkType: hard - -"@babel/highlight@npm:^7.16.7": - version: 7.16.10 - resolution: "@babel/highlight@npm:7.16.10" - dependencies: - "@babel/helper-validator-identifier": ^7.16.7 - chalk: ^2.0.0 - js-tokens: ^4.0.0 - checksum: 1f1bdd752a90844f4efc22166a46303fb651ba0fd75a06daba3ebae2575ab3edc1da9827c279872a3aaf305f50a18473c5fa1966752726a2b253065fd4c0745e - languageName: node - linkType: hard - "@babel/highlight@npm:^7.18.6": version: 7.18.6 resolution: "@babel/highlight@npm:7.18.6" @@ -479,16 +243,7 @@ __metadata: languageName: node linkType: hard -"@babel/parser@npm:^7.1.0, @babel/parser@npm:^7.14.7, @babel/parser@npm:^7.16.0, @babel/parser@npm:^7.16.3": - version: 7.16.4 - resolution: "@babel/parser@npm:7.16.4" - bin: - parser: ./bin/babel-parser.js - checksum: ce0a8f92f440f2a12bc932f070a7b60c5133bf8a63f461841f9e39af0194f573707959d606c6fad1a2fd496a45148553afd9b74d3b8dd36cdb7861598d1f3e36 - languageName: node - linkType: hard - -"@babel/parser@npm:^7.20.13, @babel/parser@npm:^7.20.7": +"@babel/parser@npm:^7.1.0, @babel/parser@npm:^7.14.7, @babel/parser@npm:^7.20.13, @babel/parser@npm:^7.20.7": version: 7.20.15 resolution: "@babel/parser@npm:7.20.15" bin: @@ -651,18 +406,7 @@ __metadata: languageName: node linkType: hard -"@babel/template@npm:^7.16.0, @babel/template@npm:^7.3.3": - version: 7.16.0 - resolution: "@babel/template@npm:7.16.0" - dependencies: - "@babel/code-frame": ^7.16.0 - "@babel/parser": ^7.16.0 - "@babel/types": ^7.16.0 - checksum: 940f105cc6a6aee638cd8cfae80b8b80811e0ddd53b6a11f3a68431ebb998564815fb26511b5d9cb4cff66ea67130ba7498555ee015375d32f5f89ceaa6662ea - languageName: node - linkType: hard - -"@babel/template@npm:^7.18.10, @babel/template@npm:^7.20.7": +"@babel/template@npm:^7.18.10, @babel/template@npm:^7.20.7, @babel/template@npm:^7.3.3": version: 7.20.7 resolution: "@babel/template@npm:7.20.7" dependencies: @@ -673,24 +417,7 @@ __metadata: languageName: node linkType: hard -"@babel/traverse@npm:^7.16.0, @babel/traverse@npm:^7.16.3, @babel/traverse@npm:^7.7.2": - version: 7.16.3 - resolution: "@babel/traverse@npm:7.16.3" - dependencies: - "@babel/code-frame": ^7.16.0 - "@babel/generator": ^7.16.0 - "@babel/helper-function-name": ^7.16.0 - "@babel/helper-hoist-variables": ^7.16.0 - "@babel/helper-split-export-declaration": ^7.16.0 - "@babel/parser": ^7.16.3 - "@babel/types": ^7.16.0 - debug: ^4.1.0 - globals: ^11.1.0 - checksum: abb14857b1104c73124612954865e28f95a86eb6741f35851369b4f9eabc17e394c9aa6f21fba6ce23813592353090d409772be828717cbe5154a5e981a753c1 - languageName: node - linkType: hard - -"@babel/traverse@npm:^7.20.10, @babel/traverse@npm:^7.20.12, @babel/traverse@npm:^7.20.13": +"@babel/traverse@npm:^7.20.10, @babel/traverse@npm:^7.20.12, @babel/traverse@npm:^7.20.13, @babel/traverse@npm:^7.7.2": version: 7.20.13 resolution: "@babel/traverse@npm:7.20.13" dependencies: @@ -708,28 +435,7 @@ __metadata: languageName: node linkType: hard -"@babel/types@npm:^7.0.0, @babel/types@npm:^7.16.0, @babel/types@npm:^7.3.0, @babel/types@npm:^7.3.3": - version: 7.16.0 - resolution: "@babel/types@npm:7.16.0" - dependencies: - "@babel/helper-validator-identifier": ^7.15.7 - to-fast-properties: ^2.0.0 - checksum: 5b483da5c6e6f2394fba7ee1da8787a0c9cddd33491271c4da702e49e6faf95ce41d7c8bf9a4ee47f2ef06bdb35096f4d0f6ae4b5bea35ebefe16309d22344b7 - languageName: node - linkType: hard - -"@babel/types@npm:^7.18.6, @babel/types@npm:^7.19.0, @babel/types@npm:^7.20.2, @babel/types@npm:^7.20.7": - version: 7.20.7 - resolution: "@babel/types@npm:7.20.7" - dependencies: - "@babel/helper-string-parser": ^7.19.4 - "@babel/helper-validator-identifier": ^7.19.1 - to-fast-properties: ^2.0.0 - checksum: b39af241f0b72bba67fd6d0d23914f6faec8c0eba8015c181cbd5ea92e59fc91a52a1ab490d3520c7dbd19ddb9ebb76c476308f6388764f16d8201e37fae6811 - languageName: node - linkType: hard - -"@babel/types@npm:^7.8.3": +"@babel/types@npm:^7.0.0, @babel/types@npm:^7.18.6, @babel/types@npm:^7.19.0, @babel/types@npm:^7.20.2, @babel/types@npm:^7.20.7, @babel/types@npm:^7.3.0, @babel/types@npm:^7.3.3, @babel/types@npm:^7.8.3": version: 7.21.3 resolution: "@babel/types@npm:7.21.3" dependencies: @@ -1250,7 +956,8 @@ __metadata: "@foxglove/crc": ^0.0.3 "@foxglove/eslint-plugin": 0.21.0 "@foxglove/tsconfig": 1.1.0 - "@mcap/core": "*" + "@mcap/core": "workspace:*" + "@mcap/support": "workspace:*" "@types/diff": ^5.0.2 "@types/js-yaml": ^4.0.5 "@types/node": 18.13.0 @@ -1293,7 +1000,6 @@ __metadata: "@foxglove/wasm-zstd": 1.0.1 "@mcap/core": "*" "@mcap/support": "*" - "@types/jest": 29.4.0 "@types/lodash": 4.14.191 "@types/node": 18.13.0 "@typescript-eslint/eslint-plugin": 5.52.0 @@ -1304,13 +1010,10 @@ __metadata: eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 eslint-plugin-import: 2.27.5 - eslint-plugin-jest: 27.2.1 eslint-plugin-prettier: 4.2.1 - jest: 29.4.3 lodash: 4.17.21 prettier: 2.8.4 protobufjs: 7.2.2 - ts-jest: 29.0.5 ts-node: 10.9.1 tsconfig-paths: 4.1.2 typescript: 4.9.5 @@ -1374,7 +1077,6 @@ __metadata: "@foxglove/wasm-zstd": 1.0.1 "@mcap/core": "*" "@mcap/support": "*" - "@types/jest": 29.4.0 "@types/lodash": 4.14.191 "@types/node": 18.13.0 "@typescript-eslint/eslint-plugin": 5.52.0 @@ -1385,13 +1087,10 @@ __metadata: eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 eslint-plugin-import: 2.27.5 - eslint-plugin-jest: 27.2.1 eslint-plugin-prettier: 4.2.1 - jest: 29.4.3 lodash: 4.17.21 prettier: 2.8.4 protobufjs: 7.2.2 - ts-jest: 29.0.5 ts-node: 10.9.1 tsconfig-paths: 4.1.2 typescript: 4.9.5 @@ -1417,15 +1116,6 @@ __metadata: languageName: node linkType: hard -"@foxglove/rosmsg-msgs-common@npm:^2.0.0": - version: 2.1.0 - resolution: "@foxglove/rosmsg-msgs-common@npm:2.1.0" - dependencies: - "@foxglove/rosmsg": ^3.1.0 - checksum: 1e6bcff0f21f431ab41dd703b7b4cbd76007fdea297c1c0d6b9cbddd9affe0f4991ea846be861c6863078555e8927bc64160db5a3a73f3d32d0f14c39e51a285 - languageName: node - linkType: hard - "@foxglove/rosmsg-msgs-common@npm:^3.0.0": version: 3.0.0 resolution: "@foxglove/rosmsg-msgs-common@npm:3.0.0" @@ -1504,17 +1194,7 @@ __metadata: languageName: node linkType: hard -"@foxglove/schemas@npm:^1.0.0": - version: 1.0.0 - resolution: "@foxglove/schemas@npm:1.0.0" - dependencies: - "@foxglove/rosmsg-msgs-common": ^2.0.0 - tslib: ^2 - checksum: c6fa851381441534ef465f247615c61a3963e80d02a94c7b07a49bb9468ab704c43e6fe7f07f83bacaf5bdc8bce8bf96b02fef5e62c9c427c5e10bd060c24fec - languageName: node - linkType: hard - -"@foxglove/schemas@npm:^1.1.0": +"@foxglove/schemas@npm:^1.0.0, @foxglove/schemas@npm:^1.1.0": version: 1.2.0 resolution: "@foxglove/schemas@npm:1.2.0" dependencies: @@ -1836,6 +1516,20 @@ __metadata: languageName: node linkType: hard +"@jest/types@npm:^29.5.0": + version: 29.5.0 + resolution: "@jest/types@npm:29.5.0" + dependencies: + "@jest/schemas": ^29.4.3 + "@types/istanbul-lib-coverage": ^2.0.0 + "@types/istanbul-reports": ^3.0.0 + "@types/node": "*" + "@types/yargs": ^17.0.8 + chalk: ^4.0.0 + checksum: 1811f94b19cf8a9460a289c4f056796cfc373480e0492692a6125a553cd1a63824bd846d7bb78820b7b6f758f6dd3c2d4558293bb676d541b2fa59c70fdf9d39 + languageName: node + linkType: hard + "@jridgewell/gen-mapping@npm:^0.1.0": version: 0.1.1 resolution: "@jridgewell/gen-mapping@npm:0.1.1" @@ -1898,7 +1592,7 @@ __metadata: languageName: node linkType: hard -"@mcap/core@*, @mcap/core@workspace:typescript/core": +"@mcap/core@*, @mcap/core@workspace:*, @mcap/core@workspace:typescript/core": version: 0.0.0-use.local resolution: "@mcap/core@workspace:typescript/core" dependencies: @@ -1928,7 +1622,7 @@ __metadata: languageName: unknown linkType: soft -"@mcap/support@*, @mcap/support@workspace:typescript/support": +"@mcap/support@*, @mcap/support@workspace:*, @mcap/support@workspace:typescript/support": version: 0.0.0-use.local resolution: "@mcap/support@workspace:typescript/support" dependencies: @@ -1942,6 +1636,7 @@ __metadata: "@foxglove/wasm-bz2": ^0.1.1 "@foxglove/wasm-lz4": ^1.0.2 "@foxglove/wasm-zstd": ^1.0.1 + "@mcap/core": "workspace:*" "@protobufjs/base64": ^1.1.2 "@types/jest": 29.4.0 "@types/node": 18.13.0 @@ -2266,27 +1961,13 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:*": - version: 16.11.9 - resolution: "@types/node@npm:16.11.9" - checksum: baec2e6471ee58fd1e9874e6f47ab95a918a0f46b42424392e4c13e3e07f078f8b72b6b48073b0b14f710ed66c5f1b2e497df43f28151000f11f5f299382cfe2 - languageName: node - linkType: hard - -"@types/node@npm:18.13.0": +"@types/node@npm:*, @types/node@npm:18.13.0, @types/node@npm:>=13.7.0": version: 18.13.0 resolution: "@types/node@npm:18.13.0" checksum: 4ea10f8802848b01672bce938f678b6774ca2cee0c9774f12275ab064ae07818419c3e2e41d6257ce7ba846d1ea26c63214aa1dfa4166fa3746291752b8c6416 languageName: node linkType: hard -"@types/node@npm:>=13.7.0": - version: 16.11.11 - resolution: "@types/node@npm:16.11.11" - checksum: 1c472bd63f23ca0e4effc96e6e0c693b83b027f613a1f41b6d1bd7791f65ce171a351e0a5c92575cb59e84ad0a930875397bfbbb91a7c925ddbbb558f7461af8 - languageName: node - linkType: hard - "@types/prettier@npm:^2.1.5": version: 2.4.2 resolution: "@types/prettier@npm:2.4.2" @@ -2495,16 +2176,7 @@ __metadata: languageName: node linkType: hard -"acorn@npm:^8.4.1": - version: 8.6.0 - resolution: "acorn@npm:8.6.0" - bin: - acorn: bin/acorn - checksum: 9d0de73b73cb6ea8ccd8263a8144d9e2c4b6af90ea0c429997538af0ebbe83c5addecee814b2a7f91f7f615d0bd1547cc7137b3fa236ce058adc64feccee850b - languageName: node - linkType: hard - -"acorn@npm:^8.8.0": +"acorn@npm:^8.4.1, acorn@npm:^8.8.0": version: 8.8.2 resolution: "acorn@npm:8.8.2" bin: @@ -2861,7 +2533,7 @@ __metadata: languageName: node linkType: hard -"braces@npm:^3.0.1, braces@npm:^3.0.2, braces@npm:~3.0.2": +"braces@npm:^3.0.2, braces@npm:~3.0.2": version: 3.0.2 resolution: "braces@npm:3.0.2" dependencies: @@ -2870,21 +2542,6 @@ __metadata: languageName: node linkType: hard -"browserslist@npm:^4.17.5": - version: 4.18.1 - resolution: "browserslist@npm:4.18.1" - dependencies: - caniuse-lite: ^1.0.30001280 - electron-to-chromium: ^1.3.896 - escalade: ^3.1.1 - node-releases: ^2.0.1 - picocolors: ^1.0.0 - bin: - browserslist: cli.js - checksum: ae58322deef15960fc2e601d71bc081b571cfab6705999a3d24db5325b9cfadf5f676615f4460207a93e600549c33d60d37b4502007fe9e737b3cc19e20575d5 - languageName: node - linkType: hard - "browserslist@npm:^4.21.3": version: 4.21.5 resolution: "browserslist@npm:4.21.5" @@ -2988,13 +2645,6 @@ __metadata: languageName: node linkType: hard -"caniuse-lite@npm:^1.0.30001280": - version: 1.0.30001282 - resolution: "caniuse-lite@npm:1.0.30001282" - checksum: 62797fd756e88bfa01f0f983bea9de7814293b209456e8f0b20596b03d2880246f63dc90f947a1fa63f92806ebefbb86fc7811dbecb7839927886d07996938be - languageName: node - linkType: hard - "caniuse-lite@npm:^1.0.30001449": version: 1.0.30001456 resolution: "caniuse-lite@npm:1.0.30001456" @@ -3421,7 +3071,7 @@ __metadata: languageName: node linkType: hard -"debug@npm:4, debug@npm:^4.3.3, debug@npm:^4.3.4": +"debug@npm:4, debug@npm:^4.1.0, debug@npm:^4.1.1, debug@npm:^4.3.2, debug@npm:^4.3.3, debug@npm:^4.3.4": version: 4.3.4 resolution: "debug@npm:4.3.4" dependencies: @@ -3442,18 +3092,6 @@ __metadata: languageName: node linkType: hard -"debug@npm:^4.1.0, debug@npm:^4.1.1, debug@npm:^4.3.2": - version: 4.3.2 - resolution: "debug@npm:4.3.2" - dependencies: - ms: 2.1.2 - peerDependenciesMeta: - supports-color: - optional: true - checksum: 820ea160e267e23c953c9ed87e7ad93494d8cda2f7349af5e7e3bb236d23707ee3022f477d5a7d2ee86ef2bf7d60aa9ab22d1f58080d7deb9dccd073585e1e43 - languageName: node - linkType: hard - "dedent@npm:^0.7.0": version: 0.7.0 resolution: "dedent@npm:0.7.0" @@ -3475,16 +3113,7 @@ __metadata: languageName: node linkType: hard -"define-properties@npm:^1.1.3": - version: 1.1.3 - resolution: "define-properties@npm:1.1.3" - dependencies: - object-keys: ^1.0.12 - checksum: da80dba55d0cd76a5a7ab71ef6ea0ebcb7b941f803793e4e0257b384cb772038faa0c31659d244e82c4342edef841c1a1212580006a05a5068ee48223d787317 - languageName: node - linkType: hard - -"define-properties@npm:^1.1.4": +"define-properties@npm:^1.1.3, define-properties@npm:^1.1.4": version: 1.2.0 resolution: "define-properties@npm:1.2.0" dependencies: @@ -3606,13 +3235,6 @@ __metadata: languageName: node linkType: hard -"electron-to-chromium@npm:^1.3.896": - version: 1.3.903 - resolution: "electron-to-chromium@npm:1.3.903" - checksum: 0f96af03efee4691c6e4cf76524baf8ea5dc38cb2e74282e5c6d2dad65bc2aa0e99afdcb386bf26eed4512a2356457fddbd6963167a1ee381fd41e5b29be90ee - languageName: node - linkType: hard - "electron-to-chromium@npm:^1.4.284": version: 1.4.301 resolution: "electron-to-chromium@npm:1.4.301" @@ -3666,35 +3288,7 @@ __metadata: languageName: node linkType: hard -"es-abstract@npm:^1.19.0": - version: 1.19.1 - resolution: "es-abstract@npm:1.19.1" - dependencies: - call-bind: ^1.0.2 - es-to-primitive: ^1.2.1 - function-bind: ^1.1.1 - get-intrinsic: ^1.1.1 - get-symbol-description: ^1.0.0 - has: ^1.0.3 - has-symbols: ^1.0.2 - internal-slot: ^1.0.3 - is-callable: ^1.2.4 - is-negative-zero: ^2.0.1 - is-regex: ^1.1.4 - is-shared-array-buffer: ^1.0.1 - is-string: ^1.0.7 - is-weakref: ^1.0.1 - object-inspect: ^1.11.0 - object-keys: ^1.1.1 - object.assign: ^4.1.2 - string.prototype.trimend: ^1.0.4 - string.prototype.trimstart: ^1.0.4 - unbox-primitive: ^1.0.1 - checksum: b6be8410672c5364db3fb01eb786e30c7b4bb32b4af63d381c08840f4382c4a168e7855cd338bf59d4f1a1a1138f4d748d1fd40ec65aaa071876f9e9fbfed949 - languageName: node - linkType: hard - -"es-abstract@npm:^1.20.4": +"es-abstract@npm:^1.19.0, es-abstract@npm:^1.20.4": version: 1.21.1 resolution: "es-abstract@npm:1.21.1" dependencies: @@ -4371,18 +3965,7 @@ __metadata: languageName: node linkType: hard -"get-intrinsic@npm:^1.0.2, get-intrinsic@npm:^1.1.0, get-intrinsic@npm:^1.1.1": - version: 1.1.1 - resolution: "get-intrinsic@npm:1.1.1" - dependencies: - function-bind: ^1.1.1 - has: ^1.0.3 - has-symbols: ^1.0.1 - checksum: a9fe2ca8fa3f07f9b0d30fb202bcd01f3d9b9b6b732452e79c48e79f7d6d8d003af3f9e38514250e3553fdc83c61650851cb6870832ac89deaaceb08e3721a17 - languageName: node - linkType: hard - -"get-intrinsic@npm:^1.1.3, get-intrinsic@npm:^1.2.0": +"get-intrinsic@npm:^1.0.2, get-intrinsic@npm:^1.1.1, get-intrinsic@npm:^1.1.3, get-intrinsic@npm:^1.2.0": version: 1.2.0 resolution: "get-intrinsic@npm:1.2.0" dependencies: @@ -4526,27 +4109,13 @@ __metadata: languageName: node linkType: hard -"graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0": - version: 4.2.9 - resolution: "graceful-fs@npm:4.2.9" - checksum: 68ea4e07ff2c041ada184f9278b830375f8e0b75154e3f080af6b70f66172fabb4108d19b3863a96b53fc068a310b9b6493d86d1291acc5f3861eb4b79d26ad6 - languageName: node - linkType: hard - -"graceful-fs@npm:^4.2.6": +"graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9": version: 4.2.11 resolution: "graceful-fs@npm:4.2.11" checksum: ac85f94da92d8eb6b7f5a8b20ce65e43d66761c55ce85ac96df6865308390da45a8d3f0296dd3a663de65d30ba497bd46c696cc1e248c72b13d6d567138a4fc7 languageName: node linkType: hard -"graceful-fs@npm:^4.2.9": - version: 4.2.10 - resolution: "graceful-fs@npm:4.2.10" - checksum: 3f109d70ae123951905d85032ebeae3c2a5a7a997430df00ea30df0e3a6c60cf6689b109654d6fdacd28810a053348c4d14642da1d075049e6be1ba5216218da - languageName: node - linkType: hard - "grapheme-splitter@npm:^1.0.4": version: 1.0.4 resolution: "grapheme-splitter@npm:1.0.4" @@ -4554,14 +4123,7 @@ __metadata: languageName: node linkType: hard -"has-bigints@npm:^1.0.1": - version: 1.0.1 - resolution: "has-bigints@npm:1.0.1" - checksum: 44ab55868174470065d2e0f8f6def1c990d12b82162a8803c679699fa8a39f966e336f2a33c185092fe8aea7e8bf2e85f1c26add5f29d98f2318bd270096b183 - languageName: node - linkType: hard - -"has-bigints@npm:^1.0.2": +"has-bigints@npm:^1.0.1, has-bigints@npm:^1.0.2": version: 1.0.2 resolution: "has-bigints@npm:1.0.2" checksum: 390e31e7be7e5c6fe68b81babb73dfc35d413604d7ee5f56da101417027a4b4ce6a27e46eff97ad040c835b5d228676eae99a9b5c3bc0e23c8e81a49241ff45b @@ -4605,14 +4167,7 @@ __metadata: languageName: node linkType: hard -"has-symbols@npm:^1.0.1, has-symbols@npm:^1.0.2": - version: 1.0.2 - resolution: "has-symbols@npm:1.0.2" - checksum: 2309c426071731be792b5be43b3da6fb4ed7cbe8a9a6bcfca1862587709f01b33d575ce8f5c264c1eaad09fca2f9a8208c0a2be156232629daa2dd0c0740976b - languageName: node - linkType: hard - -"has-symbols@npm:^1.0.3": +"has-symbols@npm:^1.0.2, has-symbols@npm:^1.0.3": version: 1.0.3 resolution: "has-symbols@npm:1.0.3" checksum: a054c40c631c0d5741a8285010a0777ea0c068f99ed43e5d6eb12972da223f8af553a455132fdb0801bdcfa0e0f443c0c03a68d8555aa529b3144b446c3f2410 @@ -4799,17 +4354,6 @@ __metadata: languageName: node linkType: hard -"internal-slot@npm:^1.0.3": - version: 1.0.3 - resolution: "internal-slot@npm:1.0.3" - dependencies: - get-intrinsic: ^1.1.0 - has: ^1.0.3 - side-channel: ^1.0.4 - checksum: 1944f92e981e47aebc98a88ff0db579fd90543d937806104d0b96557b10c1f170c51fb777b97740a8b6ddeec585fca8c39ae99fd08a8e058dfc8ab70937238bf - languageName: node - linkType: hard - "internal-slot@npm:^1.0.4": version: 1.0.5 resolution: "internal-slot@npm:1.0.5" @@ -4874,20 +4418,13 @@ __metadata: languageName: node linkType: hard -"is-callable@npm:^1.1.3, is-callable@npm:^1.2.7": +"is-callable@npm:^1.1.3, is-callable@npm:^1.1.4, is-callable@npm:^1.2.7": version: 1.2.7 resolution: "is-callable@npm:1.2.7" checksum: 61fd57d03b0d984e2ed3720fb1c7a897827ea174bd44402878e059542ea8c4aeedee0ea0985998aa5cc2736b2fa6e271c08587addb5b3959ac52cf665173d1ac languageName: node linkType: hard -"is-callable@npm:^1.1.4, is-callable@npm:^1.2.4": - version: 1.2.4 - resolution: "is-callable@npm:1.2.4" - checksum: 1a28d57dc435797dae04b173b65d6d1e77d4f16276e9eff973f994eadcfdc30a017e6a597f092752a083c1103cceb56c91e3dadc6692fedb9898dfaba701575f - languageName: node - linkType: hard - "is-core-module@npm:^2.11.0, is-core-module@npm:^2.9.0": version: 2.11.0 resolution: "is-core-module@npm:2.11.0" @@ -4897,15 +4434,6 @@ __metadata: languageName: node linkType: hard -"is-core-module@npm:^2.2.0": - version: 2.8.0 - resolution: "is-core-module@npm:2.8.0" - dependencies: - has: ^1.0.3 - checksum: f8b52714891e1a6c6577fcb8d5e057bab064a7a30954aab6beb5092e311473eb8da57afd334de4981dc32409ffca998412efc3a2edceb9e397cef6098d21dd91 - languageName: node - linkType: hard - "is-date-object@npm:^1.0.1": version: 1.0.5 resolution: "is-date-object@npm:1.0.5" @@ -4952,13 +4480,6 @@ __metadata: languageName: node linkType: hard -"is-negative-zero@npm:^2.0.1": - version: 2.0.1 - resolution: "is-negative-zero@npm:2.0.1" - checksum: a46f2e0cb5e16fdb8f2011ed488979386d7e68d381966682e3f4c98fc126efe47f26827912baca2d06a02a644aee458b9cba307fb389f6b161e759125db7a3b8 - languageName: node - linkType: hard - "is-negative-zero@npm:^2.0.2": version: 2.0.2 resolution: "is-negative-zero@npm:2.0.2" @@ -5006,13 +4527,6 @@ __metadata: languageName: node linkType: hard -"is-shared-array-buffer@npm:^1.0.1": - version: 1.0.1 - resolution: "is-shared-array-buffer@npm:1.0.1" - checksum: 2ffb92533e64e2876e6cfe6906871d28400b6f1a53130fe652ec8007bc0e5044d05e7af8e31bdc992fbba520bd92938cfbeedd0f286be92f250c7c76191c4d90 - languageName: node - linkType: hard - "is-shared-array-buffer@npm:^1.0.2": version: 1.0.2 resolution: "is-shared-array-buffer@npm:1.0.2" @@ -5067,15 +4581,6 @@ __metadata: languageName: node linkType: hard -"is-weakref@npm:^1.0.1": - version: 1.0.1 - resolution: "is-weakref@npm:1.0.1" - dependencies: - call-bind: ^1.0.0 - checksum: fdafb7b955671dd2f9658ff47c86e4025c0650fc68a3542a40e5a75898a763b1abd6b1e1f9f13207eed49541cdd76af67d73c44989ea358b201b70274cf8f6c1 - languageName: node - linkType: hard - "is-weakref@npm:^1.0.2": version: 1.0.2 resolution: "is-weakref@npm:1.0.2" @@ -5099,20 +4604,7 @@ __metadata: languageName: node linkType: hard -"istanbul-lib-instrument@npm:^5.0.4": - version: 5.1.0 - resolution: "istanbul-lib-instrument@npm:5.1.0" - dependencies: - "@babel/core": ^7.12.3 - "@babel/parser": ^7.14.7 - "@istanbuljs/schema": ^0.1.2 - istanbul-lib-coverage: ^3.2.0 - semver: ^6.3.0 - checksum: 8b82e733c69fe9f94d2e21f3e5760c9bedb110329aa75df4bd40df95f1cac3bf38767e43f35b125cc547ceca7376b72ce7d95cc5238b7e9088345c7b589233d3 - languageName: node - linkType: hard - -"istanbul-lib-instrument@npm:^5.1.0": +"istanbul-lib-instrument@npm:^5.0.4, istanbul-lib-instrument@npm:^5.1.0": version: 5.2.1 resolution: "istanbul-lib-instrument@npm:5.2.1" dependencies: @@ -5524,7 +5016,21 @@ __metadata: languageName: node linkType: hard -"jest-util@npm:^29.0.0, jest-util@npm:^29.4.3": +"jest-util@npm:^29.0.0": + version: 29.5.0 + resolution: "jest-util@npm:29.5.0" + dependencies: + "@jest/types": ^29.5.0 + "@types/node": "*" + chalk: ^4.0.0 + ci-info: ^3.2.0 + graceful-fs: ^4.2.9 + picomatch: ^2.2.3 + checksum: fd9212950d34d2ecad8c990dda0d8ea59a8a554b0c188b53ea5d6c4a0829a64f2e1d49e6e85e812014933d17426d7136da4785f9cf76fff1799de51b88bc85d3 + languageName: node + linkType: hard + +"jest-util@npm:^29.4.3": version: 29.4.3 resolution: "jest-util@npm:29.4.3" dependencies: @@ -5699,17 +5205,6 @@ __metadata: languageName: node linkType: hard -"json5@npm:^2.1.2": - version: 2.2.0 - resolution: "json5@npm:2.2.0" - dependencies: - minimist: ^1.2.5 - bin: - json5: lib/cli.js - checksum: e88fc5274bb58fc99547baa777886b069d2dd96d9cfc4490b305fd16d711dabd5979e35a4f90873cefbeb552e216b041a304fe56702bedba76e19bc7845f208d - languageName: node - linkType: hard - "json5@npm:^2.2.2, json5@npm:^2.2.3": version: 2.2.3 resolution: "json5@npm:2.2.3" @@ -6016,17 +5511,7 @@ __metadata: languageName: node linkType: hard -"micromatch@npm:^4.0.4": - version: 4.0.4 - resolution: "micromatch@npm:4.0.4" - dependencies: - braces: ^3.0.1 - picomatch: ^2.2.3 - checksum: ef3d1c88e79e0a68b0e94a03137676f3324ac18a908c245a9e5936f838079fcc108ac7170a5fadc265a9c2596963462e402841406bda1a4bb7b68805601d631c - languageName: node - linkType: hard - -"micromatch@npm:^4.0.5": +"micromatch@npm:^4.0.4, micromatch@npm:^4.0.5": version: 4.0.5 resolution: "micromatch@npm:4.0.5" dependencies: @@ -6043,16 +5528,7 @@ __metadata: languageName: node linkType: hard -"minimatch@npm:^3.0.4": - version: 3.0.4 - resolution: "minimatch@npm:3.0.4" - dependencies: - brace-expansion: ^1.1.7 - checksum: 66ac295f8a7b59788000ea3749938b0970344c841750abd96694f80269b926ebcafad3deeb3f1da2522978b119e6ae3a5869b63b13a7859a456b3408bd18a078 - languageName: node - linkType: hard - -"minimatch@npm:^3.0.5, minimatch@npm:^3.1.2": +"minimatch@npm:^3.0.4, minimatch@npm:^3.0.5, minimatch@npm:^3.1.2": version: 3.1.2 resolution: "minimatch@npm:3.1.2" dependencies: @@ -6079,14 +5555,7 @@ __metadata: languageName: node linkType: hard -"minimist@npm:^1.2.0, minimist@npm:^1.2.5": - version: 1.2.5 - resolution: "minimist@npm:1.2.5" - checksum: 86706ce5b36c16bfc35c5fe3dbb01d5acdc9a22f2b6cc810b6680656a1d2c0e44a0159c9a3ba51fb072bb5c203e49e10b51dcd0eec39c481f4c42086719bae52 - languageName: node - linkType: hard - -"minimist@npm:^1.2.6": +"minimist@npm:^1.2.0, minimist@npm:^1.2.6": version: 1.2.8 resolution: "minimist@npm:1.2.8" checksum: 75a6d645fb122dad29c06a7597bddea977258957ed88d7a6df59b5cd3fe4a527e253e9bbf2e783e4b73657f9098b96a5fe96ab8a113655d4109108577ecf85b0 @@ -6264,13 +5733,6 @@ __metadata: languageName: node linkType: hard -"node-releases@npm:^2.0.1": - version: 2.0.1 - resolution: "node-releases@npm:2.0.1" - checksum: b20dd8d4bced11f75060f0387e05e76b9dc4a0451f7bb3516eade6f50499ea7768ba95d8a60d520c193402df1e58cb3fe301510cc1c1ad68949c3d57b5149866 - languageName: node - linkType: hard - "node-releases@npm:^2.0.8": version: 2.0.10 resolution: "node-releases@npm:2.0.10" @@ -6324,39 +5786,20 @@ __metadata: languageName: node linkType: hard -"object-inspect@npm:^1.11.0, object-inspect@npm:^1.9.0": - version: 1.11.0 - resolution: "object-inspect@npm:1.11.0" - checksum: 8c64f89ce3a7b96b6925879ad5f6af71d498abc217e136660efecd97452991216f375a7eb47cb1cb50643df939bf0c7cc391567b7abc6a924d04679705e58e27 - languageName: node - linkType: hard - -"object-inspect@npm:^1.12.2": +"object-inspect@npm:^1.12.2, object-inspect@npm:^1.9.0": version: 1.12.3 resolution: "object-inspect@npm:1.12.3" checksum: dabfd824d97a5f407e6d5d24810d888859f6be394d8b733a77442b277e0808860555176719c5905e765e3743a7cada6b8b0a3b85e5331c530fd418cc8ae991db languageName: node linkType: hard -"object-keys@npm:^1.0.12, object-keys@npm:^1.1.1": +"object-keys@npm:^1.1.1": version: 1.1.1 resolution: "object-keys@npm:1.1.1" checksum: b363c5e7644b1e1b04aa507e88dcb8e3a2f52b6ffd0ea801e4c7a62d5aa559affe21c55a07fd4b1fd55fc03a33c610d73426664b20032405d7b92a1414c34d6a languageName: node linkType: hard -"object.assign@npm:^4.1.2": - version: 4.1.2 - resolution: "object.assign@npm:4.1.2" - dependencies: - call-bind: ^1.0.0 - define-properties: ^1.1.3 - has-symbols: ^1.0.1 - object-keys: ^1.1.1 - checksum: d621d832ed7b16ac74027adb87196804a500d80d9aca536fccb7ba48d33a7e9306a75f94c1d29cbfa324bc091bfc530bc24789568efdaee6a47fcfa298993814 - languageName: node - linkType: hard - "object.assign@npm:^4.1.4": version: 4.1.4 resolution: "object.assign@npm:4.1.4" @@ -6515,7 +5958,7 @@ __metadata: languageName: node linkType: hard -"path-parse@npm:^1.0.6, path-parse@npm:^1.0.7": +"path-parse@npm:^1.0.7": version: 1.0.7 resolution: "path-parse@npm:1.0.7" checksum: 49abf3d81115642938a8700ec580da6e830dde670be21893c62f4e10bd7dd4c3742ddc603fe24f898cba7eb0c6bc1777f8d9ac14185d34540c6d4d80cd9cae8a @@ -6536,14 +5979,7 @@ __metadata: languageName: node linkType: hard -"picomatch@npm:^2.0.4, picomatch@npm:^2.2.3": - version: 2.3.0 - resolution: "picomatch@npm:2.3.0" - checksum: 16818720ea7c5872b6af110760dee856c8e4cd79aed1c7a006d076b1cc09eff3ae41ca5019966694c33fbd2e1cc6ea617ab10e4adac6df06556168f13be3fca2 - languageName: node - linkType: hard - -"picomatch@npm:^2.2.1, picomatch@npm:^2.3.1": +"picomatch@npm:^2.0.4, picomatch@npm:^2.2.1, picomatch@npm:^2.2.3, picomatch@npm:^2.3.1": version: 2.3.1 resolution: "picomatch@npm:2.3.1" checksum: 050c865ce81119c4822c45d3c84f1ced46f93a0126febae20737bd05ca20589c564d6e9226977df859ed5e03dc73f02584a2b0faad36e896936238238b0446cf @@ -6892,7 +6328,7 @@ __metadata: languageName: node linkType: hard -"resolve@npm:^1.1.7, resolve@npm:^1.22.1": +"resolve@npm:^1.1.7, resolve@npm:^1.20.0, resolve@npm:^1.22.1": version: 1.22.1 resolution: "resolve@npm:1.22.1" dependencies: @@ -6905,17 +6341,7 @@ __metadata: languageName: node linkType: hard -"resolve@npm:^1.20.0": - version: 1.20.0 - resolution: "resolve@npm:1.20.0" - dependencies: - is-core-module: ^2.2.0 - path-parse: ^1.0.6 - checksum: 40cf70b2cde00ef57f99daf2dc63c6a56d6c14a1b7fc51735d06a6f0a3b97cb67b4fb7ef6c747b4e13a7baba83b0ef625d7c4ce92a483cd5af923c3b65fd16fe - languageName: node - linkType: hard - -"resolve@patch:resolve@^1.1.7#~builtin, resolve@patch:resolve@^1.22.1#~builtin": +"resolve@patch:resolve@^1.1.7#~builtin, resolve@patch:resolve@^1.20.0#~builtin, resolve@patch:resolve@^1.22.1#~builtin": version: 1.22.1 resolution: "resolve@patch:resolve@npm%3A1.22.1#~builtin::version=1.22.1&hash=c3c19d" dependencies: @@ -6928,16 +6354,6 @@ __metadata: languageName: node linkType: hard -"resolve@patch:resolve@^1.20.0#~builtin": - version: 1.20.0 - resolution: "resolve@patch:resolve@npm%3A1.20.0#~builtin::version=1.20.0&hash=c3c19d" - dependencies: - is-core-module: ^2.2.0 - path-parse: ^1.0.6 - checksum: a0dd7d16a8e47af23afa9386df2dff10e3e0debb2c7299a42e581d9d9b04d7ad5d2c53f24f1e043f7b3c250cbdc71150063e53d0b6559683d37f790b7c8c3cd5 - languageName: node - linkType: hard - "restore-cursor@npm:^3.1.0": version: 3.1.0 resolution: "restore-cursor@npm:3.1.0" @@ -7025,14 +6441,14 @@ __metadata: languageName: node linkType: hard -"semver@npm:7.x, semver@npm:^7.3.5": - version: 7.3.5 - resolution: "semver@npm:7.3.5" +"semver@npm:7.x, semver@npm:^7.3.5, semver@npm:^7.3.7, semver@npm:^7.3.8": + version: 7.3.8 + resolution: "semver@npm:7.3.8" dependencies: lru-cache: ^6.0.0 bin: semver: bin/semver.js - checksum: 5eafe6102bea2a7439897c1856362e31cc348ccf96efd455c8b5bc2c61e6f7e7b8250dc26b8828c1d76a56f818a7ee907a36ae9fb37a599d3d24609207001d60 + checksum: ba9c7cbbf2b7884696523450a61fee1a09930d888b7a8d7579025ad93d459b2d1949ee5bbfeb188b2be5f4ac163544c5e98491ad6152df34154feebc2cc337c1 languageName: node linkType: hard @@ -7045,17 +6461,6 @@ __metadata: languageName: node linkType: hard -"semver@npm:^7.3.7, semver@npm:^7.3.8": - version: 7.3.8 - resolution: "semver@npm:7.3.8" - dependencies: - lru-cache: ^6.0.0 - bin: - semver: bin/semver.js - checksum: ba9c7cbbf2b7884696523450a61fee1a09930d888b7a8d7579025ad93d459b2d1949ee5bbfeb188b2be5f4ac163544c5e98491ad6152df34154feebc2cc337c1 - languageName: node - linkType: hard - "set-blocking@npm:^2.0.0": version: 2.0.0 resolution: "set-blocking@npm:2.0.0" @@ -7102,14 +6507,7 @@ __metadata: languageName: node linkType: hard -"signal-exit@npm:^3.0.2, signal-exit@npm:^3.0.3": - version: 3.0.6 - resolution: "signal-exit@npm:3.0.6" - checksum: b819ac81ba757af559dad0804233ae31bf6f054591cd8a671e9cbcf09f21c72ec3076fe87d1e04861f5b33b47d63f0694b568de99c99cd733ee2060515beb6d5 - languageName: node - linkType: hard - -"signal-exit@npm:^3.0.7": +"signal-exit@npm:^3.0.2, signal-exit@npm:^3.0.3, signal-exit@npm:^3.0.7": version: 3.0.7 resolution: "signal-exit@npm:3.0.7" checksum: a2f098f247adc367dffc27845853e9959b9e88b01cb301658cfe4194352d8d2bb32e18467c786a7fe15f1d44b233ea35633d076d5e737870b7139949d1ab6318 @@ -7186,13 +6584,6 @@ __metadata: languageName: node linkType: hard -"source-map@npm:^0.5.0": - version: 0.5.7 - resolution: "source-map@npm:0.5.7" - checksum: 5dc2043b93d2f194142c7f38f74a24670cd7a0063acdaf4bf01d2964b402257ae843c2a8fa822ad5b71013b5fcafa55af7421383da919752f22ff488bc553f4d - languageName: node - linkType: hard - "source-map@npm:^0.6.0, source-map@npm:^0.6.1": version: 0.6.1 resolution: "source-map@npm:0.6.1" @@ -7246,16 +6637,6 @@ __metadata: languageName: node linkType: hard -"string.prototype.trimend@npm:^1.0.4": - version: 1.0.4 - resolution: "string.prototype.trimend@npm:1.0.4" - dependencies: - call-bind: ^1.0.2 - define-properties: ^1.1.3 - checksum: 17e5aa45c3983f582693161f972c1c1fa4bbbdf22e70e582b00c91b6575f01680dc34e83005b98e31abe4d5d29e0b21fcc24690239c106c7b2315aade6a898ac - languageName: node - linkType: hard - "string.prototype.trimend@npm:^1.0.6": version: 1.0.6 resolution: "string.prototype.trimend@npm:1.0.6" @@ -7267,16 +6648,6 @@ __metadata: languageName: node linkType: hard -"string.prototype.trimstart@npm:^1.0.4": - version: 1.0.4 - resolution: "string.prototype.trimstart@npm:1.0.4" - dependencies: - call-bind: ^1.0.2 - define-properties: ^1.1.3 - checksum: 3fb06818d3cccac5fa3f5f9873d984794ca0e9f6616fae6fcc745885d9efed4e17fe15f832515d9af5e16c279857fdbffdfc489ca4ed577811b017721b30302f - languageName: node - linkType: hard - "string.prototype.trimstart@npm:^1.0.6": version: 1.0.6 resolution: "string.prototype.trimstart@npm:1.0.6" @@ -7567,14 +6938,7 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2": - version: 2.3.1 - resolution: "tslib@npm:2.3.1" - checksum: de17a98d4614481f7fcb5cd53ffc1aaf8654313be0291e1bfaee4b4bb31a20494b7d218ff2e15017883e8ea9626599b3b0e0229c18383ba9dce89da2adf15cb9 - languageName: node - linkType: hard - -"tslib@npm:^2.5.0": +"tslib@npm:^2, tslib@npm:^2.5.0": version: 2.5.0 resolution: "tslib@npm:2.5.0" checksum: ae3ed5f9ce29932d049908ebfdf21b3a003a85653a9a140d614da6b767a93ef94f460e52c3d787f0e4f383546981713f165037dc2274df212ea9f8a4541004e1 @@ -7678,18 +7042,6 @@ __metadata: languageName: node linkType: hard -"unbox-primitive@npm:^1.0.1": - version: 1.0.1 - resolution: "unbox-primitive@npm:1.0.1" - dependencies: - function-bind: ^1.1.1 - has-bigints: ^1.0.1 - has-symbols: ^1.0.2 - which-boxed-primitive: ^1.0.2 - checksum: 89d950e18fb45672bc6b3c961f1e72c07beb9640c7ceed847b571ba6f7d2af570ae1a2584cfee268b9d9ea1e3293f7e33e0bc29eaeb9f8e8a0bab057ff9e6bba - languageName: node - linkType: hard - "unbox-primitive@npm:^1.0.2": version: 1.0.2 resolution: "unbox-primitive@npm:1.0.2" From 3dacc7b82e44db2443f0541ce54ef4da70d70d1a Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Thu, 29 Jun 2023 17:01:35 -0700 Subject: [PATCH 03/27] update src from studio/packages/mcap-support --- .../examples/bag2mcap/scripts/bag2mcap.ts | 4 - typescript/support/package.json | 12 +- typescript/support/src/decompressHandlers.ts | 10 +- .../support/src/fixtures/byte-vector.ts | 4 - typescript/support/src/parseChannel.test.ts | 65 ++--- typescript/support/src/parseChannel.ts | 73 ++---- .../support/src/parseFlatbufferSchema.test.ts | 23 +- .../support/src/parseFlatbufferSchema.ts | 23 +- typescript/support/src/parseJsonSchema.ts | 9 +- .../support/src/parseProtobufSchema.test.ts | 240 ++++++++++++++++++ typescript/support/src/parseProtobufSchema.ts | 82 ++++++ typescript/support/src/types.ts | 4 - yarn.lock | 36 +-- 13 files changed, 428 insertions(+), 157 deletions(-) create mode 100644 typescript/support/src/parseProtobufSchema.test.ts create mode 100644 typescript/support/src/parseProtobufSchema.ts diff --git a/typescript/examples/bag2mcap/scripts/bag2mcap.ts b/typescript/examples/bag2mcap/scripts/bag2mcap.ts index 3d911e1063..6c8a2a680c 100644 --- a/typescript/examples/bag2mcap/scripts/bag2mcap.ts +++ b/typescript/examples/bag2mcap/scripts/bag2mcap.ts @@ -1,7 +1,3 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/ - // convert a ROS1 .bag file to an mcap file with protobuf schema and message encoding import { Bag } from "@foxglove/rosbag"; diff --git a/typescript/support/package.json b/typescript/support/package.json index 2aa98b022b..d777806970 100644 --- a/typescript/support/package.json +++ b/typescript/support/package.json @@ -66,16 +66,16 @@ }, "dependencies": { "@foxglove/message-definition": "^0.2.0", - "@foxglove/rosmsg": "^4.0.0", - "@foxglove/rosmsg-serialization": "^2.0.0", - "@foxglove/rosmsg2-serialization": "^2.0.0", - "@foxglove/schemas": "^1.1.0", + "@foxglove/rosmsg": "^4.2.2", + "@foxglove/rosmsg-serialization": "^2.0.1", + "@foxglove/rosmsg2-serialization": "^2.0.2", + "@foxglove/schemas": "^1.3.1", "@foxglove/wasm-bz2": "^0.1.1", "@foxglove/wasm-lz4": "^1.0.2", "@foxglove/wasm-zstd": "^1.0.1", "@protobufjs/base64": "^1.1.2", - "flatbuffers": "^23.1.21", - "flatbuffers_reflection": "^0.0.3", + "flatbuffers": "^23.5.26", + "flatbuffers_reflection": "^0.0.6", "protobufjs": "^7.2.2" } } diff --git a/typescript/support/src/decompressHandlers.ts b/typescript/support/src/decompressHandlers.ts index b67df875a9..69f9d1031a 100644 --- a/typescript/support/src/decompressHandlers.ts +++ b/typescript/support/src/decompressHandlers.ts @@ -1,14 +1,12 @@ -type DecompressHandlers = { - [compression: string]: (buffer: Uint8Array, decompressedSize: bigint) => Uint8Array; -}; +import { McapTypes } from "@mcap/core"; -let handlersPromise: Promise | undefined; -export async function loadDecompressHandlers(): Promise { +let handlersPromise: Promise | undefined; +export async function loadDecompressHandlers(): Promise { return await (handlersPromise ??= _loadDecompressHandlers()); } // eslint-disable-next-line no-underscore-dangle -async function _loadDecompressHandlers(): Promise { +async function _loadDecompressHandlers(): Promise { const [decompressZstd, decompressLZ4, bzip2] = await Promise.all([ import("@foxglove/wasm-zstd").then(async (mod) => { await mod.isLoaded; diff --git a/typescript/support/src/fixtures/byte-vector.ts b/typescript/support/src/fixtures/byte-vector.ts index 4b7e56030a..1abb175805 100644 --- a/typescript/support/src/fixtures/byte-vector.ts +++ b/typescript/support/src/fixtures/byte-vector.ts @@ -1,7 +1,3 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/ - // automatically generated by the FlatBuffers compiler, do not modify /* eslint-disable */ import * as flatbuffers from "flatbuffers"; diff --git a/typescript/support/src/parseChannel.test.ts b/typescript/support/src/parseChannel.test.ts index d9d3ab65aa..7f46003718 100644 --- a/typescript/support/src/parseChannel.test.ts +++ b/typescript/support/src/parseChannel.test.ts @@ -2,7 +2,6 @@ import fs from "fs"; import { FileDescriptorSet, IFileDescriptorSet } from "protobufjs/ext/descriptor"; import { parseChannel } from "./parseChannel"; -import { protobufToDescriptor } from "./protobufDescriptors"; describe("parseChannel", () => { it("works with json/jsonschema", () => { @@ -16,9 +15,9 @@ describe("parseChannel", () => { ), }, }); - expect(channel.deserializer(new TextEncoder().encode(JSON.stringify({ value: "hi" })))).toEqual( - { value: "hi" }, - ); + expect(channel.deserialize(new TextEncoder().encode(JSON.stringify({ value: "hi" })))).toEqual({ + value: "hi", + }); }); it("works with flatbuffer", () => { @@ -27,7 +26,7 @@ describe("parseChannel", () => { messageEncoding: "flatbuffer", schema: { name: "reflection.Schema", encoding: "flatbuffer", data: reflectionSchema }, }); - const deserialized = channel.deserializer(reflectionSchema) as { + const deserialized = channel.deserialize(reflectionSchema) as { objects: Record[]; }; expect(deserialized.objects.length).toEqual(10); @@ -35,45 +34,15 @@ describe("parseChannel", () => { }); it("works with protobuf", () => { - const fds = FileDescriptorSet.encode(protobufToDescriptor(FileDescriptorSet.root)).finish(); + const fds = FileDescriptorSet.encode(FileDescriptorSet.root.toDescriptor("proto3")).finish(); const channel = parseChannel({ messageEncoding: "protobuf", schema: { name: "google.protobuf.FileDescriptorSet", encoding: "protobuf", data: fds }, }); - const deserialized = channel.deserializer(fds) as IFileDescriptorSet; + const deserialized = channel.deserialize(fds) as IFileDescriptorSet; expect(deserialized.file[0]!.name).toEqual("google_protobuf.proto"); }); - it("handles protobuf repeated enum having multiple default value aliases", () => { - /* - syntax = "proto3"; - - enum ExampleEnum { - option allow_alias = true; - UNKNOWN = 0; - WHATEVER = 0; - FOO = 1; - BAR = 2; - } - - message ExampleMessage { - repeated ExampleEnum data = 1; - } - */ - const channel = parseChannel({ - messageEncoding: "protobuf", - schema: { - name: "ExampleMessage", - encoding: "protobuf", - data: Buffer.from( - "0A8D010A156578616D706C655F6D6573736167652E70726F746F222C0A0E4578616D706C654D657373616765121A0A046461746118012003280E320C2E4578616D706C65456E756D2A3E0A0B4578616D706C65456E756D120B0A07554E4B4E4F574E1000120C0A085748415445564552100012070A03464F4F100112070A0342415210021A021001620670726F746F33", - "hex", - ), - }, - }); - expect(channel.deserializer(Buffer.from("0A0101", "hex"))).toEqual({ data: [1] }); - }); - it("works with ros1", () => { const channel = parseChannel({ messageEncoding: "ros1", @@ -84,7 +53,7 @@ describe("parseChannel", () => { }, }); - const obj = channel.deserializer(new Uint8Array([4, 0, 0, 0, 65, 66, 67, 68])); + const obj = channel.deserialize(new Uint8Array([4, 0, 0, 0, 65, 66, 67, 68])); expect(obj).toEqual({ data: "ABCD" }); }); @@ -98,7 +67,25 @@ describe("parseChannel", () => { }, }); - const obj = channel.deserializer(new Uint8Array([0, 1, 0, 0, 5, 0, 0, 0, 65, 66, 67, 68, 0])); + const obj = channel.deserialize(new Uint8Array([0, 1, 0, 0, 5, 0, 0, 0, 65, 66, 67, 68, 0])); + expect(obj).toEqual({ data: "ABCD" }); + }); + + it("works with ros2idl", () => { + const channel = parseChannel({ + messageEncoding: "cdr", + schema: { + name: "foo_msgs/Bar", + encoding: "ros2idl", + data: new TextEncoder().encode(` + module foo_msgs { + struct Bar {string data;}; + }; + `), + }, + }); + + const obj = channel.deserialize(new Uint8Array([0, 1, 0, 0, 5, 0, 0, 0, 65, 66, 67, 68, 0])); expect(obj).toEqual({ data: "ABCD" }); }); }); diff --git a/typescript/support/src/parseChannel.ts b/typescript/support/src/parseChannel.ts index 9c12f3ad77..a3fc81aad8 100644 --- a/typescript/support/src/parseChannel.ts +++ b/typescript/support/src/parseChannel.ts @@ -1,13 +1,11 @@ import { MessageDefinition } from "@foxglove/message-definition"; -import { parse as parseMessageDefinition } from "@foxglove/rosmsg"; +import { parse as parseMessageDefinition, parseRos2idl } from "@foxglove/rosmsg"; import { MessageReader } from "@foxglove/rosmsg-serialization"; import { MessageReader as ROS2MessageReader } from "@foxglove/rosmsg2-serialization"; -import { FileDescriptorSet, IFileDescriptorSet } from "protobufjs/ext/descriptor"; import { parseFlatbufferSchema } from "./parseFlatbufferSchema"; import { parseJsonSchema } from "./parseJsonSchema"; -import { protobufDefinitionsToDatatypes, stripLeadingDot } from "./protobufDefinitionsToDatatypes"; -import { protobufFromDescriptor } from "./protobufDescriptors"; +import { parseProtobufSchema } from "./parseProtobufSchema"; import { MessageDefinitionMap } from "./types"; type Channel = { @@ -16,7 +14,7 @@ type Channel = { }; export type ParsedChannel = { - deserializer: (data: ArrayBufferView) => unknown; + deserialize: (data: ArrayBufferView) => unknown; datatypes: MessageDefinitionMap; }; @@ -52,7 +50,7 @@ export function parseChannel(channel: Channel): ParsedChannel { } const textDecoder = new TextDecoder(); let datatypes: MessageDefinitionMap = new Map(); - let deserializer = (data: ArrayBufferView) => JSON.parse(textDecoder.decode(data)) as unknown; + let deserialize = (data: ArrayBufferView) => JSON.parse(textDecoder.decode(data)) as unknown; if (channel.schema != undefined) { const schema = channel.schema.data.length > 0 @@ -67,11 +65,11 @@ export function parseChannel(channel: Channel): ParsedChannel { channel.schema.name, ); datatypes = parsedDatatypes; - deserializer = (data) => + deserialize = (data) => postprocessValue(JSON.parse(textDecoder.decode(data)) as Record); } } - return { deserializer, datatypes }; + return { deserialize, datatypes }; } if (channel.messageEncoding === "flatbuffer") { @@ -97,49 +95,7 @@ export function parseChannel(channel: Channel): ParsedChannel { } is not supported (expected protobuf)`, ); } - const descriptorSet = FileDescriptorSet.decode(channel.schema.data); - - // Modify the definition of google.protobuf.Timestamp so it gets parsed as {sec, nsec}, - // compatible with the rest of Studio. - for (const file of (descriptorSet as unknown as IFileDescriptorSet).file) { - if (file.package === "google.protobuf") { - for (const message of file.messageType ?? []) { - if (message.name === "Timestamp" || message.name === "Duration") { - for (const field of message.field ?? []) { - if (field.name === "seconds") { - field.name = "sec"; - } else if (field.name === "nanos") { - field.name = "nsec"; - } - } - } - } - } - } - - const root = protobufFromDescriptor(descriptorSet); - root.resolveAll(); - const type = root.lookupType(channel.schema.name); - - const deserializer = (data: ArrayBufferView) => { - return type.toObject( - type.decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength)), - { defaults: true }, - ); - }; - - const datatypes: MessageDefinitionMap = new Map(); - protobufDefinitionsToDatatypes(datatypes, type); - - if (!datatypes.has(channel.schema.name)) { - throw new Error( - `Protobuf schema does not contain an entry for '${ - channel.schema.name - }'. The schema name should be fully-qualified, e.g. '${stripLeadingDot(type.fullName)}'.`, - ); - } - - return { deserializer, datatypes }; + return parseProtobufSchema(channel.schema.name, channel.schema.data); } if (channel.messageEncoding === "ros1") { @@ -157,26 +113,31 @@ export function parseChannel(channel: Channel): ParsedChannel { const reader = new MessageReader(parsedDefinitions); return { datatypes: parsedDefinitionsToDatatypes(parsedDefinitions, channel.schema.name), - deserializer: (data) => reader.readMessage(data), + deserialize: (data) => reader.readMessage(data), }; } if (channel.messageEncoding === "cdr") { - if (channel.schema?.encoding !== "ros2msg") { + if (channel.schema?.encoding !== "ros2msg" && channel.schema?.encoding !== "ros2idl") { throw new Error( `Message encoding ${channel.messageEncoding} with ${ channel.schema == undefined ? "no encoding" : `schema encoding '${channel.schema.encoding}'` - } is not supported (expected ros2msg)`, + } is not supported (expected "ros2msg" or "ros2idl")`, ); } const schema = new TextDecoder().decode(channel.schema.data); - const parsedDefinitions = parseMessageDefinition(schema, { ros2: true }); + const isIdl = channel.schema.encoding === "ros2idl"; + + const parsedDefinitions = isIdl + ? parseRos2idl(schema) + : parseMessageDefinition(schema, { ros2: true }); + const reader = new ROS2MessageReader(parsedDefinitions); return { datatypes: parsedDefinitionsToDatatypes(parsedDefinitions, channel.schema.name), - deserializer: (data) => reader.readMessage(data), + deserialize: (data) => reader.readMessage(data), }; } diff --git a/typescript/support/src/parseFlatbufferSchema.test.ts b/typescript/support/src/parseFlatbufferSchema.test.ts index 9787e6ab74..c74b4f1a72 100644 --- a/typescript/support/src/parseFlatbufferSchema.test.ts +++ b/typescript/support/src/parseFlatbufferSchema.test.ts @@ -1,7 +1,3 @@ -/* eslint @typescript-eslint/no-explicit-any: 0 */ -/* eslint @typescript-eslint/no-unsafe-member-access: 0 */ -/* eslint @typescript-eslint/no-unsafe-call: 0 */ - import { ByteBuffer, Builder } from "flatbuffers"; import { Schema, BaseType, Type } from "flatbuffers_reflection"; import fs from "fs"; @@ -316,11 +312,11 @@ describe("parseFlatbufferSchema", () => { // $ flatc -b --schema reflection/reflection.fbs // In https://github.com/google/flatbuffers const reflectionSchemaBuffer: Buffer = fs.readFileSync(`${__dirname}/fixtures/reflection.bfbs`); - const { datatypes, deserializer } = parseFlatbufferSchema( + const { datatypes, deserialize } = parseFlatbufferSchema( "reflection.Schema", reflectionSchemaBuffer, ); - const deserialized: any = deserializer(reflectionSchemaBuffer); + const deserialized: any = deserialize(reflectionSchemaBuffer); const reflectionSchemaByteBuffer: ByteBuffer = new ByteBuffer(reflectionSchemaBuffer); const schema = Schema.getRootAsSchema(reflectionSchemaByteBuffer); // Spot check individual components to ensure that they got deserialized correctly. @@ -337,7 +333,7 @@ describe("parseFlatbufferSchema", () => { }); it("parses non-root table schema", () => { const reflectionSchemaBuffer: Buffer = fs.readFileSync(`${__dirname}/fixtures/reflection.bfbs`); - const { datatypes, deserializer } = parseFlatbufferSchema( + const { datatypes, deserialize } = parseFlatbufferSchema( "reflection.Type", reflectionSchemaBuffer, ); @@ -352,7 +348,14 @@ describe("parseFlatbufferSchema", () => { Type.addIndex(builder, 123); builder.finish(Type.endType(builder)); - expect(deserializer(builder.asUint8Array())).toEqual({ base_type: 7, index: 123 }); + expect(deserialize(builder.asUint8Array())).toEqual({ + base_size: 4n, + base_type: 7, + element: 0n, + element_size: 0n, + fixed_length: 0n, + index: 123, + }); }); it("converts uint8 vectors to uint8arrays", () => { const builder = new Builder(); @@ -378,7 +381,7 @@ describe("parseFlatbufferSchema", () => { const byteVectorBin = Uint8Array.from(builder.asUint8Array()); const byteVectorSchemaArray = fs.readFileSync(`${__dirname}/fixtures/ByteVector.bfbs`); - const { deserializer } = parseFlatbufferSchema("ByteVector", byteVectorSchemaArray); - expect(deserializer(byteVectorBin)).toEqual({ data: new Uint8Array([1, 2, 3]) }); + const { deserialize } = parseFlatbufferSchema("ByteVector", byteVectorSchemaArray); + expect(deserialize(byteVectorBin)).toEqual({ data: new Uint8Array([1, 2, 3]) }); }); }); diff --git a/typescript/support/src/parseFlatbufferSchema.ts b/typescript/support/src/parseFlatbufferSchema.ts index 935d9a7ef2..60c5725ff8 100644 --- a/typescript/support/src/parseFlatbufferSchema.ts +++ b/typescript/support/src/parseFlatbufferSchema.ts @@ -145,14 +145,18 @@ function typeForField(schema: SchemaT, field: FieldT): MessageDefinitionField[] return fields; } -// Note: Currently this does not support "lazy" message reading in the style of the ros1 message -// reader, and so will relatively inefficiently deserialize the entire flatbuffer message. +/** + * Parse a flatbuffer binary schema and produce datatypes and a deserializer function. + * + * Note: Currently this does not support "lazy" message reading in the style of the ros1 message + * reader, and so will relatively inefficiently deserialize the entire flatbuffer message. + */ export function parseFlatbufferSchema( schemaName: string, schemaArray: Uint8Array, ): { datatypes: MessageDefinitionMap; - deserializer: (buffer: ArrayBufferView) => unknown; + deserialize: (buffer: ArrayBufferView) => unknown; } { const datatypes: MessageDefinitionMap = new Map(); const schemaBuffer = new ByteBuffer(schemaArray); @@ -183,7 +187,12 @@ export function parseFlatbufferSchema( } } const parser = new Parser(rawSchema); - const deserializer = (buffer: ArrayBufferView) => { + // We set readDefaults=true to ensure that the reader receives default values for unset fields, or + // fields that were explicitly set but with ForceDefaults(false) on the writer side. This is + // necessary because `datatypes` does not include information about default values from the + // schema. See discussion: + const toObject = parser.toObjectLambda(typeIndex, /*readDefaults=*/ true); + const deserialize = (buffer: ArrayBufferView) => { const byteBuffer = new ByteBuffer( new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength), ); @@ -191,9 +200,9 @@ export function parseFlatbufferSchema( byteBuffer, typeIndex, byteBuffer.readInt32(byteBuffer.position()) + byteBuffer.position(), + false, ); - const obj = parser.toObject(table); - return obj; + return toObject(table); }; - return { datatypes, deserializer }; + return { datatypes, deserialize }; } diff --git a/typescript/support/src/parseJsonSchema.ts b/typescript/support/src/parseJsonSchema.ts index ca4215bae8..575ceea4b6 100644 --- a/typescript/support/src/parseJsonSchema.ts +++ b/typescript/support/src/parseJsonSchema.ts @@ -1,11 +1,11 @@ -/* eslint @typescript-eslint/no-unsafe-member-access: 0 */ -/* eslint @typescript-eslint/no-unsafe-assignment: 0 */ - import { MessageDefinitionField } from "@foxglove/message-definition"; import * as base64 from "@protobufjs/base64"; import { MessageDefinitionMap } from "./types"; +/** + * Parse a JSON Schema and produce datatypes and a deserializer function. + */ export function parseJsonSchema( rootJsonSchema: Record, rootTypeName: string, @@ -36,12 +36,15 @@ export function parseJsonSchema( schema.properties as Record>, )) { if (Array.isArray(fieldSchema.oneOf)) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access if (fieldSchema.oneOf.every((alternative) => typeof alternative.const === "number")) { for (const alternative of fieldSchema.oneOf) { fields.push({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access name: alternative.title, type: "uint32", isConstant: true, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access value: alternative.const, }); } diff --git a/typescript/support/src/parseProtobufSchema.test.ts b/typescript/support/src/parseProtobufSchema.test.ts new file mode 100644 index 0000000000..7949ac279e --- /dev/null +++ b/typescript/support/src/parseProtobufSchema.test.ts @@ -0,0 +1,240 @@ +import { parseProtobufSchema } from "./parseProtobufSchema"; + +describe("parseProtobufSchema", () => { + it("handles protobuf repeated enum having multiple default value aliases", () => { + /* + syntax = "proto3"; + + enum ExampleEnum { + option allow_alias = true; + UNKNOWN = 0; + WHATEVER = 0; + FOO = 1; + BAR = 2; + } + + message ExampleMessage { + repeated ExampleEnum data = 1; + } + */ + const channel = parseProtobufSchema( + "ExampleMessage", + Buffer.from( + "0A8D010A156578616D706C655F6D6573736167652E70726F746F222C0A0E4578616D706C654D657373616765121A0A046461746118012003280E320C2E4578616D706C65456E756D2A3E0A0B4578616D706C65456E756D120B0A07554E4B4E4F574E1000120C0A085748415445564552100012070A03464F4F100112070A0342415210021A021001620670726F746F33", + "hex", + ), + ); + expect(channel.deserialize(Buffer.from("0A0101", "hex"))).toEqual({ data: [1] }); + }); + + it("handles protobuf int64 values", () => { + /* + syntax = "proto3"; + + message Int64Test { + int64 int64 = 1; + uint64 uint64 = 2; + sint64 sint64 = 3; + fixed64 fixed64 = 4; + sfixed64 sfixed64 = 5; + map int64map = 6; + map uint64map = 7; + map sint64map = 8; + map fixed64map = 9; + map sfixed64map = 10; + repeated Nested nested = 11; + } + + message Nested { + int64 int64 = 1; + uint64 uint64 = 2; + sint64 sint64 = 3; + fixed64 fixed64 = 4; + sfixed64 sfixed64 = 5; + map int64map = 6; + map uint64map = 7; + map sint64map = 8; + map fixed64map = 9; + map sfixed64map = 10; + } + */ + const channel = parseProtobufSchema( + "Int64Test", + Buffer.from( + "CvILCg9JbnQ2NFRlc3QucHJvdG8igwYKCUludDY0VGVzdBIUCgVpbnQ2NBgBIAEoA1IFaW50NjQSFgoGdWludDY0GAIgASgEUgZ1aW50NjQSFgoGc2ludDY0GAMgASgSUgZzaW50NjQSGAoHZml4ZWQ2NBgEIAEoBlIHZml4ZWQ2NBIaCghzZml4ZWQ2NBgFIAEoEFIIc2ZpeGVkNjQSNAoIaW50NjRtYXAYBiADKAsyGC5JbnQ2NFRlc3QuSW50NjRtYXBFbnRyeVIIaW50NjRtYXASNwoJdWludDY0bWFwGAcgAygLMhkuSW50NjRUZXN0LlVpbnQ2NG1hcEVudHJ5Ugl1aW50NjRtYXASNwoJc2ludDY0bWFwGAggAygLMhkuSW50NjRUZXN0LlNpbnQ2NG1hcEVudHJ5UglzaW50NjRtYXASOgoKZml4ZWQ2NG1hcBgJIAMoCzIaLkludDY0VGVzdC5GaXhlZDY0bWFwRW50cnlSCmZpeGVkNjRtYXASPQoLc2ZpeGVkNjRtYXAYCiADKAsyGy5JbnQ2NFRlc3QuU2ZpeGVkNjRtYXBFbnRyeVILc2ZpeGVkNjRtYXASHwoGbmVzdGVkGAsgAygLMgcuTmVzdGVkUgZuZXN0ZWQaOwoNSW50NjRtYXBFbnRyeRIQCgNrZXkYASABKANSA2tleRIUCgV2YWx1ZRgCIAEoA1IFdmFsdWU6AjgBGjwKDlVpbnQ2NG1hcEVudHJ5EhAKA2tleRgBIAEoBFIDa2V5EhQKBXZhbHVlGAIgASgEUgV2YWx1ZToCOAEaPAoOU2ludDY0bWFwRW50cnkSEAoDa2V5GAEgASgSUgNrZXkSFAoFdmFsdWUYAiABKBJSBXZhbHVlOgI4ARo9Cg9GaXhlZDY0bWFwRW50cnkSEAoDa2V5GAEgASgGUgNrZXkSFAoFdmFsdWUYAiABKAZSBXZhbHVlOgI4ARo+ChBTZml4ZWQ2NG1hcEVudHJ5EhAKA2tleRgBIAEoEFIDa2V5EhQKBXZhbHVlGAIgASgQUgV2YWx1ZToCOAEi0AUKBk5lc3RlZBIUCgVpbnQ2NBgBIAEoA1IFaW50NjQSFgoGdWludDY0GAIgASgEUgZ1aW50NjQSFgoGc2ludDY0GAMgASgSUgZzaW50NjQSGAoHZml4ZWQ2NBgEIAEoBlIHZml4ZWQ2NBIaCghzZml4ZWQ2NBgFIAEoEFIIc2ZpeGVkNjQSMQoIaW50NjRtYXAYBiADKAsyFS5OZXN0ZWQuSW50NjRtYXBFbnRyeVIIaW50NjRtYXASNAoJdWludDY0bWFwGAcgAygLMhYuTmVzdGVkLlVpbnQ2NG1hcEVudHJ5Ugl1aW50NjRtYXASNAoJc2ludDY0bWFwGAggAygLMhYuTmVzdGVkLlNpbnQ2NG1hcEVudHJ5UglzaW50NjRtYXASNwoKZml4ZWQ2NG1hcBgJIAMoCzIXLk5lc3RlZC5GaXhlZDY0bWFwRW50cnlSCmZpeGVkNjRtYXASOgoLc2ZpeGVkNjRtYXAYCiADKAsyGC5OZXN0ZWQuU2ZpeGVkNjRtYXBFbnRyeVILc2ZpeGVkNjRtYXAaOwoNSW50NjRtYXBFbnRyeRIQCgNrZXkYASABKANSA2tleRIUCgV2YWx1ZRgCIAEoA1IFdmFsdWU6AjgBGjwKDlVpbnQ2NG1hcEVudHJ5EhAKA2tleRgBIAEoBFIDa2V5EhQKBXZhbHVlGAIgASgEUgV2YWx1ZToCOAEaPAoOU2ludDY0bWFwRW50cnkSEAoDa2V5GAEgASgSUgNrZXkSFAoFdmFsdWUYAiABKBJSBXZhbHVlOgI4ARo9Cg9GaXhlZDY0bWFwRW50cnkSEAoDa2V5GAEgASgGUgNrZXkSFAoFdmFsdWUYAiABKAZSBXZhbHVlOgI4ARo+ChBTZml4ZWQ2NG1hcEVudHJ5EhAKA2tleRgBIAEoEFIDa2V5EhQKBXZhbHVlGAIgASgQUgV2YWx1ZToCOAFiBnByb3RvMw==", + "base64", + ), + ); + expect( + channel.deserialize( + Buffer.from( + "088c95f0c4c5a9d28ff20110bc85a0cfc8e0c8e38a0118e7d59ff6f4acdbe01b21bc02e8890423c78a298c0a9c584c491ff23216088c95f0c4c5a9d28ff201108c95f0c4c5a9d28ff2013a1608bc85a0cfc8e0c8e38a0110bc85a0cfc8e0c8e38a01421408e7d59ff6f4acdbe01b10e7d59ff6f4acdbe01b4a1209bc02e8890423c78a11bc02e8890423c78a5212098c0a9c584c491ff2118c0a9c584c491ff25aa001088c95f0c4c5a9d28ff20110bc85a0cfc8e0c8e38a0118e7d59ff6f4acdbe01b21bc02e8890423c78a298c0a9c584c491ff23216088c95f0c4c5a9d28ff201108c95f0c4c5a9d28ff2013a1608bc85a0cfc8e0c8e38a0110bc85a0cfc8e0c8e38a01421408e7d59ff6f4acdbe01b10e7d59ff6f4acdbe01b4a1209bc02e8890423c78a11bc02e8890423c78a5212098c0a9c584c491ff2118c0a9c584c491ff2", + "hex", + ), + ), + ).toEqual({ + int64: -999999999999997300n, + uint64: 10000000000000000700n, + sint64: -999999999999997300n, + fixed64: 10000000000000000700n, + sfixed64: -999999999999997300n, + int64map: [{ key: -999999999999997300n, value: -999999999999997300n }], + uint64map: [{ key: 10000000000000000700n, value: 10000000000000000700n }], + sint64map: [{ key: -999999999999997300n, value: -999999999999997300n }], + fixed64map: [{ key: 10000000000000000700n, value: 10000000000000000700n }], + sfixed64map: [{ key: -999999999999997300n, value: -999999999999997300n }], + nested: [ + { + int64: -999999999999997300n, + uint64: 10000000000000000700n, + sint64: -999999999999997300n, + fixed64: 10000000000000000700n, + sfixed64: -999999999999997300n, + int64map: [{ key: -999999999999997300n, value: -999999999999997300n }], + uint64map: [{ key: 10000000000000000700n, value: 10000000000000000700n }], + sint64map: [{ key: -999999999999997300n, value: -999999999999997300n }], + fixed64map: [{ key: 10000000000000000700n, value: 10000000000000000700n }], + sfixed64map: [{ key: -999999999999997300n, value: -999999999999997300n }], + }, + ], + }); + }); + + it("converts protobuf time/duration int64 to number rather than bigint", () => { + const poseInFrameChannel = parseProtobufSchema( + "foxglove.PoseInFrame", + Buffer.from( + "CmcKGWZveGdsb3ZlL1F1YXRlcm5pb24ucHJvdG8SCGZveGdsb3ZlIjgKClF1YXRlcm5pb24SCQoBeBgBIAEoARIJCgF5GAIgASgBEgkKAXoYAyABKAESCQoBdxgEIAEoAWIGcHJvdG8zClYKFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8SCGZveGdsb3ZlIioKB1ZlY3RvcjMSCQoBeBgBIAEoARIJCgF5GAIgASgBEgkKAXoYAyABKAFiBnByb3RvMwqyAQoTZm94Z2xvdmUvUG9zZS5wcm90bxIIZm94Z2xvdmUaGWZveGdsb3ZlL1F1YXRlcm5pb24ucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8iVgoEUG9zZRIjCghwb3NpdGlvbhgBIAEoCzIRLmZveGdsb3ZlLlZlY3RvcjMSKQoLb3JpZW50YXRpb24YAiABKAsyFC5mb3hnbG92ZS5RdWF0ZXJuaW9uYgZwcm90bzMK7wEKH2dvb2dsZS9wcm90b2J1Zi90aW1lc3RhbXAucHJvdG8SD2dvb2dsZS5wcm90b2J1ZiIrCglUaW1lc3RhbXASDwoHc2Vjb25kcxgBIAEoAxINCgVuYW5vcxgCIAEoBUKFAQoTY29tLmdvb2dsZS5wcm90b2J1ZkIOVGltZXN0YW1wUHJvdG9QAVoyZ29vZ2xlLmdvbGFuZy5vcmcvcHJvdG9idWYvdHlwZXMva25vd24vdGltZXN0YW1wcGL4AQGiAgNHUEKqAh5Hb29nbGUuUHJvdG9idWYuV2VsbEtub3duVHlwZXNiBnByb3RvMwrSAQoaZm94Z2xvdmUvUG9zZUluRnJhbWUucHJvdG8SCGZveGdsb3ZlGhNmb3hnbG92ZS9Qb3NlLnByb3RvGh9nb29nbGUvcHJvdG9idWYvdGltZXN0YW1wLnByb3RvImwKC1Bvc2VJbkZyYW1lEi0KCXRpbWVzdGFtcBgBIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASEAoIZnJhbWVfaWQYAiABKAkSHAoEcG9zZRgDIAEoCzIOLmZveGdsb3ZlLlBvc2ViBnByb3RvMw==", + "base64", + ), + ); + + const poseInFrame = poseInFrameChannel.deserialize( + Buffer.from( + "CgwIx8LcoQYQuJzV6wESA2ZvbxooChsJAAAAAAAA8D8RAAAAAAAAAEAZAAAAAAAACEASCSEAAAAAAADwPw==", + "base64", + ), + ); + expect(poseInFrame).toMatchInlineSnapshot(` + { + "frame_id": "foo", + "pose": { + "orientation": { + "w": 1, + "x": 0, + "y": 0, + "z": 0, + }, + "position": { + "x": 1, + "y": 2, + "z": 3, + }, + }, + "timestamp": { + "nsec": 494227000, + "sec": 1681334599, + }, + } + `); + + expect(() => + poseInFrameChannel.deserialize( + Buffer.from( + "CgsIgICAgICAgBAQARIDZm9vGigKGwkAAAAAAADwPxEAAAAAAAAAQBkAAAAAAAAIQBIJIQAAAAAAAPA/", + "base64", + ), + ), + ).toThrow( + "Timestamps with seconds greater than 2^53-1 are not supported (found seconds=9007199254740992, nanos=1)", + ); + + const sceneUpdateChannel = parseProtobufSchema( + "foxglove.SceneUpdate", + Buffer.from( + "Cl0KFGZveGdsb3ZlL0NvbG9yLnByb3RvEghmb3hnbG92ZSIzCgVDb2xvchIJCgFyGAEgASgBEgkKAWcYAiABKAESCQoBYhgDIAEoARIJCgFhGAQgASgBYgZwcm90bzMKZwoZZm94Z2xvdmUvUXVhdGVybmlvbi5wcm90bxIIZm94Z2xvdmUiOAoKUXVhdGVybmlvbhIJCgF4GAEgASgBEgkKAXkYAiABKAESCQoBehgDIAEoARIJCgF3GAQgASgBYgZwcm90bzMKVgoWZm94Z2xvdmUvVmVjdG9yMy5wcm90bxIIZm94Z2xvdmUiKgoHVmVjdG9yMxIJCgF4GAEgASgBEgkKAXkYAiABKAESCQoBehgDIAEoAWIGcHJvdG8zCrIBChNmb3hnbG92ZS9Qb3NlLnByb3RvEghmb3hnbG92ZRoZZm94Z2xvdmUvUXVhdGVybmlvbi5wcm90bxoWZm94Z2xvdmUvVmVjdG9yMy5wcm90byJWCgRQb3NlEiMKCHBvc2l0aW9uGAEgASgLMhEuZm94Z2xvdmUuVmVjdG9yMxIpCgtvcmllbnRhdGlvbhgCIAEoCzIULmZveGdsb3ZlLlF1YXRlcm5pb25iBnByb3RvMwqHAgodZm94Z2xvdmUvQXJyb3dQcmltaXRpdmUucHJvdG8SCGZveGdsb3ZlGhRmb3hnbG92ZS9Db2xvci5wcm90bxoTZm94Z2xvdmUvUG9zZS5wcm90byKoAQoOQXJyb3dQcmltaXRpdmUSHAoEcG9zZRgBIAEoCzIOLmZveGdsb3ZlLlBvc2USFAoMc2hhZnRfbGVuZ3RoGAIgASgBEhYKDnNoYWZ0X2RpYW1ldGVyGAMgASgBEhMKC2hlYWRfbGVuZ3RoGAQgASgBEhUKDWhlYWRfZGlhbWV0ZXIYBSABKAESHgoFY29sb3IYBiABKAsyDy5mb3hnbG92ZS5Db2xvcmIGcHJvdG8zCuMBChxmb3hnbG92ZS9DdWJlUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8ibgoNQ3ViZVByaW1pdGl2ZRIcCgRwb3NlGAEgASgLMg4uZm94Z2xvdmUuUG9zZRIfCgRzaXplGAIgASgLMhEuZm94Z2xvdmUuVmVjdG9yMxIeCgVjb2xvchgDIAEoCzIPLmZveGdsb3ZlLkNvbG9yYgZwcm90bzMKlQIKIGZveGdsb3ZlL0N5bGluZGVyUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8imwEKEUN5bGluZGVyUHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEh8KBHNpemUYAiABKAsyES5mb3hnbG92ZS5WZWN0b3IzEhQKDGJvdHRvbV9zY2FsZRgDIAEoARIRCgl0b3Bfc2NhbGUYBCABKAESHgoFY29sb3IYBSABKAsyDy5mb3hnbG92ZS5Db2xvcmIGcHJvdG8zClsKG2ZveGdsb3ZlL0tleVZhbHVlUGFpci5wcm90bxIIZm94Z2xvdmUiKgoMS2V5VmFsdWVQYWlyEgsKA2tleRgBIAEoCRINCgV2YWx1ZRgCIAEoCWIGcHJvdG8zClQKFWZveGdsb3ZlL1BvaW50My5wcm90bxIIZm94Z2xvdmUiKQoGUG9pbnQzEgkKAXgYASABKAESCQoBeRgCIAEoARIJCgF6GAMgASgBYgZwcm90bzMKpAMKHGZveGdsb3ZlL0xpbmVQcmltaXRpdmUucHJvdG8SCGZveGdsb3ZlGhRmb3hnbG92ZS9Db2xvci5wcm90bxoVZm94Z2xvdmUvUG9pbnQzLnByb3RvGhNmb3hnbG92ZS9Qb3NlLnByb3RvIq8CCg1MaW5lUHJpbWl0aXZlEioKBHR5cGUYASABKA4yHC5mb3hnbG92ZS5MaW5lUHJpbWl0aXZlLlR5cGUSHAoEcG9zZRgCIAEoCzIOLmZveGdsb3ZlLlBvc2USEQoJdGhpY2tuZXNzGAMgASgBEhcKD3NjYWxlX2ludmFyaWFudBgEIAEoCBIgCgZwb2ludHMYBSADKAsyEC5mb3hnbG92ZS5Qb2ludDMSHgoFY29sb3IYBiABKAsyDy5mb3hnbG92ZS5Db2xvchIfCgZjb2xvcnMYByADKAsyDy5mb3hnbG92ZS5Db2xvchIPCgdpbmRpY2VzGAggAygHIjQKBFR5cGUSDgoKTElORV9TVFJJUBAAEg0KCUxJTkVfTE9PUBABEg0KCUxJTkVfTElTVBACYgZwcm90bzMKrgIKHWZveGdsb3ZlL01vZGVsUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8itwEKDk1vZGVsUHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEiAKBXNjYWxlGAIgASgLMhEuZm94Z2xvdmUuVmVjdG9yMxIeCgVjb2xvchgDIAEoCzIPLmZveGdsb3ZlLkNvbG9yEhYKDm92ZXJyaWRlX2NvbG9yGAQgASgIEgsKA3VybBgFIAEoCRISCgptZWRpYV90eXBlGAYgASgJEgwKBGRhdGEYByABKAxiBnByb3RvMwrnAQoeZm94Z2xvdmUvU3BoZXJlUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8icAoPU3BoZXJlUHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEh8KBHNpemUYAiABKAsyES5mb3hnbG92ZS5WZWN0b3IzEh4KBWNvbG9yGAMgASgLMg8uZm94Z2xvdmUuQ29sb3JiBnByb3RvMwr4AQocZm94Z2xvdmUvVGV4dFByaW1pdGl2ZS5wcm90bxIIZm94Z2xvdmUaFGZveGdsb3ZlL0NvbG9yLnByb3RvGhNmb3hnbG92ZS9Qb3NlLnByb3RvIpoBCg1UZXh0UHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEhEKCWJpbGxib2FyZBgCIAEoCBIRCglmb250X3NpemUYAyABKAESFwoPc2NhbGVfaW52YXJpYW50GAQgASgIEh4KBWNvbG9yGAUgASgLMg8uZm94Z2xvdmUuQ29sb3ISDAoEdGV4dBgGIAEoCWIGcHJvdG8zCqYCCiRmb3hnbG92ZS9UcmlhbmdsZUxpc3RQcmltaXRpdmUucHJvdG8SCGZveGdsb3ZlGhRmb3hnbG92ZS9Db2xvci5wcm90bxoVZm94Z2xvdmUvUG9pbnQzLnByb3RvGhNmb3hnbG92ZS9Qb3NlLnByb3RvIqkBChVUcmlhbmdsZUxpc3RQcmltaXRpdmUSHAoEcG9zZRgBIAEoCzIOLmZveGdsb3ZlLlBvc2USIAoGcG9pbnRzGAIgAygLMhAuZm94Z2xvdmUuUG9pbnQzEh4KBWNvbG9yGAMgASgLMg8uZm94Z2xvdmUuQ29sb3ISHwoGY29sb3JzGAQgAygLMg8uZm94Z2xvdmUuQ29sb3ISDwoHaW5kaWNlcxgFIAMoB2IGcHJvdG8zCusBCh5nb29nbGUvcHJvdG9idWYvZHVyYXRpb24ucHJvdG8SD2dvb2dsZS5wcm90b2J1ZiIqCghEdXJhdGlvbhIPCgdzZWNvbmRzGAEgASgDEg0KBW5hbm9zGAIgASgFQoMBChNjb20uZ29vZ2xlLnByb3RvYnVmQg1EdXJhdGlvblByb3RvUAFaMWdvb2dsZS5nb2xhbmcub3JnL3Byb3RvYnVmL3R5cGVzL2tub3duL2R1cmF0aW9ucGL4AQGiAgNHUEKqAh5Hb29nbGUuUHJvdG9idWYuV2VsbEtub3duVHlwZXNiBnByb3RvMwrvAQofZ29vZ2xlL3Byb3RvYnVmL3RpbWVzdGFtcC5wcm90bxIPZ29vZ2xlLnByb3RvYnVmIisKCVRpbWVzdGFtcBIPCgdzZWNvbmRzGAEgASgDEg0KBW5hbm9zGAIgASgFQoUBChNjb20uZ29vZ2xlLnByb3RvYnVmQg5UaW1lc3RhbXBQcm90b1ABWjJnb29nbGUuZ29sYW5nLm9yZy9wcm90b2J1Zi90eXBlcy9rbm93bi90aW1lc3RhbXBwYvgBAaICA0dQQqoCHkdvb2dsZS5Qcm90b2J1Zi5XZWxsS25vd25UeXBlc2IGcHJvdG8zCrIHChpmb3hnbG92ZS9TY2VuZUVudGl0eS5wcm90bxIIZm94Z2xvdmUaHWZveGdsb3ZlL0Fycm93UHJpbWl0aXZlLnByb3RvGhxmb3hnbG92ZS9DdWJlUHJpbWl0aXZlLnByb3RvGiBmb3hnbG92ZS9DeWxpbmRlclByaW1pdGl2ZS5wcm90bxobZm94Z2xvdmUvS2V5VmFsdWVQYWlyLnByb3RvGhxmb3hnbG92ZS9MaW5lUHJpbWl0aXZlLnByb3RvGh1mb3hnbG92ZS9Nb2RlbFByaW1pdGl2ZS5wcm90bxoeZm94Z2xvdmUvU3BoZXJlUHJpbWl0aXZlLnByb3RvGhxmb3hnbG92ZS9UZXh0UHJpbWl0aXZlLnByb3RvGiRmb3hnbG92ZS9UcmlhbmdsZUxpc3RQcmltaXRpdmUucHJvdG8aHmdvb2dsZS9wcm90b2J1Zi9kdXJhdGlvbi5wcm90bxofZ29vZ2xlL3Byb3RvYnVmL3RpbWVzdGFtcC5wcm90byKjBAoLU2NlbmVFbnRpdHkSLQoJdGltZXN0YW1wGAEgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIQCghmcmFtZV9pZBgCIAEoCRIKCgJpZBgDIAEoCRIrCghsaWZldGltZRgEIAEoCzIZLmdvb2dsZS5wcm90b2J1Zi5EdXJhdGlvbhIUCgxmcmFtZV9sb2NrZWQYBSABKAgSKAoIbWV0YWRhdGEYBiADKAsyFi5mb3hnbG92ZS5LZXlWYWx1ZVBhaXISKAoGYXJyb3dzGAcgAygLMhguZm94Z2xvdmUuQXJyb3dQcmltaXRpdmUSJgoFY3ViZXMYCCADKAsyFy5mb3hnbG92ZS5DdWJlUHJpbWl0aXZlEioKB3NwaGVyZXMYCSADKAsyGS5mb3hnbG92ZS5TcGhlcmVQcmltaXRpdmUSLgoJY3lsaW5kZXJzGAogAygLMhsuZm94Z2xvdmUuQ3lsaW5kZXJQcmltaXRpdmUSJgoFbGluZXMYCyADKAsyFy5mb3hnbG92ZS5MaW5lUHJpbWl0aXZlEjIKCXRyaWFuZ2xlcxgMIAMoCzIfLmZveGdsb3ZlLlRyaWFuZ2xlTGlzdFByaW1pdGl2ZRImCgV0ZXh0cxgNIAMoCzIXLmZveGdsb3ZlLlRleHRQcmltaXRpdmUSKAoGbW9kZWxzGA4gAygLMhguZm94Z2xvdmUuTW9kZWxQcmltaXRpdmViBnByb3RvMwr+AQoiZm94Z2xvdmUvU2NlbmVFbnRpdHlEZWxldGlvbi5wcm90bxIIZm94Z2xvdmUaH2dvb2dsZS9wcm90b2J1Zi90aW1lc3RhbXAucHJvdG8ipAEKE1NjZW5lRW50aXR5RGVsZXRpb24SLQoJdGltZXN0YW1wGAEgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIwCgR0eXBlGAIgASgOMiIuZm94Z2xvdmUuU2NlbmVFbnRpdHlEZWxldGlvbi5UeXBlEgoKAmlkGAMgASgJIiAKBFR5cGUSDwoLTUFUQ0hJTkdfSUQQABIHCgNBTEwQAWIGcHJvdG8zCtgBChpmb3hnbG92ZS9TY2VuZVVwZGF0ZS5wcm90bxIIZm94Z2xvdmUaGmZveGdsb3ZlL1NjZW5lRW50aXR5LnByb3RvGiJmb3hnbG92ZS9TY2VuZUVudGl0eURlbGV0aW9uLnByb3RvImgKC1NjZW5lVXBkYXRlEjAKCWRlbGV0aW9ucxgBIAMoCzIdLmZveGdsb3ZlLlNjZW5lRW50aXR5RGVsZXRpb24SJwoIZW50aXRpZXMYAiADKAsyFS5mb3hnbG92ZS5TY2VuZUVudGl0eWIGcHJvdG8z", + "base64", + ), + ); + + const sceneUpdate = sceneUpdateChannel.deserialize( + Buffer.from("EhwKDAjHwtyhBhC4nNXrASIMCMfC3KEGELic1esB", "base64"), + ); + expect(sceneUpdate).toMatchInlineSnapshot(` + { + "deletions": [], + "entities": [ + { + "arrows": [], + "cubes": [], + "cylinders": [], + "frame_id": "", + "frame_locked": false, + "id": "", + "lifetime": { + "nsec": 494227000, + "sec": 1681334599, + }, + "lines": [], + "metadata": [], + "models": [], + "spheres": [], + "texts": [], + "timestamp": { + "nsec": 494227000, + "sec": 1681334599, + }, + "triangles": [], + }, + ], + } + `); + + expect(sceneUpdateChannel.datatypes.get("google.protobuf.Timestamp")).toMatchInlineSnapshot(` + { + "definitions": [ + { + "isArray": false, + "name": "sec", + "type": "int64", + }, + { + "isArray": false, + "name": "nsec", + "type": "int32", + }, + ], + } + `); + expect(sceneUpdateChannel.datatypes.get("google.protobuf.Duration")).toMatchInlineSnapshot(` + { + "definitions": [ + { + "isArray": false, + "name": "sec", + "type": "int64", + }, + { + "isArray": false, + "name": "nsec", + "type": "int32", + }, + ], + } + `); + + // Duration too large + expect(() => + sceneUpdateChannel.deserialize(Buffer.from("EhMKBAgCEAMiCwiAgICAgICAEBAB", "base64")), + ).toThrow( + "Timestamps with seconds greater than 2^53-1 are not supported (found seconds=9007199254740992, nanos=1)", + ); + + // Timestamp too large + expect(() => + sceneUpdateChannel.deserialize(Buffer.from("EhMKCwiAgICAgICAEBABIgQIAhAD", "base64")), + ).toThrow( + "Timestamps with seconds greater than 2^53-1 are not supported (found seconds=9007199254740992, nanos=1)", + ); + }); +}); diff --git a/typescript/support/src/parseProtobufSchema.ts b/typescript/support/src/parseProtobufSchema.ts new file mode 100644 index 0000000000..22b5b85c9d --- /dev/null +++ b/typescript/support/src/parseProtobufSchema.ts @@ -0,0 +1,82 @@ +import protobufjs from "protobufjs"; +import { FileDescriptorSet } from "protobufjs/ext/descriptor"; + +import { protobufDefinitionsToDatatypes, stripLeadingDot } from "./protobufDefinitionsToDatatypes"; +import { MessageDefinitionMap } from "./types"; + +/** + * Parse a Protobuf binary schema (FileDescriptorSet) and produce datatypes and a deserializer + * function. + */ +export function parseProtobufSchema( + schemaName: string, + schemaData: Uint8Array, +): { + datatypes: MessageDefinitionMap; + deserialize: (buffer: ArrayBufferView) => unknown; +} { + const descriptorSet = FileDescriptorSet.decode(schemaData); + + const root = protobufjs.Root.fromDescriptor(descriptorSet); + root.resolveAll(); + const rootType = root.lookupType(schemaName); + + // Modify the definition of google.protobuf.Timestamp and Duration so they are interpreted as + // {sec: number, nsec: number}, compatible with the rest of Studio. The standard Protobuf types + // use different names (`seconds` and `nanos`), and `seconds` is an `int64`, which would be + // deserialized as a bigint by default. + const fixTimeType = ( + type: protobufjs.ReflectionObject | null /* eslint-disable-line no-restricted-syntax */, + ) => { + if (!type || !(type instanceof protobufjs.Type)) { + return; + } + // Rename fields so that protobufDefinitionsToDatatypes uses the new names + for (const field of type.fieldsArray) { + if (field.name === "seconds") { + field.name = "sec"; + } else if (field.name === "nanos") { + field.name = "nsec"; + } + } + type.setup(); // ensure the original optimized toObject has been created + const prevToObject = type.toObject; // eslint-disable-line @typescript-eslint/unbound-method + const newToObject: typeof prevToObject = (message, options) => { + const result = prevToObject.call(type, message, options); + const { sec, nsec } = result as { sec: bigint; nsec: number }; + if (typeof sec !== "bigint" || typeof nsec !== "number") { + return result; + } + if (sec > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new Error( + `Timestamps with seconds greater than 2^53-1 are not supported (found seconds=${sec}, nanos=${nsec})`, + ); + } + return { sec: Number(sec), nsec }; + }; + type.toObject = newToObject; + }; + + fixTimeType(root.lookup(".google.protobuf.Timestamp")); + fixTimeType(root.lookup(".google.protobuf.Duration")); + + const deserialize = (data: ArrayBufferView) => { + return rootType.toObject( + rootType.decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength)), + { defaults: true }, + ); + }; + + const datatypes: MessageDefinitionMap = new Map(); + protobufDefinitionsToDatatypes(datatypes, rootType); + + if (!datatypes.has(schemaName)) { + throw new Error( + `Protobuf schema does not contain an entry for '${schemaName}'. The schema name should be fully-qualified, e.g. '${stripLeadingDot( + rootType.fullName, + )}'.`, + ); + } + + return { deserialize, datatypes }; +} diff --git a/typescript/support/src/types.ts b/typescript/support/src/types.ts index 8067d641bf..7a540949b4 100644 --- a/typescript/support/src/types.ts +++ b/typescript/support/src/types.ts @@ -1,7 +1,3 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/ - import { MessageDefinition } from "@foxglove/message-definition"; /** A map of schema name to the schema message definition */ diff --git a/yarn.lock b/yarn.lock index a80f15e996..f44f0e36d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2999,7 +2999,7 @@ __metadata: languageName: node linkType: hard -"@foxglove/rosmsg-serialization@npm:^2.0.0": +"@foxglove/rosmsg-serialization@npm:^2.0.1": version: 2.0.1 resolution: "@foxglove/rosmsg-serialization@npm:2.0.1" dependencies: @@ -3019,7 +3019,7 @@ __metadata: languageName: node linkType: hard -"@foxglove/rosmsg2-serialization@npm:^2.0.0": +"@foxglove/rosmsg2-serialization@npm:^2.0.2": version: 2.0.2 resolution: "@foxglove/rosmsg2-serialization@npm:2.0.2" dependencies: @@ -3039,7 +3039,7 @@ __metadata: languageName: node linkType: hard -"@foxglove/rosmsg@npm:^4.0.0": +"@foxglove/rosmsg@npm:^4.0.0, @foxglove/rosmsg@npm:^4.2.2": version: 4.2.2 resolution: "@foxglove/rosmsg@npm:4.2.2" dependencies: @@ -3068,7 +3068,7 @@ __metadata: languageName: node linkType: hard -"@foxglove/schemas@npm:^1.0.0, @foxglove/schemas@npm:^1.1.0": +"@foxglove/schemas@npm:^1.0.0, @foxglove/schemas@npm:^1.3.1": version: 1.3.1 resolution: "@foxglove/schemas@npm:1.3.1" dependencies: @@ -3548,10 +3548,10 @@ __metadata: dependencies: "@foxglove/eslint-plugin": 0.21.0 "@foxglove/message-definition": ^0.2.0 - "@foxglove/rosmsg": ^4.0.0 - "@foxglove/rosmsg-serialization": ^2.0.0 - "@foxglove/rosmsg2-serialization": ^2.0.0 - "@foxglove/schemas": ^1.1.0 + "@foxglove/rosmsg": ^4.2.2 + "@foxglove/rosmsg-serialization": ^2.0.1 + "@foxglove/rosmsg2-serialization": ^2.0.2 + "@foxglove/schemas": ^1.3.1 "@foxglove/tsconfig": 1.1.0 "@foxglove/wasm-bz2": ^0.1.1 "@foxglove/wasm-lz4": ^1.0.2 @@ -3570,8 +3570,8 @@ __metadata: eslint-plugin-import: 2.27.5 eslint-plugin-jest: 27.2.1 eslint-plugin-prettier: 4.2.1 - flatbuffers: ^23.1.21 - flatbuffers_reflection: ^0.0.3 + flatbuffers: ^23.5.26 + flatbuffers_reflection: ^0.0.6 jest: 29.4.3 prettier: 2.8.4 protobufjs: ^7.2.2 @@ -8135,17 +8135,17 @@ __metadata: languageName: node linkType: hard -"flatbuffers@npm:^23.1.21": - version: 23.1.21 - resolution: "flatbuffers@npm:23.1.21" - checksum: 6e890adb8300e5352d579abf7c674eec7d838c27e82981c3d9cd5a4043ff83fc7097a88eae68729eb7cff6ded2b1a896ee2009632027d613040bef669290fbc6 +"flatbuffers@npm:^23.1.21, flatbuffers@npm:^23.5.26": + version: 23.5.26 + resolution: "flatbuffers@npm:23.5.26" + checksum: 2528e549118d02fca9775cc9c17cf445741bb58b0a9575eb07bba823394a79dccb3bf97c48561720aa61d3dc9f42759129df0e05eb94c12f809cfb6126eaedd1 languageName: node linkType: hard -"flatbuffers_reflection@npm:^0.0.3": - version: 0.0.3 - resolution: "flatbuffers_reflection@npm:0.0.3" - checksum: bcef0af4ec41d6a21c9f6ae1df5be2e329c286d2ae566e09190ff585252b7ec6fbf79035c6ea255bc7c821851516b8bdcb5bf2f6c505e63a242e74c3fcd38cfe +"flatbuffers_reflection@npm:^0.0.6": + version: 0.0.6 + resolution: "flatbuffers_reflection@npm:0.0.6" + checksum: 65cc9aa9344705b50c106fed739b154b722f212e8af00663d035581914d5e7c229757410a54beb0d267bc6af0983032fe723d81af758d51c5c76cc8ed6c9f5de languageName: node linkType: hard From 9c09f137bf0f360b142850dad9c8e97b69ec21de Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 30 Jun 2023 15:29:11 -0700 Subject: [PATCH 04/27] wheeeee --- .github/workflows/ci.yml | 20 +- .vscode/settings.json | 2 +- package.json | 5 + .../TypescriptIndexedReaderTestRunner.ts | 2 +- tests/conformance/tsconfig.json | 1 - typescript/examples/bag2mcap/.eslintrc.js | 5 +- typescript/examples/bag2mcap/package.json | 5 +- .../examples/bag2mcap/scripts/bag2mcap.ts | 2 +- typescript/examples/bag2mcap/tsconfig.json | 3 +- .../examples/bag2mcap/typings/protobufjs.d.ts | 14 + typescript/examples/basicwriter/.eslintrc.js | 5 +- typescript/examples/basicwriter/package.json | 3 +- .../examples/basicwriter/scripts/main.ts | 2 +- typescript/examples/basicwriter/tsconfig.json | 3 +- .../examples/flatbufferswriter/.eslintrc.js | 5 +- .../examples/flatbufferswriter/package.json | 4 +- .../flatbufferswriter/scripts/main.ts | 2 +- .../examples/flatbufferswriter/tsconfig.json | 3 +- .../text-annotation-demo/.eslintrc.js | 5 +- .../text-annotation-demo/package.json | 3 +- .../text-annotation-demo/scripts/main.ts | 24 +- .../text-annotation-demo/tsconfig.json | 4 +- typescript/examples/validate/.eslintrc.js | 5 +- typescript/examples/validate/package.json | 5 +- .../examples/validate/scripts/validate.ts | 2 +- typescript/examples/validate/tsconfig.json | 3 +- typescript/jest.config.json | 3 + typescript/nodejs/.eslintrc.js | 28 ++ typescript/nodejs/README.md | 50 +++ typescript/nodejs/jest.config.json | 19 ++ typescript/nodejs/package.json | 57 ++++ .../src}/FileHandleReadable.ts | 2 +- .../src}/FileHandleWritable.ts | 0 typescript/nodejs/src/index.test.ts | 52 +++ .../src/nodejs => nodejs/src}/index.ts | 0 typescript/nodejs/tsconfig.cjs.json | 7 + typescript/nodejs/tsconfig.json | 11 + typescript/support/.eslintrc.js | 5 +- typescript/support/nodejs.d.ts | 1 - typescript/support/nodejs.js | 1 - typescript/support/package.json | 2 - .../support/src/parseFlatbufferSchema.test.ts | 2 + typescript/support/src/parseProtobufSchema.ts | 4 +- yarn.lock | 314 +++++++++++++++++- 44 files changed, 622 insertions(+), 73 deletions(-) create mode 100644 typescript/examples/bag2mcap/typings/protobufjs.d.ts create mode 100644 typescript/jest.config.json create mode 100644 typescript/nodejs/.eslintrc.js create mode 100644 typescript/nodejs/README.md create mode 100644 typescript/nodejs/jest.config.json create mode 100644 typescript/nodejs/package.json rename typescript/{support/src/nodejs => nodejs/src}/FileHandleReadable.ts (100%) rename typescript/{support/src/nodejs => nodejs/src}/FileHandleWritable.ts (100%) create mode 100644 typescript/nodejs/src/index.test.ts rename typescript/{support/src/nodejs => nodejs/src}/index.ts (100%) create mode 100644 typescript/nodejs/tsconfig.cjs.json create mode 100644 typescript/nodejs/tsconfig.json delete mode 100644 typescript/support/nodejs.d.ts delete mode 100644 typescript/support/nodejs.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddabc438ad..0adca55d0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -213,10 +213,9 @@ jobs: - run: yarn prettier:check - run: yarn workspace @mcap/core lint:ci - run: yarn workspace @mcap/core typecheck - - run: yarn workspace @mcap/core test - run: yarn workspace @mcap/support lint:ci - run: yarn workspace @mcap/support typecheck - - run: yarn workspace @mcap/support test + - run: yarn typescript:test - name: Publish @mcap/core to NPM if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/core/v') }} @@ -224,11 +223,18 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} - - name: Publish @mcap/support to NPM - if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/support/v') }} - run: yarn workspace @mcap/support publish --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + # To be enabled once API has settled + # - name: Publish @mcap/nodejs to NPM + # if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/nodejs/v') }} + # run: yarn workspace @mcap/nodejs publish --access public + # env: + # NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + + # - name: Publish @mcap/support to NPM + # if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/support/v') }} + # run: yarn workspace @mcap/support publish --access public + # env: + # NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} typescript-examples: runs-on: ubuntu-latest diff --git a/.vscode/settings.json b/.vscode/settings.json index 0c0acd744c..c4fbc14b68 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -14,7 +14,7 @@ "eslint.options": { "reportUnusedDisableDirectives": "error" }, - "jest.jestCommandLine": "yarn workspace @mcap/core test", + "jest.jestCommandLine": "yarn typescript:test", "cSpell.enabled": true, diff --git a/package.json b/package.json index 7021d08fee..7778ee4f99 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "typescript/core", "typescript/examples/*", "typescript/support", + "typescript/nodejs", "website" ] }, @@ -17,6 +18,7 @@ "typedoc": "typedoc --out __docs__/typescript --options typescript/typedoc.json", "start": "yarn workspace website start", "spellcheck": "cspell --relative '**'", + "typescript:test": "yarn jest --config typescript/jest.config.json", "typescript:build": "yarn workspace @mcap/core build && yarn workspace @mcap/support build", "typescript:clean": "yarn workspace @mcap/core build --clean && yarn workspace @mcap/support build --clean", "test:conformance:generate-inputs": "yarn workspace @foxglove/mcap-conformance generate-inputs --data-dir \"$(pwd)/tests/conformance/data\"", @@ -25,7 +27,10 @@ "packageManager": "yarn@3.5.0", "devDependencies": { "cspell": "^6.26.3", + "jest": "29.4.3", "prettier": "^2.8.4", + "ts-jest": "29.0.5", + "ts-node": "10.9.1", "typedoc": "^0.23.25" } } diff --git a/tests/conformance/scripts/run-tests/runners/TypescriptIndexedReaderTestRunner.ts b/tests/conformance/scripts/run-tests/runners/TypescriptIndexedReaderTestRunner.ts index 0af798935b..cf2ed0706b 100644 --- a/tests/conformance/scripts/run-tests/runners/TypescriptIndexedReaderTestRunner.ts +++ b/tests/conformance/scripts/run-tests/runners/TypescriptIndexedReaderTestRunner.ts @@ -1,5 +1,5 @@ import { McapIndexedReader } from "@mcap/core"; -import { FileHandleReadable } from "@mcap/support/nodejs"; +import { FileHandleReadable } from "@mcap/nodejs"; import fs from "fs/promises"; import { TestFeatures, TestVariant } from "variants/types"; diff --git a/tests/conformance/tsconfig.json b/tests/conformance/tsconfig.json index 05b1a76f34..0ced8703c3 100644 --- a/tests/conformance/tsconfig.json +++ b/tests/conformance/tsconfig.json @@ -7,7 +7,6 @@ "lib": ["es2020", "dom"], "paths": { "@mcap/core": ["../../typescript/core/src"], - "@mcap/support/nodejs": ["../../typescript/src/nodejs"], "@mcap/support": ["../../typescript/support/src"] }, diff --git a/typescript/examples/bag2mcap/.eslintrc.js b/typescript/examples/bag2mcap/.eslintrc.js index 8eb03eee3d..cabff403f4 100644 --- a/typescript/examples/bag2mcap/.eslintrc.js +++ b/typescript/examples/bag2mcap/.eslintrc.js @@ -8,8 +8,11 @@ module.exports = { files: ["*.ts", "*.tsx"], extends: ["plugin:@foxglove/typescript"], parserOptions: { - project: "tsconfig.json", + project: ["tsconfig.json", "../../*/tsconfig.json"], tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, }, }, ], diff --git a/typescript/examples/bag2mcap/package.json b/typescript/examples/bag2mcap/package.json index 7891900b50..8fd2ebbe3a 100644 --- a/typescript/examples/bag2mcap/package.json +++ b/typescript/examples/bag2mcap/package.json @@ -29,8 +29,9 @@ "@foxglove/wasm-bz2": "0.1.1", "@foxglove/wasm-lz4": "1.0.2", "@foxglove/wasm-zstd": "1.0.1", - "@mcap/core": "*", - "@mcap/support": "*", + "@mcap/core": "workspace:*", + "@mcap/nodejs": "workspace:*", + "@mcap/support": "workspace:*", "@types/lodash": "4.14.191", "@types/node": "18.13.0", "@typescript-eslint/eslint-plugin": "5.52.0", diff --git a/typescript/examples/bag2mcap/scripts/bag2mcap.ts b/typescript/examples/bag2mcap/scripts/bag2mcap.ts index 6c8a2a680c..42f175da0c 100644 --- a/typescript/examples/bag2mcap/scripts/bag2mcap.ts +++ b/typescript/examples/bag2mcap/scripts/bag2mcap.ts @@ -9,8 +9,8 @@ import Bzip2 from "@foxglove/wasm-bz2"; import decompressLZ4 from "@foxglove/wasm-lz4"; import zstd from "@foxglove/wasm-zstd"; import { McapWriter, McapTypes } from "@mcap/core"; +import { FileHandleWritable } from "@mcap/nodejs"; import { ProtobufDescriptor, protobufToDescriptor } from "@mcap/support"; -import { FileHandleWritable } from "@mcap/support/nodejs"; import { program } from "commander"; import { open } from "fs/promises"; import protobufjs from "protobufjs"; diff --git a/typescript/examples/bag2mcap/tsconfig.json b/typescript/examples/bag2mcap/tsconfig.json index 5d97e53644..b838aaccfb 100644 --- a/typescript/examples/bag2mcap/tsconfig.json +++ b/typescript/examples/bag2mcap/tsconfig.json @@ -8,7 +8,8 @@ "lib": ["es2020", "dom"], "paths": { "@mcap/core": ["../../core/src"], - "@mcap/support": ["../../support/src"] + "@mcap/support": ["../../support/src"], + "@mcap/nodejs": ["../../nodejs/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/examples/bag2mcap/typings/protobufjs.d.ts b/typescript/examples/bag2mcap/typings/protobufjs.d.ts new file mode 100644 index 0000000000..8c0445835e --- /dev/null +++ b/typescript/examples/bag2mcap/typings/protobufjs.d.ts @@ -0,0 +1,14 @@ +import protobufjs from "protobufjs"; +import descriptor from "protobufjs/ext/descriptor"; + +// https://github.com/protobufjs/protobuf.js/issues/1499 +declare module "protobufjs" { + interface ReflectionObject { + toDescriptor( + protoVersion: string, + ): protobufjs.Message & descriptor.IFileDescriptorSet; + } + declare namespace ReflectionObject { + export const fromDescriptor: (desc: protobufjs.Message) => protobufjs.Root; + } +} diff --git a/typescript/examples/basicwriter/.eslintrc.js b/typescript/examples/basicwriter/.eslintrc.js index 8eb03eee3d..cabff403f4 100644 --- a/typescript/examples/basicwriter/.eslintrc.js +++ b/typescript/examples/basicwriter/.eslintrc.js @@ -8,8 +8,11 @@ module.exports = { files: ["*.ts", "*.tsx"], extends: ["plugin:@foxglove/typescript"], parserOptions: { - project: "tsconfig.json", + project: ["tsconfig.json", "../../*/tsconfig.json"], tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, }, }, ], diff --git a/typescript/examples/basicwriter/package.json b/typescript/examples/basicwriter/package.json index bf1a291e54..7b4cd6d30a 100644 --- a/typescript/examples/basicwriter/package.json +++ b/typescript/examples/basicwriter/package.json @@ -21,7 +21,8 @@ }, "devDependencies": { "@foxglove/eslint-plugin": "0.21.0", - "@mcap/core": "*", + "@mcap/core": "workspace:*", + "@mcap/nodejs": "workspace:*", "@mcap/support": "*", "@types/node": "18.13.0", "eslint": "8.34.0", diff --git a/typescript/examples/basicwriter/scripts/main.ts b/typescript/examples/basicwriter/scripts/main.ts index 94aa90d910..3e6f6c9858 100644 --- a/typescript/examples/basicwriter/scripts/main.ts +++ b/typescript/examples/basicwriter/scripts/main.ts @@ -1,6 +1,6 @@ import { McapWriter } from "@mcap/core"; +import { FileHandleWritable } from "@mcap/nodejs"; import { open } from "fs/promises"; -import { FileHandleWritable } from "@mcap/support/nodejs"; async function main() { const mcapFilePath = "output.mcap"; diff --git a/typescript/examples/basicwriter/tsconfig.json b/typescript/examples/basicwriter/tsconfig.json index 5d97e53644..b838aaccfb 100644 --- a/typescript/examples/basicwriter/tsconfig.json +++ b/typescript/examples/basicwriter/tsconfig.json @@ -8,7 +8,8 @@ "lib": ["es2020", "dom"], "paths": { "@mcap/core": ["../../core/src"], - "@mcap/support": ["../../support/src"] + "@mcap/support": ["../../support/src"], + "@mcap/nodejs": ["../../nodejs/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/examples/flatbufferswriter/.eslintrc.js b/typescript/examples/flatbufferswriter/.eslintrc.js index 8eb03eee3d..cabff403f4 100644 --- a/typescript/examples/flatbufferswriter/.eslintrc.js +++ b/typescript/examples/flatbufferswriter/.eslintrc.js @@ -8,8 +8,11 @@ module.exports = { files: ["*.ts", "*.tsx"], extends: ["plugin:@foxglove/typescript"], parserOptions: { - project: "tsconfig.json", + project: ["tsconfig.json", "../../*/tsconfig.json"], tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, }, }, ], diff --git a/typescript/examples/flatbufferswriter/package.json b/typescript/examples/flatbufferswriter/package.json index 966c881c26..dfe1225c7c 100644 --- a/typescript/examples/flatbufferswriter/package.json +++ b/typescript/examples/flatbufferswriter/package.json @@ -22,8 +22,8 @@ "devDependencies": { "@foxglove/eslint-plugin": "0.21.0", "@foxglove/schemas": "^1.0.0", - "@mcap/core": "*", - "@mcap/support": "*", + "@mcap/core": "workspace:*", + "@mcap/nodejs": "workspace:*", "@types/node": "18.13.0", "eslint": "8.34.0", "eslint-config-prettier": "8.6.0", diff --git a/typescript/examples/flatbufferswriter/scripts/main.ts b/typescript/examples/flatbufferswriter/scripts/main.ts index 3ac09ef8fc..59ba0c4144 100644 --- a/typescript/examples/flatbufferswriter/scripts/main.ts +++ b/typescript/examples/flatbufferswriter/scripts/main.ts @@ -1,9 +1,9 @@ import { Grid, NumericType } from "@foxglove/schemas"; import { McapWriter } from "@mcap/core"; +import { FileHandleWritable } from "@mcap/nodejs"; import { Builder } from "flatbuffers"; import fs from "fs"; import { open } from "fs/promises"; -import { FileHandleWritable } from "@mcap/support/nodejs"; import { buildGridMessage, buildTfMessage } from "./flatbufferUtils"; diff --git a/typescript/examples/flatbufferswriter/tsconfig.json b/typescript/examples/flatbufferswriter/tsconfig.json index 5d97e53644..b838aaccfb 100644 --- a/typescript/examples/flatbufferswriter/tsconfig.json +++ b/typescript/examples/flatbufferswriter/tsconfig.json @@ -8,7 +8,8 @@ "lib": ["es2020", "dom"], "paths": { "@mcap/core": ["../../core/src"], - "@mcap/support": ["../../support/src"] + "@mcap/support": ["../../support/src"], + "@mcap/nodejs": ["../../nodejs/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/examples/text-annotation-demo/.eslintrc.js b/typescript/examples/text-annotation-demo/.eslintrc.js index 8eb03eee3d..cabff403f4 100644 --- a/typescript/examples/text-annotation-demo/.eslintrc.js +++ b/typescript/examples/text-annotation-demo/.eslintrc.js @@ -8,8 +8,11 @@ module.exports = { files: ["*.ts", "*.tsx"], extends: ["plugin:@foxglove/typescript"], parserOptions: { - project: "tsconfig.json", + project: ["tsconfig.json", "../../*/tsconfig.json"], tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, }, }, ], diff --git a/typescript/examples/text-annotation-demo/package.json b/typescript/examples/text-annotation-demo/package.json index ed299e2caa..ccc7c7784a 100644 --- a/typescript/examples/text-annotation-demo/package.json +++ b/typescript/examples/text-annotation-demo/package.json @@ -21,7 +21,8 @@ }, "devDependencies": { "@foxglove/eslint-plugin": "0.21.0", - "@mcap/core": "*", + "@mcap/core": "workspace:*", + "@mcap/nodejs": "workspace:*", "@types/node": "18.13.0", "eslint": "8.34.0", "eslint-config-prettier": "8.6.0", diff --git a/typescript/examples/text-annotation-demo/scripts/main.ts b/typescript/examples/text-annotation-demo/scripts/main.ts index e9908213ad..4bb99b0121 100644 --- a/typescript/examples/text-annotation-demo/scripts/main.ts +++ b/typescript/examples/text-annotation-demo/scripts/main.ts @@ -4,30 +4,12 @@ import { ImageAnnotations as ImageAnnotationsSchema, } from "@foxglove/schemas/jsonschema"; import { Time } from "@foxglove/schemas/schemas/typescript/Time"; -import { McapWriter, IWritable } from "@mcap/core"; -import { open, FileHandle } from "fs/promises"; +import { McapWriter } from "@mcap/core"; +import { FileHandleWritable } from "@mcap/nodejs"; +import { open } from "fs/promises"; import Scene from "./Scene"; -// Mcap IWritable interface for nodejs FileHandle -class FileHandleWritable implements IWritable { - private handle: FileHandle; - private totalBytesWritten = 0; - - constructor(handle: FileHandle) { - this.handle = handle; - } - - async write(buffer: Uint8Array): Promise { - const written = await this.handle.write(buffer); - this.totalBytesWritten += written.bytesWritten; - } - - position(): bigint { - return BigInt(this.totalBytesWritten); - } -} - const framesPerSecond = 30; const lengthSeconds = 10; // seconds diff --git a/typescript/examples/text-annotation-demo/tsconfig.json b/typescript/examples/text-annotation-demo/tsconfig.json index 13b7e779c7..b838aaccfb 100644 --- a/typescript/examples/text-annotation-demo/tsconfig.json +++ b/typescript/examples/text-annotation-demo/tsconfig.json @@ -7,7 +7,9 @@ "outDir": "./dist/esm", "lib": ["es2020", "dom"], "paths": { - "@mcap/core": ["../../core/src"] + "@mcap/core": ["../../core/src"], + "@mcap/support": ["../../support/src"], + "@mcap/nodejs": ["../../nodejs/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/examples/validate/.eslintrc.js b/typescript/examples/validate/.eslintrc.js index 8eb03eee3d..cabff403f4 100644 --- a/typescript/examples/validate/.eslintrc.js +++ b/typescript/examples/validate/.eslintrc.js @@ -8,8 +8,11 @@ module.exports = { files: ["*.ts", "*.tsx"], extends: ["plugin:@foxglove/typescript"], parserOptions: { - project: "tsconfig.json", + project: ["tsconfig.json", "../../*/tsconfig.json"], tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, }, }, ], diff --git a/typescript/examples/validate/package.json b/typescript/examples/validate/package.json index da3343f970..42e9836e9e 100644 --- a/typescript/examples/validate/package.json +++ b/typescript/examples/validate/package.json @@ -29,8 +29,9 @@ "@foxglove/wasm-bz2": "0.1.1", "@foxglove/wasm-lz4": "1.0.2", "@foxglove/wasm-zstd": "1.0.1", - "@mcap/core": "*", - "@mcap/support": "*", + "@mcap/core": "workspace:*", + "@mcap/nodejs": "workspace:*", + "@mcap/support": "workspace:*", "@types/lodash": "4.14.191", "@types/node": "18.13.0", "@typescript-eslint/eslint-plugin": "5.52.0", diff --git a/typescript/examples/validate/scripts/validate.ts b/typescript/examples/validate/scripts/validate.ts index c6740449ef..5585609ec8 100644 --- a/typescript/examples/validate/scripts/validate.ts +++ b/typescript/examples/validate/scripts/validate.ts @@ -8,8 +8,8 @@ import { McapStreamReader, McapTypes, } from "@mcap/core"; +import { FileHandleReadable } from "@mcap/nodejs"; import { loadDecompressHandlers, protobufFromBinaryDescriptor } from "@mcap/support"; -import { FileHandleReadable } from "@mcap/support/nodejs"; import { program } from "commander"; import { createReadStream } from "fs"; import fs from "fs/promises"; diff --git a/typescript/examples/validate/tsconfig.json b/typescript/examples/validate/tsconfig.json index 5d97e53644..b838aaccfb 100644 --- a/typescript/examples/validate/tsconfig.json +++ b/typescript/examples/validate/tsconfig.json @@ -8,7 +8,8 @@ "lib": ["es2020", "dom"], "paths": { "@mcap/core": ["../../core/src"], - "@mcap/support": ["../../support/src"] + "@mcap/support": ["../../support/src"], + "@mcap/nodejs": ["../../nodejs/src"] }, // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 diff --git a/typescript/jest.config.json b/typescript/jest.config.json new file mode 100644 index 0000000000..f631460334 --- /dev/null +++ b/typescript/jest.config.json @@ -0,0 +1,3 @@ +{ + "projects": ["/*/jest.config.json"] +} diff --git a/typescript/nodejs/.eslintrc.js b/typescript/nodejs/.eslintrc.js new file mode 100644 index 0000000000..85bdbf92d7 --- /dev/null +++ b/typescript/nodejs/.eslintrc.js @@ -0,0 +1,28 @@ +/* eslint-env node */ +module.exports = { + env: { es2020: true }, + ignorePatterns: ["dist"], + extends: ["plugin:@foxglove/base", "plugin:@foxglove/jest", "plugin:import/recommended"], + overrides: [ + { + files: ["*.ts", "*.tsx"], + extends: ["plugin:@foxglove/typescript"], + parserOptions: { + project: "../*/tsconfig.json", + tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, + }, + }, + ], + rules: { + "no-warning-comments": ["error", { terms: ["fixme"], location: "anywhere" }], + }, + settings: { + "import/resolver": { + typescript: true, + node: true, + }, + }, +}; diff --git a/typescript/nodejs/README.md b/typescript/nodejs/README.md new file mode 100644 index 0000000000..391fc68de8 --- /dev/null +++ b/typescript/nodejs/README.md @@ -0,0 +1,50 @@ +# @mcap/nodejs + +[MCAP](https://github.com/foxglove/mcap) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. + +The `@mcap/nodejs` package provides utilities for working with MCAP files from Node.js. + +## Usage examples + +### Reading MCAP files + +```ts +import { loadDecompressHandlers } from "@mcap/support"; +import { FileHandleReadable } from "@mcap/nodejs"; +import { McapIndexedReader } from "@mcap/core"; +import { open } from "fs/promises"; + +const decompressHandlers = await loadDecompressHandlers(); +const fileHandle = await open("file.mcap", "r"); +const reader = await McapIndexedReader.Initialize({ + readable: new FileHandleReadable(fileHandle), + decompressHandlers, +}); +``` + +### Writing MCAP files + +```ts +import zstd from "@foxglove/wasm-zstd"; +import { FileHandleWritable } from "@mcap/support/nodejs"; +import { McapWriter } from "@mcap/core"; +import { open } from "fs/promises"; + +await zstd.isLoaded; +const fileHandle = await open("file.mcap", "wx"); +const writer = new McapWriter({ + writable: new FileHandleWritable(fileHandle), + compressChunk: (data) => ({ + compression: "zstd", + compressedData: zstd.compress(data), + }), +}); +``` + +## License + +`@mcap/nodejs` is licensed under the [MIT License](https://opensource.org/licenses/MIT). + +## Stay in touch + +Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/typescript/nodejs/jest.config.json b/typescript/nodejs/jest.config.json new file mode 100644 index 0000000000..09337c2e52 --- /dev/null +++ b/typescript/nodejs/jest.config.json @@ -0,0 +1,19 @@ +{ + "testMatch": ["/src/**/*.test.ts"], + "transform": { + "^.+\\.ts$": [ + "ts-jest", + { + "diagnostics": { + "//": "add 6133 (unused variables) to default ignore codes", + "ignoreCodes": [6059, 18002, 18003, 6133] + } + } + ] + }, + "moduleNameMapper": { + "^@mcap/core$": "/../core/src" + }, + "//": "Native find is slow because it does not exclude files: https://github.com/facebook/jest/pull/11264#issuecomment-825377579", + "haste": { "forceNodeFilesystemAPI": true } +} diff --git a/typescript/nodejs/package.json b/typescript/nodejs/package.json new file mode 100644 index 0000000000..a111d20f00 --- /dev/null +++ b/typescript/nodejs/package.json @@ -0,0 +1,57 @@ +{ + "name": "@mcap/nodejs", + "version": "0.1.0", + "description": "Support library for using MCAP in Node.js", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/foxglove/mcap.git" + }, + "author": { + "name": "Foxglove Technologies", + "email": "support@foxglove.dev" + }, + "homepage": "https://foxglove.dev/", + "module": "dist/esm/src/index.js", + "main": "dist/cjs/src/index.js", + "typings": "dist/esm/src/index.d.ts", + "typedoc": { + "entryPoint": "src/index.ts" + }, + "files": [ + "dist", + "src" + ], + "engines": { + "node": ">=14.0.0" + }, + "scripts": { + "build": "tsc -b tsconfig.json tsconfig.cjs.json", + "prepack": "yarn build", + "typecheck": "tsc -p tsconfig.json --noEmit", + "lint:ci": "eslint --report-unused-disable-directives .", + "lint": "eslint --report-unused-disable-directives --fix .", + "test": "jest" + }, + "devDependencies": { + "@foxglove/eslint-plugin": "0.21.0", + "@foxglove/tsconfig": "1.1.0", + "@mcap/core": "workspace:*", + "@types/jest": "29.4.0", + "@types/node": "18.13.0", + "@typescript-eslint/eslint-plugin": "5.52.0", + "@typescript-eslint/parser": "5.52.0", + "eslint": "8.34.0", + "eslint-config-prettier": "8.6.0", + "eslint-import-resolver-typescript": "3.5.5", + "eslint-plugin-es": "4.1.0", + "eslint-plugin-filenames": "1.3.2", + "eslint-plugin-import": "2.27.5", + "eslint-plugin-jest": "27.2.1", + "eslint-plugin-prettier": "4.2.1", + "jest": "29.4.3", + "prettier": "2.8.4", + "ts-jest": "29.0.5", + "typescript": "4.9.5" + } +} diff --git a/typescript/support/src/nodejs/FileHandleReadable.ts b/typescript/nodejs/src/FileHandleReadable.ts similarity index 100% rename from typescript/support/src/nodejs/FileHandleReadable.ts rename to typescript/nodejs/src/FileHandleReadable.ts index 6196335042..e09e2cda8d 100644 --- a/typescript/support/src/nodejs/FileHandleReadable.ts +++ b/typescript/nodejs/src/FileHandleReadable.ts @@ -1,5 +1,5 @@ -import { FileHandle } from "fs/promises"; import type { McapTypes } from "@mcap/core"; +import { FileHandle } from "fs/promises"; /** * IReadable implementation for FileHandle. diff --git a/typescript/support/src/nodejs/FileHandleWritable.ts b/typescript/nodejs/src/FileHandleWritable.ts similarity index 100% rename from typescript/support/src/nodejs/FileHandleWritable.ts rename to typescript/nodejs/src/FileHandleWritable.ts diff --git a/typescript/nodejs/src/index.test.ts b/typescript/nodejs/src/index.test.ts new file mode 100644 index 0000000000..b9969eb09d --- /dev/null +++ b/typescript/nodejs/src/index.test.ts @@ -0,0 +1,52 @@ +import { McapWriter, McapIndexedReader } from "@mcap/core"; +import { mkdtemp, open, rm } from "fs/promises"; +import { tmpdir } from "os"; +import path from "path"; + +import { FileHandleReadable } from "./FileHandleReadable"; +import { FileHandleWritable } from "./FileHandleWritable"; + +async function collect(iterable: AsyncIterable): Promise { + const result: T[] = []; + for await (const item of iterable) { + result.push(item); + } + return result; +} + +describe("FileHandleReadable & FileHandleWritable", () => { + it("roundtrips", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "mcap-test")); + try { + const handle = await open(path.join(dir, "test.mcap"), "wx+"); + + const header = { library: "lib", profile: "prof" }; + const writer = new McapWriter({ writable: new FileHandleWritable(handle) }); + await writer.start(header); + const channel = { + topic: "foo", + schemaId: 0, + messageEncoding: "enc", + metadata: new Map(), + }; + const channelId = await writer.registerChannel(channel); + const message = { + channelId, + sequence: 1, + logTime: 1n, + publishTime: 2n, + data: new Uint8Array([1, 2, 3]), + }; + await writer.addMessage(message); + await writer.end(); + + const reader = await McapIndexedReader.Initialize({ + readable: new FileHandleReadable(handle), + }); + expect(reader.header).toEqual({ ...header, type: "Header" }); + expect(await collect(reader.readMessages())).toEqual([{ ...message, type: "Message" }]); + } finally { + await rm(dir, { recursive: true }); + } + }); +}); diff --git a/typescript/support/src/nodejs/index.ts b/typescript/nodejs/src/index.ts similarity index 100% rename from typescript/support/src/nodejs/index.ts rename to typescript/nodejs/src/index.ts diff --git a/typescript/nodejs/tsconfig.cjs.json b/typescript/nodejs/tsconfig.cjs.json new file mode 100644 index 0000000000..ac31cb3343 --- /dev/null +++ b/typescript/nodejs/tsconfig.cjs.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist/cjs", + "module": "commonjs" + } +} diff --git a/typescript/nodejs/tsconfig.json b/typescript/nodejs/tsconfig.json new file mode 100644 index 0000000000..6a6b398a2e --- /dev/null +++ b/typescript/nodejs/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "@foxglove/tsconfig/base", + "include": ["./src/**/*"], + "compilerOptions": { + "outDir": "./dist/esm", + "lib": ["es2020", "dom"], + "composite": true, + "incremental": true + }, + "references": [{ "path": "../core" }] +} diff --git a/typescript/support/.eslintrc.js b/typescript/support/.eslintrc.js index 073f6b2fa3..1b945176f6 100644 --- a/typescript/support/.eslintrc.js +++ b/typescript/support/.eslintrc.js @@ -8,8 +8,11 @@ module.exports = { files: ["*.ts", "*.tsx"], extends: ["plugin:@foxglove/typescript"], parserOptions: { - project: "tsconfig.json", + project: "../*/tsconfig.json", tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, }, }, ], diff --git a/typescript/support/nodejs.d.ts b/typescript/support/nodejs.d.ts deleted file mode 100644 index 07ef4f5705..0000000000 --- a/typescript/support/nodejs.d.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./src/nodejs"; diff --git a/typescript/support/nodejs.js b/typescript/support/nodejs.js deleted file mode 100644 index 557ef34e07..0000000000 --- a/typescript/support/nodejs.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./dist/cjs/nodejs/index.js"); diff --git a/typescript/support/package.json b/typescript/support/package.json index d777806970..a5da728e67 100644 --- a/typescript/support/package.json +++ b/typescript/support/package.json @@ -50,7 +50,6 @@ "@types/node": "18.13.0", "@typescript-eslint/eslint-plugin": "5.52.0", "@typescript-eslint/parser": "5.52.0", - "cspell": "^6.26.3", "eslint": "8.34.0", "eslint-config-prettier": "8.6.0", "eslint-plugin-es": "4.1.0", @@ -61,7 +60,6 @@ "jest": "29.4.3", "prettier": "2.8.4", "ts-jest": "29.0.5", - "typedoc": "^0.23.25", "typescript": "4.9.5" }, "dependencies": { diff --git a/typescript/support/src/parseFlatbufferSchema.test.ts b/typescript/support/src/parseFlatbufferSchema.test.ts index c74b4f1a72..d7cba6298f 100644 --- a/typescript/support/src/parseFlatbufferSchema.test.ts +++ b/typescript/support/src/parseFlatbufferSchema.test.ts @@ -316,6 +316,7 @@ describe("parseFlatbufferSchema", () => { "reflection.Schema", reflectionSchemaBuffer, ); + /* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call */ const deserialized: any = deserialize(reflectionSchemaBuffer); const reflectionSchemaByteBuffer: ByteBuffer = new ByteBuffer(reflectionSchemaBuffer); const schema = Schema.getRootAsSchema(reflectionSchemaByteBuffer); @@ -330,6 +331,7 @@ describe("parseFlatbufferSchema", () => { expect(datatypes.keys()).toContain("reflection.Enum"); expect(datatypes.keys()).toContain("reflection.Object"); expect(datatypes.get("reflection.Enum")).toEqual(enumSchema); + /* eslint-enable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call */ }); it("parses non-root table schema", () => { const reflectionSchemaBuffer: Buffer = fs.readFileSync(`${__dirname}/fixtures/reflection.bfbs`); diff --git a/typescript/support/src/parseProtobufSchema.ts b/typescript/support/src/parseProtobufSchema.ts index 22b5b85c9d..8726158f08 100644 --- a/typescript/support/src/parseProtobufSchema.ts +++ b/typescript/support/src/parseProtobufSchema.ts @@ -25,9 +25,7 @@ export function parseProtobufSchema( // {sec: number, nsec: number}, compatible with the rest of Studio. The standard Protobuf types // use different names (`seconds` and `nanos`), and `seconds` is an `int64`, which would be // deserialized as a bigint by default. - const fixTimeType = ( - type: protobufjs.ReflectionObject | null /* eslint-disable-line no-restricted-syntax */, - ) => { + const fixTimeType = (type: protobufjs.ReflectionObject | null) => { if (!type || !(type instanceof protobufjs.Type)) { return; } diff --git a/yarn.lock b/yarn.lock index f44f0e36d8..00f89280a5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2862,8 +2862,9 @@ __metadata: "@foxglove/wasm-bz2": 0.1.1 "@foxglove/wasm-lz4": 1.0.2 "@foxglove/wasm-zstd": 1.0.1 - "@mcap/core": "*" - "@mcap/support": "*" + "@mcap/core": "workspace:*" + "@mcap/nodejs": "workspace:*" + "@mcap/support": "workspace:*" "@types/lodash": 4.14.191 "@types/node": 18.13.0 "@typescript-eslint/eslint-plugin": 5.52.0 @@ -2889,7 +2890,8 @@ __metadata: resolution: "@foxglove/mcap-example-basicwriter@workspace:typescript/examples/basicwriter" dependencies: "@foxglove/eslint-plugin": 0.21.0 - "@mcap/core": "*" + "@mcap/core": "workspace:*" + "@mcap/nodejs": "workspace:*" "@mcap/support": "*" "@types/node": 18.13.0 eslint: 8.34.0 @@ -2910,8 +2912,8 @@ __metadata: dependencies: "@foxglove/eslint-plugin": 0.21.0 "@foxglove/schemas": ^1.0.0 - "@mcap/core": "*" - "@mcap/support": "*" + "@mcap/core": "workspace:*" + "@mcap/nodejs": "workspace:*" "@types/node": 18.13.0 eslint: 8.34.0 eslint-config-prettier: 8.6.0 @@ -2939,8 +2941,9 @@ __metadata: "@foxglove/wasm-bz2": 0.1.1 "@foxglove/wasm-lz4": 1.0.2 "@foxglove/wasm-zstd": 1.0.1 - "@mcap/core": "*" - "@mcap/support": "*" + "@mcap/core": "workspace:*" + "@mcap/nodejs": "workspace:*" + "@mcap/support": "workspace:*" "@types/lodash": 4.14.191 "@types/node": 18.13.0 "@typescript-eslint/eslint-plugin": 5.52.0 @@ -3084,7 +3087,8 @@ __metadata: dependencies: "@foxglove/eslint-plugin": 0.21.0 "@foxglove/schemas": 1.3.0 - "@mcap/core": "*" + "@mcap/core": "workspace:*" + "@mcap/nodejs": "workspace:*" "@types/node": 18.13.0 eslint: 8.34.0 eslint-config-prettier: 8.6.0 @@ -3542,6 +3546,32 @@ __metadata: languageName: unknown linkType: soft +"@mcap/nodejs@workspace:*, @mcap/nodejs@workspace:typescript/nodejs": + version: 0.0.0-use.local + resolution: "@mcap/nodejs@workspace:typescript/nodejs" + dependencies: + "@foxglove/eslint-plugin": 0.21.0 + "@foxglove/tsconfig": 1.1.0 + "@mcap/core": "workspace:*" + "@types/jest": 29.4.0 + "@types/node": 18.13.0 + "@typescript-eslint/eslint-plugin": 5.52.0 + "@typescript-eslint/parser": 5.52.0 + eslint: 8.34.0 + eslint-config-prettier: 8.6.0 + eslint-import-resolver-typescript: 3.5.5 + eslint-plugin-es: 4.1.0 + eslint-plugin-filenames: 1.3.2 + eslint-plugin-import: 2.27.5 + eslint-plugin-jest: 27.2.1 + eslint-plugin-prettier: 4.2.1 + jest: 29.4.3 + prettier: 2.8.4 + ts-jest: 29.0.5 + typescript: 4.9.5 + languageName: unknown + linkType: soft + "@mcap/support@*, @mcap/support@workspace:*, @mcap/support@workspace:typescript/support": version: 0.0.0-use.local resolution: "@mcap/support@workspace:typescript/support" @@ -3562,7 +3592,6 @@ __metadata: "@types/node": 18.13.0 "@typescript-eslint/eslint-plugin": 5.52.0 "@typescript-eslint/parser": 5.52.0 - cspell: ^6.26.3 eslint: 8.34.0 eslint-config-prettier: 8.6.0 eslint-plugin-es: 4.1.0 @@ -3576,7 +3605,6 @@ __metadata: prettier: 2.8.4 protobufjs: ^7.2.2 ts-jest: 29.0.5 - typedoc: ^0.23.25 typescript: 4.9.5 languageName: unknown linkType: soft @@ -3671,6 +3699,20 @@ __metadata: languageName: node linkType: hard +"@pkgr/utils@npm:^2.3.1": + version: 2.4.1 + resolution: "@pkgr/utils@npm:2.4.1" + dependencies: + cross-spawn: ^7.0.3 + fast-glob: ^3.2.12 + is-glob: ^4.0.3 + open: ^9.1.0 + picocolors: ^1.0.0 + tslib: ^2.5.0 + checksum: 654682860272541a40485b01e0763b155ec31faeba85b2c51e38b59c4ff1f8918c37b87b5ecbda3ff482d8486eba086e92b991fe4a8ed62efbbbdf83c0f64409 + languageName: node + linkType: hard + "@polka/url@npm:^1.0.0-next.20": version: 1.0.0-next.21 resolution: "@polka/url@npm:1.0.0-next.21" @@ -5525,6 +5567,13 @@ __metadata: languageName: node linkType: hard +"big-integer@npm:^1.6.44": + version: 1.6.51 + resolution: "big-integer@npm:1.6.51" + checksum: 3d444173d1b2e20747e2c175568bedeebd8315b0637ea95d75fd27830d3b8e8ba36c6af40374f36bdaea7b5de376dcada1b07587cb2a79a928fccdb6e6e3c518 + languageName: node + linkType: hard + "big.js@npm:^5.2.2": version: 5.2.2 resolution: "big.js@npm:5.2.2" @@ -5610,6 +5659,15 @@ __metadata: languageName: node linkType: hard +"bplist-parser@npm:^0.2.0": + version: 0.2.0 + resolution: "bplist-parser@npm:0.2.0" + dependencies: + big-integer: ^1.6.44 + checksum: d5339dd16afc51de6c88f88f58a45b72ed6a06aa31f5557d09877575f220b7c1d3fbe375da0b62e6a10d4b8ed80523567e351f24014f5bc886ad523758142cdd + languageName: node + linkType: hard + "brace-expansion@npm:^1.1.7": version: 1.1.11 resolution: "brace-expansion@npm:1.1.11" @@ -5687,6 +5745,15 @@ __metadata: languageName: node linkType: hard +"bundle-name@npm:^3.0.0": + version: 3.0.0 + resolution: "bundle-name@npm:3.0.0" + dependencies: + run-applescript: ^5.0.0 + checksum: edf2b1fbe6096ed32e7566947ace2ea937ee427391744d7510a2880c4b9a5b3543d3f6c551236a29e5c87d3195f8e2912516290e638c15bcbede7b37cc375615 + languageName: node + linkType: hard + "bytes@npm:3.0.0": version: 3.0.0 resolution: "bytes@npm:3.0.0" @@ -6852,6 +6919,28 @@ __metadata: languageName: node linkType: hard +"default-browser-id@npm:^3.0.0": + version: 3.0.0 + resolution: "default-browser-id@npm:3.0.0" + dependencies: + bplist-parser: ^0.2.0 + untildify: ^4.0.0 + checksum: 279c7ad492542e5556336b6c254a4eaf31b2c63a5433265655ae6e47301197b6cfb15c595a6fdc6463b2ff8e1a1a1ed3cba56038a60e1527ba4ab1628c6b9941 + languageName: node + linkType: hard + +"default-browser@npm:^4.0.0": + version: 4.0.0 + resolution: "default-browser@npm:4.0.0" + dependencies: + bundle-name: ^3.0.0 + default-browser-id: ^3.0.0 + execa: ^7.1.1 + titleize: ^3.0.0 + checksum: 40c5af984799042b140300be5639c9742599bda76dc9eba5ac9ad5943c83dd36cebc4471eafcfddf8e0ec817166d5ba89d56f08e66a126c7c7908a179cead1a7 + languageName: node + linkType: hard + "default-gateway@npm:^6.0.3": version: 6.0.3 resolution: "default-gateway@npm:6.0.3" @@ -6875,6 +6964,13 @@ __metadata: languageName: node linkType: hard +"define-lazy-prop@npm:^3.0.0": + version: 3.0.0 + resolution: "define-lazy-prop@npm:3.0.0" + checksum: 54884f94caac0791bf6395a3ec530ce901cf71c47b0196b8754f3fd17edb6c0e80149c1214429d851873bb0d689dbe08dcedbb2306dc45c8534a5934723851b6 + languageName: node + linkType: hard + "define-properties@npm:^1.1.3, define-properties@npm:^1.1.4": version: 1.2.0 resolution: "define-properties@npm:1.2.0" @@ -7234,6 +7330,16 @@ __metadata: languageName: node linkType: hard +"enhanced-resolve@npm:^5.12.0": + version: 5.15.0 + resolution: "enhanced-resolve@npm:5.15.0" + dependencies: + graceful-fs: ^4.2.4 + tapable: ^2.2.0 + checksum: fbd8cdc9263be71cc737aa8a7d6c57b43d6aa38f6cc75dde6fcd3598a130cc465f979d2f4d01bb3bf475acb43817749c79f8eef9be048683602ca91ab52e4f11 + languageName: node + linkType: hard + "enhanced-resolve@npm:^5.14.0": version: 5.14.0 resolution: "enhanced-resolve@npm:5.14.0" @@ -7424,6 +7530,25 @@ __metadata: languageName: node linkType: hard +"eslint-import-resolver-typescript@npm:3.5.5": + version: 3.5.5 + resolution: "eslint-import-resolver-typescript@npm:3.5.5" + dependencies: + debug: ^4.3.4 + enhanced-resolve: ^5.12.0 + eslint-module-utils: ^2.7.4 + get-tsconfig: ^4.5.0 + globby: ^13.1.3 + is-core-module: ^2.11.0 + is-glob: ^4.0.3 + synckit: ^0.8.5 + peerDependencies: + eslint: "*" + eslint-plugin-import: "*" + checksum: 27e6276fdff5d377c9036362ff736ac29852106e883ff589ea9092dc57d4bc2a67a82d75134221124f05045f9a7e2114a159b2c827d1f9f64d091f7afeab0f58 + languageName: node + linkType: hard + "eslint-module-utils@npm:^2.7.4": version: 2.7.4 resolution: "eslint-module-utils@npm:2.7.4" @@ -7834,6 +7959,23 @@ __metadata: languageName: node linkType: hard +"execa@npm:^7.1.1": + version: 7.1.1 + resolution: "execa@npm:7.1.1" + dependencies: + cross-spawn: ^7.0.3 + get-stream: ^6.0.1 + human-signals: ^4.3.0 + is-stream: ^3.0.0 + merge-stream: ^2.0.0 + npm-run-path: ^5.1.0 + onetime: ^6.0.0 + signal-exit: ^3.0.7 + strip-final-newline: ^3.0.0 + checksum: 21fa46fc69314ace4068cf820142bdde5b643a5d89831c2c9349479c1555bff137a291b8e749e7efca36535e4e0a8c772c11008ca2e84d2cbd6ca141a3c8f937 + languageName: node + linkType: hard + "exit@npm:^0.1.2": version: 0.1.2 resolution: "exit@npm:0.1.2" @@ -8417,7 +8559,7 @@ __metadata: languageName: node linkType: hard -"get-stream@npm:^6.0.0": +"get-stream@npm:^6.0.0, get-stream@npm:^6.0.1": version: 6.0.1 resolution: "get-stream@npm:6.0.1" checksum: e04ecece32c92eebf5b8c940f51468cd53554dcbb0ea725b2748be583c9523d00128137966afce410b9b051eb2ef16d657cd2b120ca8edafcf5a65e81af63cad @@ -8434,6 +8576,15 @@ __metadata: languageName: node linkType: hard +"get-tsconfig@npm:^4.5.0": + version: 4.6.2 + resolution: "get-tsconfig@npm:4.6.2" + dependencies: + resolve-pkg-maps: ^1.0.0 + checksum: e791e671a9b55e91efea3ca819ecd7a25beae679e31c83234bf3dd62ddd93df070c1b95ae7e29d206358ebb6408f6f79ac6d83a32a3bbd6a6d217babe23de077 + languageName: node + linkType: hard + "github-slugger@npm:^1.4.0": version: 1.5.0 resolution: "github-slugger@npm:1.5.0" @@ -8583,6 +8734,19 @@ __metadata: languageName: node linkType: hard +"globby@npm:^13.1.3": + version: 13.2.0 + resolution: "globby@npm:13.2.0" + dependencies: + dir-glob: ^3.0.1 + fast-glob: ^3.2.11 + ignore: ^5.2.0 + merge2: ^1.4.1 + slash: ^4.0.0 + checksum: 0a3dd786571788adef1c894f22112834cff5bbe061ae6e0a01c5118c39d44b3f1937ef1dae3f8b9bc24756eba84a0923e565b1ad9a4ec52831d7e2a04c035e75 + languageName: node + linkType: hard + "gopd@npm:^1.0.1": version: 1.0.1 resolution: "gopd@npm:1.0.1" @@ -9068,6 +9232,13 @@ __metadata: languageName: node linkType: hard +"human-signals@npm:^4.3.0": + version: 4.3.1 + resolution: "human-signals@npm:4.3.1" + checksum: 6f12958df3f21b6fdaf02d90896c271df00636a31e2bbea05bddf817a35c66b38a6fdac5863e2df85bd52f34958997f1f50350ff97249e1dff8452865d5235d1 + languageName: node + linkType: hard + "humanize-ms@npm:^1.2.1": version: 1.2.1 resolution: "humanize-ms@npm:1.2.1" @@ -9415,6 +9586,15 @@ __metadata: languageName: node linkType: hard +"is-docker@npm:^3.0.0": + version: 3.0.0 + resolution: "is-docker@npm:3.0.0" + bin: + is-docker: cli.js + checksum: b698118f04feb7eaf3338922bd79cba064ea54a1c3db6ec8c0c8d8ee7613e7e5854d802d3ef646812a8a3ace81182a085dfa0a71cc68b06f3fa794b9783b3c90 + languageName: node + linkType: hard + "is-extendable@npm:^0.1.0": version: 0.1.1 resolution: "is-extendable@npm:0.1.1" @@ -9459,6 +9639,17 @@ __metadata: languageName: node linkType: hard +"is-inside-container@npm:^1.0.0": + version: 1.0.0 + resolution: "is-inside-container@npm:1.0.0" + dependencies: + is-docker: ^3.0.0 + bin: + is-inside-container: cli.js + checksum: c50b75a2ab66ab3e8b92b3bc534e1ea72ca25766832c0623ac22d134116a98bcf012197d1caabe1d1c4bd5f84363d4aa5c36bb4b585fbcaf57be172cd10a1a03 + languageName: node + linkType: hard + "is-installed-globally@npm:^0.4.0": version: 0.4.0 resolution: "is-installed-globally@npm:0.4.0" @@ -9597,6 +9788,13 @@ __metadata: languageName: node linkType: hard +"is-stream@npm:^3.0.0": + version: 3.0.0 + resolution: "is-stream@npm:3.0.0" + checksum: 172093fe99119ffd07611ab6d1bcccfe8bc4aa80d864b15f43e63e54b7abc71e779acd69afdb854c4e2a67fdc16ae710e370eda40088d1cfc956a50ed82d8f16 + languageName: node + linkType: hard + "is-string@npm:^1.0.5, is-string@npm:^1.0.7": version: 1.0.7 resolution: "is-string@npm:1.0.7" @@ -10948,6 +11146,13 @@ __metadata: languageName: node linkType: hard +"mimic-fn@npm:^4.0.0": + version: 4.0.0 + resolution: "mimic-fn@npm:4.0.0" + checksum: 995dcece15ee29aa16e188de6633d43a3db4611bcf93620e7e62109ec41c79c0f34277165b8ce5e361205049766e371851264c21ac64ca35499acb5421c2ba56 + languageName: node + linkType: hard + "mimic-response@npm:^1.0.0, mimic-response@npm:^1.0.1": version: 1.0.1 resolution: "mimic-response@npm:1.0.1" @@ -11306,6 +11511,15 @@ __metadata: languageName: node linkType: hard +"npm-run-path@npm:^5.1.0": + version: 5.1.0 + resolution: "npm-run-path@npm:5.1.0" + dependencies: + path-key: ^4.0.0 + checksum: dc184eb5ec239d6a2b990b43236845332ef12f4e0beaa9701de724aa797fe40b6bbd0157fb7639d24d3ab13f5d5cf22d223a19c6300846b8126f335f788bee66 + languageName: node + linkType: hard + "npmlog@npm:^6.0.0": version: 6.0.2 resolution: "npmlog@npm:6.0.2" @@ -11451,6 +11665,15 @@ __metadata: languageName: node linkType: hard +"onetime@npm:^6.0.0": + version: 6.0.0 + resolution: "onetime@npm:6.0.0" + dependencies: + mimic-fn: ^4.0.0 + checksum: 0846ce78e440841335d4e9182ef69d5762e9f38aa7499b19f42ea1c4cd40f0b4446094c455c713f9adac3f4ae86f613bb5e30c99e52652764d06a89f709b3788 + languageName: node + linkType: hard + "open@npm:^8.0.9, open@npm:^8.4.0": version: 8.4.2 resolution: "open@npm:8.4.2" @@ -11462,6 +11685,18 @@ __metadata: languageName: node linkType: hard +"open@npm:^9.1.0": + version: 9.1.0 + resolution: "open@npm:9.1.0" + dependencies: + default-browser: ^4.0.0 + define-lazy-prop: ^3.0.0 + is-inside-container: ^1.0.0 + is-wsl: ^2.2.0 + checksum: 3993c0f61d51fed8ac290e99c9c3cf45d3b6cfb3e2aa2b74cafd312c3486c22fd81df16ac8f3ab91dd8a4e3e729a16fc2480cfc406c4833416cf908acf1ae7c9 + languageName: node + linkType: hard + "opener@npm:^1.5.2": version: 1.5.2 resolution: "opener@npm:1.5.2" @@ -11721,6 +11956,13 @@ __metadata: languageName: node linkType: hard +"path-key@npm:^4.0.0": + version: 4.0.0 + resolution: "path-key@npm:4.0.0" + checksum: 8e6c314ae6d16b83e93032c61020129f6f4484590a777eed709c4a01b50e498822b00f76ceaf94bc64dbd90b327df56ceadce27da3d83393790f1219e07721d7 + languageName: node + linkType: hard + "path-parse@npm:^1.0.7": version: 1.0.7 resolution: "path-parse@npm:1.0.7" @@ -13115,6 +13357,13 @@ __metadata: languageName: node linkType: hard +"resolve-pkg-maps@npm:^1.0.0": + version: 1.0.0 + resolution: "resolve-pkg-maps@npm:1.0.0" + checksum: 1012afc566b3fdb190a6309cc37ef3b2dcc35dff5fa6683a9d00cd25c3247edfbc4691b91078c97adc82a29b77a2660c30d791d65dab4fc78bfc473f60289977 + languageName: node + linkType: hard + "resolve.exports@npm:^2.0.0": version: 2.0.0 resolution: "resolve.exports@npm:2.0.0" @@ -13230,7 +13479,10 @@ __metadata: resolution: "root-workspace-0b6124@workspace:." dependencies: cspell: ^6.26.3 + jest: 29.4.3 prettier: ^2.8.4 + ts-jest: 29.0.5 + ts-node: 10.9.1 typedoc: ^0.23.25 languageName: unknown linkType: soft @@ -13256,6 +13508,15 @@ __metadata: languageName: node linkType: hard +"run-applescript@npm:^5.0.0": + version: 5.0.0 + resolution: "run-applescript@npm:5.0.0" + dependencies: + execa: ^5.0.0 + checksum: d00c2dbfa5b2d774de7451194b8b125f40f65fc183de7d9dcae97f57f59433586d3c39b9001e111c38bfa24c3436c99df1bb4066a2a0c90d39a8c4cd6889af77 + languageName: node + linkType: hard + "run-parallel@npm:^1.1.9": version: 1.2.0 resolution: "run-parallel@npm:1.2.0" @@ -13998,6 +14259,13 @@ __metadata: languageName: node linkType: hard +"strip-final-newline@npm:^3.0.0": + version: 3.0.0 + resolution: "strip-final-newline@npm:3.0.0" + checksum: 23ee263adfa2070cd0f23d1ac14e2ed2f000c9b44229aec9c799f1367ec001478469560abefd00c5c99ee6f0b31c137d53ec6029c53e9f32a93804e18c201050 + languageName: node + linkType: hard + "strip-json-comments@npm:^3.1.0, strip-json-comments@npm:^3.1.1": version: 3.1.1 resolution: "strip-json-comments@npm:3.1.1" @@ -14091,6 +14359,16 @@ __metadata: languageName: node linkType: hard +"synckit@npm:^0.8.5": + version: 0.8.5 + resolution: "synckit@npm:0.8.5" + dependencies: + "@pkgr/utils": ^2.3.1 + tslib: ^2.5.0 + checksum: 8a9560e5d8f3d94dc3cf5f7b9c83490ffa30d320093560a37b88f59483040771fd1750e76b9939abfbb1b5a23fd6dfbae77f6b338abffe7cae7329cd9b9bb86b + languageName: node + linkType: hard + "tapable@npm:^1.0.0": version: 1.1.3 resolution: "tapable@npm:1.1.3" @@ -14194,6 +14472,13 @@ __metadata: languageName: node linkType: hard +"titleize@npm:^3.0.0": + version: 3.0.0 + resolution: "titleize@npm:3.0.0" + checksum: 71fbbeabbfb36ccd840559f67f21e356e1d03da2915b32d2ae1a60ddcc13a124be2739f696d2feb884983441d159a18649e8d956648d591bdad35c430a6b6d28 + languageName: node + linkType: hard + "tmpl@npm:1.0.5": version: 1.0.5 resolution: "tmpl@npm:1.0.5" @@ -14693,6 +14978,13 @@ __metadata: languageName: node linkType: hard +"untildify@npm:^4.0.0": + version: 4.0.0 + resolution: "untildify@npm:4.0.0" + checksum: 39ced9c418a74f73f0a56e1ba4634b4d959422dff61f4c72a8e39f60b99380c1b45ed776fbaa0a4101b157e4310d873ad7d114e8534ca02609b4916bb4187fb9 + languageName: node + linkType: hard + "update-browserslist-db@npm:^1.0.10": version: 1.0.10 resolution: "update-browserslist-db@npm:1.0.10" From 40f8d27b850689d4eb939c1e2423fd32371d0960 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 30 Jun 2023 18:11:18 -0700 Subject: [PATCH 05/27] typedoc updates --- package.json | 6 +++--- python/mcap-protobuf-support/README.md | 2 +- python/mcap-ros1-support/README.md | 2 +- python/mcap-ros2-support/README.md | 2 +- typescript/README.md | 7 ++++--- typescript/core/README.md | 4 ++-- typescript/nodejs/README.md | 4 ++-- typescript/support/README.md | 4 ++-- typescript/typedoc.json | 2 +- 9 files changed, 17 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 7778ee4f99..902fe5009d 100644 --- a/package.json +++ b/package.json @@ -15,12 +15,12 @@ "prettier": "prettier --write .", "prettier:check": "prettier --check .", "docs:swift:start": "swift package --disable-sandbox preview-documentation --target MCAP", - "typedoc": "typedoc --out __docs__/typescript --options typescript/typedoc.json", + "typedoc": "yarn typescript:build && typedoc --out __docs__/typescript --options typescript/typedoc.json", "start": "yarn workspace website start", "spellcheck": "cspell --relative '**'", "typescript:test": "yarn jest --config typescript/jest.config.json", - "typescript:build": "yarn workspace @mcap/core build && yarn workspace @mcap/support build", - "typescript:clean": "yarn workspace @mcap/core build --clean && yarn workspace @mcap/support build --clean", + "typescript:build": "yarn workspace @mcap/core build && yarn workspace @mcap/support build && yarn workspace @mcap/nodejs build", + "typescript:clean": "yarn workspace @mcap/core build --clean && yarn workspace @mcap/support build --clean && yarn workspace @mcap/nodejs build --clean", "test:conformance:generate-inputs": "yarn workspace @foxglove/mcap-conformance generate-inputs --data-dir \"$(pwd)/tests/conformance/data\"", "test:conformance": "yarn workspace @foxglove/mcap-conformance run-tests --data-dir \"$(pwd)/tests/conformance/data\"" }, diff --git a/python/mcap-protobuf-support/README.md b/python/mcap-protobuf-support/README.md index 31698fa70b..9d89c0fa8b 100644 --- a/python/mcap-protobuf-support/README.md +++ b/python/mcap-protobuf-support/README.md @@ -35,4 +35,4 @@ pipenv run python point_cloud_example.py output.mcap ## Stay in touch -Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions, share feedback, and stay up to date on what our team is working on. +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/python/mcap-ros1-support/README.md b/python/mcap-ros1-support/README.md index 6b20603152..48088f23e5 100644 --- a/python/mcap-ros1-support/README.md +++ b/python/mcap-ros1-support/README.md @@ -36,5 +36,5 @@ ros_writer.finish() ## Stay in touch -Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions, +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/python/mcap-ros2-support/README.md b/python/mcap-ros2-support/README.md index 2123ddd7fd..28d11a2d25 100644 --- a/python/mcap-ros2-support/README.md +++ b/python/mcap-ros2-support/README.md @@ -22,5 +22,5 @@ for msg in read_ros2_messages("my_data.mcap"): ## Stay in touch -Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions, +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/typescript/README.md b/typescript/README.md index b90b762053..8ed50bddd2 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -1,8 +1,9 @@ # TypeScript libraries for MCAP -[MCAP](https://github.com/foxglove/mcap) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. +[MCAP](https://mcap.dev/) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. The following NPM packages are provided for use with JavaScript and TypeScript: -- **@mcap/core** – low-level readers and writers -- **@mcap/support** – support for well-known compression formats and encodings, and Node.js and browser environments +- [**@mcap/core**](./core) – low-level readers and writers +- [**@mcap/support**](./support) – support for well-known compression formats and encodings +- [**@mcap/nodejs**](./nodejs) – support for Node.js environment diff --git a/typescript/core/README.md b/typescript/core/README.md index 852080048c..2e007000cd 100644 --- a/typescript/core/README.md +++ b/typescript/core/README.md @@ -1,6 +1,6 @@ # @mcap/core -[MCAP](https://github.com/foxglove/mcap) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. +[MCAP](https://mcap.dev/) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. The `@mcap/core` package provides low-level readers and writers for the MCAP format in TypeScript. @@ -14,4 +14,4 @@ Examples of how to use the `@mcap/core` APIs can be found in the [TypeScript exa ## Stay in touch -Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions, share feedback, and stay up to date on what our team is working on. +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/typescript/nodejs/README.md b/typescript/nodejs/README.md index 391fc68de8..b175e47638 100644 --- a/typescript/nodejs/README.md +++ b/typescript/nodejs/README.md @@ -1,6 +1,6 @@ # @mcap/nodejs -[MCAP](https://github.com/foxglove/mcap) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. +[MCAP](https://mcap.dev/) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. The `@mcap/nodejs` package provides utilities for working with MCAP files from Node.js. @@ -47,4 +47,4 @@ const writer = new McapWriter({ ## Stay in touch -Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions, share feedback, and stay up to date on what our team is working on. +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/typescript/support/README.md b/typescript/support/README.md index 7524dcbc24..e598f235fc 100644 --- a/typescript/support/README.md +++ b/typescript/support/README.md @@ -1,6 +1,6 @@ # @mcap/support -[MCAP](https://github.com/foxglove/mcap) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. +[MCAP](https://mcap.dev/) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. The `@mcap/support` package provides utilities for working with MCAP files that use [well-known compression formats and encodings](https://mcap.dev/specification/appendix.html), from Node.js and browsers. @@ -50,4 +50,4 @@ const writer = new McapWriter({ ## Stay in touch -Join our [Slack channel](https://foxglove.dev/join-slack) to ask questions, share feedback, and stay up to date on what our team is working on. +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/typescript/typedoc.json b/typescript/typedoc.json index 6969e3d175..0cc0fe0eef 100644 --- a/typescript/typedoc.json +++ b/typescript/typedoc.json @@ -1,5 +1,5 @@ { "name": "MCAP TypeScript SDK", "entryPointStrategy": "packages", - "entryPoints": ["./core", "./support"] + "entryPoints": ["./core", "./support", "./nodejs"] } From 4ee79d31133d4cd0b16b8143f2f9525104afa3da Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 30 Jun 2023 19:49:54 -0700 Subject: [PATCH 06/27] fix cspell --- typescript/support/src/parseProtobufSchema.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/typescript/support/src/parseProtobufSchema.test.ts b/typescript/support/src/parseProtobufSchema.test.ts index 7949ac279e..97296c7d31 100644 --- a/typescript/support/src/parseProtobufSchema.test.ts +++ b/typescript/support/src/parseProtobufSchema.test.ts @@ -20,6 +20,7 @@ describe("parseProtobufSchema", () => { const channel = parseProtobufSchema( "ExampleMessage", Buffer.from( + // cspell:disable-next-line "0A8D010A156578616D706C655F6D6573736167652E70726F746F222C0A0E4578616D706C654D657373616765121A0A046461746118012003280E320C2E4578616D706C65456E756D2A3E0A0B4578616D706C65456E756D120B0A07554E4B4E4F574E1000120C0A085748415445564552100012070A03464F4F100112070A0342415210021A021001620670726F746F33", "hex", ), @@ -61,6 +62,7 @@ describe("parseProtobufSchema", () => { const channel = parseProtobufSchema( "Int64Test", Buffer.from( + // cspell:disable-next-line "CvILCg9JbnQ2NFRlc3QucHJvdG8igwYKCUludDY0VGVzdBIUCgVpbnQ2NBgBIAEoA1IFaW50NjQSFgoGdWludDY0GAIgASgEUgZ1aW50NjQSFgoGc2ludDY0GAMgASgSUgZzaW50NjQSGAoHZml4ZWQ2NBgEIAEoBlIHZml4ZWQ2NBIaCghzZml4ZWQ2NBgFIAEoEFIIc2ZpeGVkNjQSNAoIaW50NjRtYXAYBiADKAsyGC5JbnQ2NFRlc3QuSW50NjRtYXBFbnRyeVIIaW50NjRtYXASNwoJdWludDY0bWFwGAcgAygLMhkuSW50NjRUZXN0LlVpbnQ2NG1hcEVudHJ5Ugl1aW50NjRtYXASNwoJc2ludDY0bWFwGAggAygLMhkuSW50NjRUZXN0LlNpbnQ2NG1hcEVudHJ5UglzaW50NjRtYXASOgoKZml4ZWQ2NG1hcBgJIAMoCzIaLkludDY0VGVzdC5GaXhlZDY0bWFwRW50cnlSCmZpeGVkNjRtYXASPQoLc2ZpeGVkNjRtYXAYCiADKAsyGy5JbnQ2NFRlc3QuU2ZpeGVkNjRtYXBFbnRyeVILc2ZpeGVkNjRtYXASHwoGbmVzdGVkGAsgAygLMgcuTmVzdGVkUgZuZXN0ZWQaOwoNSW50NjRtYXBFbnRyeRIQCgNrZXkYASABKANSA2tleRIUCgV2YWx1ZRgCIAEoA1IFdmFsdWU6AjgBGjwKDlVpbnQ2NG1hcEVudHJ5EhAKA2tleRgBIAEoBFIDa2V5EhQKBXZhbHVlGAIgASgEUgV2YWx1ZToCOAEaPAoOU2ludDY0bWFwRW50cnkSEAoDa2V5GAEgASgSUgNrZXkSFAoFdmFsdWUYAiABKBJSBXZhbHVlOgI4ARo9Cg9GaXhlZDY0bWFwRW50cnkSEAoDa2V5GAEgASgGUgNrZXkSFAoFdmFsdWUYAiABKAZSBXZhbHVlOgI4ARo+ChBTZml4ZWQ2NG1hcEVudHJ5EhAKA2tleRgBIAEoEFIDa2V5EhQKBXZhbHVlGAIgASgQUgV2YWx1ZToCOAEi0AUKBk5lc3RlZBIUCgVpbnQ2NBgBIAEoA1IFaW50NjQSFgoGdWludDY0GAIgASgEUgZ1aW50NjQSFgoGc2ludDY0GAMgASgSUgZzaW50NjQSGAoHZml4ZWQ2NBgEIAEoBlIHZml4ZWQ2NBIaCghzZml4ZWQ2NBgFIAEoEFIIc2ZpeGVkNjQSMQoIaW50NjRtYXAYBiADKAsyFS5OZXN0ZWQuSW50NjRtYXBFbnRyeVIIaW50NjRtYXASNAoJdWludDY0bWFwGAcgAygLMhYuTmVzdGVkLlVpbnQ2NG1hcEVudHJ5Ugl1aW50NjRtYXASNAoJc2ludDY0bWFwGAggAygLMhYuTmVzdGVkLlNpbnQ2NG1hcEVudHJ5UglzaW50NjRtYXASNwoKZml4ZWQ2NG1hcBgJIAMoCzIXLk5lc3RlZC5GaXhlZDY0bWFwRW50cnlSCmZpeGVkNjRtYXASOgoLc2ZpeGVkNjRtYXAYCiADKAsyGC5OZXN0ZWQuU2ZpeGVkNjRtYXBFbnRyeVILc2ZpeGVkNjRtYXAaOwoNSW50NjRtYXBFbnRyeRIQCgNrZXkYASABKANSA2tleRIUCgV2YWx1ZRgCIAEoA1IFdmFsdWU6AjgBGjwKDlVpbnQ2NG1hcEVudHJ5EhAKA2tleRgBIAEoBFIDa2V5EhQKBXZhbHVlGAIgASgEUgV2YWx1ZToCOAEaPAoOU2ludDY0bWFwRW50cnkSEAoDa2V5GAEgASgSUgNrZXkSFAoFdmFsdWUYAiABKBJSBXZhbHVlOgI4ARo9Cg9GaXhlZDY0bWFwRW50cnkSEAoDa2V5GAEgASgGUgNrZXkSFAoFdmFsdWUYAiABKAZSBXZhbHVlOgI4ARo+ChBTZml4ZWQ2NG1hcEVudHJ5EhAKA2tleRgBIAEoEFIDa2V5EhQKBXZhbHVlGAIgASgQUgV2YWx1ZToCOAFiBnByb3RvMw==", "base64", ), @@ -68,6 +70,7 @@ describe("parseProtobufSchema", () => { expect( channel.deserialize( Buffer.from( + // cspell:disable-next-line "088c95f0c4c5a9d28ff20110bc85a0cfc8e0c8e38a0118e7d59ff6f4acdbe01b21bc02e8890423c78a298c0a9c584c491ff23216088c95f0c4c5a9d28ff201108c95f0c4c5a9d28ff2013a1608bc85a0cfc8e0c8e38a0110bc85a0cfc8e0c8e38a01421408e7d59ff6f4acdbe01b10e7d59ff6f4acdbe01b4a1209bc02e8890423c78a11bc02e8890423c78a5212098c0a9c584c491ff2118c0a9c584c491ff25aa001088c95f0c4c5a9d28ff20110bc85a0cfc8e0c8e38a0118e7d59ff6f4acdbe01b21bc02e8890423c78a298c0a9c584c491ff23216088c95f0c4c5a9d28ff201108c95f0c4c5a9d28ff2013a1608bc85a0cfc8e0c8e38a0110bc85a0cfc8e0c8e38a01421408e7d59ff6f4acdbe01b10e7d59ff6f4acdbe01b4a1209bc02e8890423c78a11bc02e8890423c78a5212098c0a9c584c491ff2118c0a9c584c491ff2", "hex", ), @@ -104,6 +107,7 @@ describe("parseProtobufSchema", () => { const poseInFrameChannel = parseProtobufSchema( "foxglove.PoseInFrame", Buffer.from( + // cspell:disable-next-line "CmcKGWZveGdsb3ZlL1F1YXRlcm5pb24ucHJvdG8SCGZveGdsb3ZlIjgKClF1YXRlcm5pb24SCQoBeBgBIAEoARIJCgF5GAIgASgBEgkKAXoYAyABKAESCQoBdxgEIAEoAWIGcHJvdG8zClYKFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8SCGZveGdsb3ZlIioKB1ZlY3RvcjMSCQoBeBgBIAEoARIJCgF5GAIgASgBEgkKAXoYAyABKAFiBnByb3RvMwqyAQoTZm94Z2xvdmUvUG9zZS5wcm90bxIIZm94Z2xvdmUaGWZveGdsb3ZlL1F1YXRlcm5pb24ucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8iVgoEUG9zZRIjCghwb3NpdGlvbhgBIAEoCzIRLmZveGdsb3ZlLlZlY3RvcjMSKQoLb3JpZW50YXRpb24YAiABKAsyFC5mb3hnbG92ZS5RdWF0ZXJuaW9uYgZwcm90bzMK7wEKH2dvb2dsZS9wcm90b2J1Zi90aW1lc3RhbXAucHJvdG8SD2dvb2dsZS5wcm90b2J1ZiIrCglUaW1lc3RhbXASDwoHc2Vjb25kcxgBIAEoAxINCgVuYW5vcxgCIAEoBUKFAQoTY29tLmdvb2dsZS5wcm90b2J1ZkIOVGltZXN0YW1wUHJvdG9QAVoyZ29vZ2xlLmdvbGFuZy5vcmcvcHJvdG9idWYvdHlwZXMva25vd24vdGltZXN0YW1wcGL4AQGiAgNHUEKqAh5Hb29nbGUuUHJvdG9idWYuV2VsbEtub3duVHlwZXNiBnByb3RvMwrSAQoaZm94Z2xvdmUvUG9zZUluRnJhbWUucHJvdG8SCGZveGdsb3ZlGhNmb3hnbG92ZS9Qb3NlLnByb3RvGh9nb29nbGUvcHJvdG9idWYvdGltZXN0YW1wLnByb3RvImwKC1Bvc2VJbkZyYW1lEi0KCXRpbWVzdGFtcBgBIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASEAoIZnJhbWVfaWQYAiABKAkSHAoEcG9zZRgDIAEoCzIOLmZveGdsb3ZlLlBvc2ViBnByb3RvMw==", "base64", ), @@ -111,6 +115,7 @@ describe("parseProtobufSchema", () => { const poseInFrame = poseInFrameChannel.deserialize( Buffer.from( + // cspell:disable-next-line "CgwIx8LcoQYQuJzV6wESA2ZvbxooChsJAAAAAAAA8D8RAAAAAAAAAEAZAAAAAAAACEASCSEAAAAAAADwPw==", "base64", ), @@ -141,6 +146,7 @@ describe("parseProtobufSchema", () => { expect(() => poseInFrameChannel.deserialize( Buffer.from( + // cspell:disable-next-line "CgsIgICAgICAgBAQARIDZm9vGigKGwkAAAAAAADwPxEAAAAAAAAAQBkAAAAAAAAIQBIJIQAAAAAAAPA/", "base64", ), @@ -152,12 +158,14 @@ describe("parseProtobufSchema", () => { const sceneUpdateChannel = parseProtobufSchema( "foxglove.SceneUpdate", Buffer.from( + // cspell:disable-next-line "Cl0KFGZveGdsb3ZlL0NvbG9yLnByb3RvEghmb3hnbG92ZSIzCgVDb2xvchIJCgFyGAEgASgBEgkKAWcYAiABKAESCQoBYhgDIAEoARIJCgFhGAQgASgBYgZwcm90bzMKZwoZZm94Z2xvdmUvUXVhdGVybmlvbi5wcm90bxIIZm94Z2xvdmUiOAoKUXVhdGVybmlvbhIJCgF4GAEgASgBEgkKAXkYAiABKAESCQoBehgDIAEoARIJCgF3GAQgASgBYgZwcm90bzMKVgoWZm94Z2xvdmUvVmVjdG9yMy5wcm90bxIIZm94Z2xvdmUiKgoHVmVjdG9yMxIJCgF4GAEgASgBEgkKAXkYAiABKAESCQoBehgDIAEoAWIGcHJvdG8zCrIBChNmb3hnbG92ZS9Qb3NlLnByb3RvEghmb3hnbG92ZRoZZm94Z2xvdmUvUXVhdGVybmlvbi5wcm90bxoWZm94Z2xvdmUvVmVjdG9yMy5wcm90byJWCgRQb3NlEiMKCHBvc2l0aW9uGAEgASgLMhEuZm94Z2xvdmUuVmVjdG9yMxIpCgtvcmllbnRhdGlvbhgCIAEoCzIULmZveGdsb3ZlLlF1YXRlcm5pb25iBnByb3RvMwqHAgodZm94Z2xvdmUvQXJyb3dQcmltaXRpdmUucHJvdG8SCGZveGdsb3ZlGhRmb3hnbG92ZS9Db2xvci5wcm90bxoTZm94Z2xvdmUvUG9zZS5wcm90byKoAQoOQXJyb3dQcmltaXRpdmUSHAoEcG9zZRgBIAEoCzIOLmZveGdsb3ZlLlBvc2USFAoMc2hhZnRfbGVuZ3RoGAIgASgBEhYKDnNoYWZ0X2RpYW1ldGVyGAMgASgBEhMKC2hlYWRfbGVuZ3RoGAQgASgBEhUKDWhlYWRfZGlhbWV0ZXIYBSABKAESHgoFY29sb3IYBiABKAsyDy5mb3hnbG92ZS5Db2xvcmIGcHJvdG8zCuMBChxmb3hnbG92ZS9DdWJlUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8ibgoNQ3ViZVByaW1pdGl2ZRIcCgRwb3NlGAEgASgLMg4uZm94Z2xvdmUuUG9zZRIfCgRzaXplGAIgASgLMhEuZm94Z2xvdmUuVmVjdG9yMxIeCgVjb2xvchgDIAEoCzIPLmZveGdsb3ZlLkNvbG9yYgZwcm90bzMKlQIKIGZveGdsb3ZlL0N5bGluZGVyUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8imwEKEUN5bGluZGVyUHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEh8KBHNpemUYAiABKAsyES5mb3hnbG92ZS5WZWN0b3IzEhQKDGJvdHRvbV9zY2FsZRgDIAEoARIRCgl0b3Bfc2NhbGUYBCABKAESHgoFY29sb3IYBSABKAsyDy5mb3hnbG92ZS5Db2xvcmIGcHJvdG8zClsKG2ZveGdsb3ZlL0tleVZhbHVlUGFpci5wcm90bxIIZm94Z2xvdmUiKgoMS2V5VmFsdWVQYWlyEgsKA2tleRgBIAEoCRINCgV2YWx1ZRgCIAEoCWIGcHJvdG8zClQKFWZveGdsb3ZlL1BvaW50My5wcm90bxIIZm94Z2xvdmUiKQoGUG9pbnQzEgkKAXgYASABKAESCQoBeRgCIAEoARIJCgF6GAMgASgBYgZwcm90bzMKpAMKHGZveGdsb3ZlL0xpbmVQcmltaXRpdmUucHJvdG8SCGZveGdsb3ZlGhRmb3hnbG92ZS9Db2xvci5wcm90bxoVZm94Z2xvdmUvUG9pbnQzLnByb3RvGhNmb3hnbG92ZS9Qb3NlLnByb3RvIq8CCg1MaW5lUHJpbWl0aXZlEioKBHR5cGUYASABKA4yHC5mb3hnbG92ZS5MaW5lUHJpbWl0aXZlLlR5cGUSHAoEcG9zZRgCIAEoCzIOLmZveGdsb3ZlLlBvc2USEQoJdGhpY2tuZXNzGAMgASgBEhcKD3NjYWxlX2ludmFyaWFudBgEIAEoCBIgCgZwb2ludHMYBSADKAsyEC5mb3hnbG92ZS5Qb2ludDMSHgoFY29sb3IYBiABKAsyDy5mb3hnbG92ZS5Db2xvchIfCgZjb2xvcnMYByADKAsyDy5mb3hnbG92ZS5Db2xvchIPCgdpbmRpY2VzGAggAygHIjQKBFR5cGUSDgoKTElORV9TVFJJUBAAEg0KCUxJTkVfTE9PUBABEg0KCUxJTkVfTElTVBACYgZwcm90bzMKrgIKHWZveGdsb3ZlL01vZGVsUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8itwEKDk1vZGVsUHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEiAKBXNjYWxlGAIgASgLMhEuZm94Z2xvdmUuVmVjdG9yMxIeCgVjb2xvchgDIAEoCzIPLmZveGdsb3ZlLkNvbG9yEhYKDm92ZXJyaWRlX2NvbG9yGAQgASgIEgsKA3VybBgFIAEoCRISCgptZWRpYV90eXBlGAYgASgJEgwKBGRhdGEYByABKAxiBnByb3RvMwrnAQoeZm94Z2xvdmUvU3BoZXJlUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8icAoPU3BoZXJlUHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEh8KBHNpemUYAiABKAsyES5mb3hnbG92ZS5WZWN0b3IzEh4KBWNvbG9yGAMgASgLMg8uZm94Z2xvdmUuQ29sb3JiBnByb3RvMwr4AQocZm94Z2xvdmUvVGV4dFByaW1pdGl2ZS5wcm90bxIIZm94Z2xvdmUaFGZveGdsb3ZlL0NvbG9yLnByb3RvGhNmb3hnbG92ZS9Qb3NlLnByb3RvIpoBCg1UZXh0UHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEhEKCWJpbGxib2FyZBgCIAEoCBIRCglmb250X3NpemUYAyABKAESFwoPc2NhbGVfaW52YXJpYW50GAQgASgIEh4KBWNvbG9yGAUgASgLMg8uZm94Z2xvdmUuQ29sb3ISDAoEdGV4dBgGIAEoCWIGcHJvdG8zCqYCCiRmb3hnbG92ZS9UcmlhbmdsZUxpc3RQcmltaXRpdmUucHJvdG8SCGZveGdsb3ZlGhRmb3hnbG92ZS9Db2xvci5wcm90bxoVZm94Z2xvdmUvUG9pbnQzLnByb3RvGhNmb3hnbG92ZS9Qb3NlLnByb3RvIqkBChVUcmlhbmdsZUxpc3RQcmltaXRpdmUSHAoEcG9zZRgBIAEoCzIOLmZveGdsb3ZlLlBvc2USIAoGcG9pbnRzGAIgAygLMhAuZm94Z2xvdmUuUG9pbnQzEh4KBWNvbG9yGAMgASgLMg8uZm94Z2xvdmUuQ29sb3ISHwoGY29sb3JzGAQgAygLMg8uZm94Z2xvdmUuQ29sb3ISDwoHaW5kaWNlcxgFIAMoB2IGcHJvdG8zCusBCh5nb29nbGUvcHJvdG9idWYvZHVyYXRpb24ucHJvdG8SD2dvb2dsZS5wcm90b2J1ZiIqCghEdXJhdGlvbhIPCgdzZWNvbmRzGAEgASgDEg0KBW5hbm9zGAIgASgFQoMBChNjb20uZ29vZ2xlLnByb3RvYnVmQg1EdXJhdGlvblByb3RvUAFaMWdvb2dsZS5nb2xhbmcub3JnL3Byb3RvYnVmL3R5cGVzL2tub3duL2R1cmF0aW9ucGL4AQGiAgNHUEKqAh5Hb29nbGUuUHJvdG9idWYuV2VsbEtub3duVHlwZXNiBnByb3RvMwrvAQofZ29vZ2xlL3Byb3RvYnVmL3RpbWVzdGFtcC5wcm90bxIPZ29vZ2xlLnByb3RvYnVmIisKCVRpbWVzdGFtcBIPCgdzZWNvbmRzGAEgASgDEg0KBW5hbm9zGAIgASgFQoUBChNjb20uZ29vZ2xlLnByb3RvYnVmQg5UaW1lc3RhbXBQcm90b1ABWjJnb29nbGUuZ29sYW5nLm9yZy9wcm90b2J1Zi90eXBlcy9rbm93bi90aW1lc3RhbXBwYvgBAaICA0dQQqoCHkdvb2dsZS5Qcm90b2J1Zi5XZWxsS25vd25UeXBlc2IGcHJvdG8zCrIHChpmb3hnbG92ZS9TY2VuZUVudGl0eS5wcm90bxIIZm94Z2xvdmUaHWZveGdsb3ZlL0Fycm93UHJpbWl0aXZlLnByb3RvGhxmb3hnbG92ZS9DdWJlUHJpbWl0aXZlLnByb3RvGiBmb3hnbG92ZS9DeWxpbmRlclByaW1pdGl2ZS5wcm90bxobZm94Z2xvdmUvS2V5VmFsdWVQYWlyLnByb3RvGhxmb3hnbG92ZS9MaW5lUHJpbWl0aXZlLnByb3RvGh1mb3hnbG92ZS9Nb2RlbFByaW1pdGl2ZS5wcm90bxoeZm94Z2xvdmUvU3BoZXJlUHJpbWl0aXZlLnByb3RvGhxmb3hnbG92ZS9UZXh0UHJpbWl0aXZlLnByb3RvGiRmb3hnbG92ZS9UcmlhbmdsZUxpc3RQcmltaXRpdmUucHJvdG8aHmdvb2dsZS9wcm90b2J1Zi9kdXJhdGlvbi5wcm90bxofZ29vZ2xlL3Byb3RvYnVmL3RpbWVzdGFtcC5wcm90byKjBAoLU2NlbmVFbnRpdHkSLQoJdGltZXN0YW1wGAEgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIQCghmcmFtZV9pZBgCIAEoCRIKCgJpZBgDIAEoCRIrCghsaWZldGltZRgEIAEoCzIZLmdvb2dsZS5wcm90b2J1Zi5EdXJhdGlvbhIUCgxmcmFtZV9sb2NrZWQYBSABKAgSKAoIbWV0YWRhdGEYBiADKAsyFi5mb3hnbG92ZS5LZXlWYWx1ZVBhaXISKAoGYXJyb3dzGAcgAygLMhguZm94Z2xvdmUuQXJyb3dQcmltaXRpdmUSJgoFY3ViZXMYCCADKAsyFy5mb3hnbG92ZS5DdWJlUHJpbWl0aXZlEioKB3NwaGVyZXMYCSADKAsyGS5mb3hnbG92ZS5TcGhlcmVQcmltaXRpdmUSLgoJY3lsaW5kZXJzGAogAygLMhsuZm94Z2xvdmUuQ3lsaW5kZXJQcmltaXRpdmUSJgoFbGluZXMYCyADKAsyFy5mb3hnbG92ZS5MaW5lUHJpbWl0aXZlEjIKCXRyaWFuZ2xlcxgMIAMoCzIfLmZveGdsb3ZlLlRyaWFuZ2xlTGlzdFByaW1pdGl2ZRImCgV0ZXh0cxgNIAMoCzIXLmZveGdsb3ZlLlRleHRQcmltaXRpdmUSKAoGbW9kZWxzGA4gAygLMhguZm94Z2xvdmUuTW9kZWxQcmltaXRpdmViBnByb3RvMwr+AQoiZm94Z2xvdmUvU2NlbmVFbnRpdHlEZWxldGlvbi5wcm90bxIIZm94Z2xvdmUaH2dvb2dsZS9wcm90b2J1Zi90aW1lc3RhbXAucHJvdG8ipAEKE1NjZW5lRW50aXR5RGVsZXRpb24SLQoJdGltZXN0YW1wGAEgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIwCgR0eXBlGAIgASgOMiIuZm94Z2xvdmUuU2NlbmVFbnRpdHlEZWxldGlvbi5UeXBlEgoKAmlkGAMgASgJIiAKBFR5cGUSDwoLTUFUQ0hJTkdfSUQQABIHCgNBTEwQAWIGcHJvdG8zCtgBChpmb3hnbG92ZS9TY2VuZVVwZGF0ZS5wcm90bxIIZm94Z2xvdmUaGmZveGdsb3ZlL1NjZW5lRW50aXR5LnByb3RvGiJmb3hnbG92ZS9TY2VuZUVudGl0eURlbGV0aW9uLnByb3RvImgKC1NjZW5lVXBkYXRlEjAKCWRlbGV0aW9ucxgBIAMoCzIdLmZveGdsb3ZlLlNjZW5lRW50aXR5RGVsZXRpb24SJwoIZW50aXRpZXMYAiADKAsyFS5mb3hnbG92ZS5TY2VuZUVudGl0eWIGcHJvdG8z", "base64", ), ); const sceneUpdate = sceneUpdateChannel.deserialize( + // cspell:disable-next-line Buffer.from("EhwKDAjHwtyhBhC4nNXrASIMCMfC3KEGELic1esB", "base64"), ); expect(sceneUpdate).toMatchInlineSnapshot(` @@ -225,6 +233,7 @@ describe("parseProtobufSchema", () => { // Duration too large expect(() => + // cspell:disable-next-line sceneUpdateChannel.deserialize(Buffer.from("EhMKBAgCEAMiCwiAgICAgICAEBAB", "base64")), ).toThrow( "Timestamps with seconds greater than 2^53-1 are not supported (found seconds=9007199254740992, nanos=1)", @@ -232,6 +241,7 @@ describe("parseProtobufSchema", () => { // Timestamp too large expect(() => + // cspell:disable-next-line sceneUpdateChannel.deserialize(Buffer.from("EhMKCwiAgICAgICAEBABIgQIAhAD", "base64")), ).toThrow( "Timestamps with seconds greater than 2^53-1 are not supported (found seconds=9007199254740992, nanos=1)", From 16a2a9ffb0d60474b0e835c1cfed385635b6c3d2 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 30 Jun 2023 19:52:51 -0700 Subject: [PATCH 07/27] fixes --- tests/conformance/.eslintrc.js | 5 ++++- tests/conformance/tsconfig.json | 6 +++++- yarn.lock | 27 ++------------------------- 3 files changed, 11 insertions(+), 27 deletions(-) diff --git a/tests/conformance/.eslintrc.js b/tests/conformance/.eslintrc.js index 99e1d0a0dc..be9e90eaf6 100644 --- a/tests/conformance/.eslintrc.js +++ b/tests/conformance/.eslintrc.js @@ -8,8 +8,11 @@ module.exports = { files: ["*.ts", "*.tsx"], extends: ["plugin:@foxglove/typescript"], parserOptions: { - project: "tsconfig.json", + project: ["tsconfig.json", "../../typescript/*/tsconfig.json"], tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, }, }, ], diff --git a/tests/conformance/tsconfig.json b/tests/conformance/tsconfig.json index 0ced8703c3..79e4f91da5 100644 --- a/tests/conformance/tsconfig.json +++ b/tests/conformance/tsconfig.json @@ -13,7 +13,11 @@ // required for tsconfig-paths https://github.com/dividab/tsconfig-paths/issues/143 "baseUrl": "." }, - "references": [{ "path": "../../typescript/core" }, { "path": "../../typescript/support" }], + "references": [ + { "path": "../../typescript/core" }, + { "path": "../../typescript/support" }, + { "path": "../../typescript/nodejs" } + ], "ts-node": { "require": ["tsconfig-paths/register"] // "files": true diff --git a/yarn.lock b/yarn.lock index 00f89280a5..8f8d664590 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7330,7 +7330,7 @@ __metadata: languageName: node linkType: hard -"enhanced-resolve@npm:^5.12.0": +"enhanced-resolve@npm:^5.12.0, enhanced-resolve@npm:^5.14.0": version: 5.15.0 resolution: "enhanced-resolve@npm:5.15.0" dependencies: @@ -7340,16 +7340,6 @@ __metadata: languageName: node linkType: hard -"enhanced-resolve@npm:^5.14.0": - version: 5.14.0 - resolution: "enhanced-resolve@npm:5.14.0" - dependencies: - graceful-fs: ^4.2.4 - tapable: ^2.2.0 - checksum: fff1aaebbf376371e5df4502e111967f6247c37611ad3550e4e7fca657f6dcb29ef7ffe88bf14e5010b78997f1ddd984a8db97af87ee0a5477771398fd326f5b - languageName: node - linkType: hard - "entities@npm:^2.0.0": version: 2.2.0 resolution: "entities@npm:2.2.0" @@ -8721,20 +8711,7 @@ __metadata: languageName: node linkType: hard -"globby@npm:^13.1.1": - version: 13.1.3 - resolution: "globby@npm:13.1.3" - dependencies: - dir-glob: ^3.0.1 - fast-glob: ^3.2.11 - ignore: ^5.2.0 - merge2: ^1.4.1 - slash: ^4.0.0 - checksum: 93f06e02002cdf368f7e3d55bd59e7b00784c7cc8fe92c7ee5082cc7171ff6109fda45e1c97a80bb48bc811dedaf7843c7c9186f5f84bde4883ab630e13c43df - languageName: node - linkType: hard - -"globby@npm:^13.1.3": +"globby@npm:^13.1.1, globby@npm:^13.1.3": version: 13.2.0 resolution: "globby@npm:13.2.0" dependencies: From 695bab93eae4ae939fe9860f87834afece6fb5a0 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 30 Jun 2023 19:56:55 -0700 Subject: [PATCH 08/27] release instructions cleanup --- RELEASE.md | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 4b0a91d425..3313e435b0 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -2,20 +2,11 @@ How to make releases. -## @mcap/core +## NPM - Checkout the version of the code you want to release -- Update package.json in typescript/core/package.json with the new version. +- Update package.json in typescript/{pkg}/package.json with the new version. - Make a PR with your changes to package.json - Wait for the PR to pass CI and merge -- Checkout main and tag the merged commit with `releases/typescript/core/v#.#.#` (replace #.#.# with the version you used in package.json) -- Push the new tag to the repo with `git push origin releases/typescript/core/v#.#.#` - -## @mcap/support - -- Checkout the version of the code you want to release -- Update package.json in typescript/support/package.json with the new version. -- Make a PR with your changes to package.json -- Wait for the PR to pass CI and merge -- Checkout main and tag the merged commit with `releases/typescript/support/v#.#.#` (replace #.#.# with the version you used in package.json) -- Push the new tag to the repo with `git push origin releases/typescript/support/v#.#.#` +- Checkout main and tag the merged commit with `releases/typescript/{pkg}/v#.#.#` (replace #.#.# with the version you used in package.json) +- Push the new tag to the repo with `git push origin releases/typescript/{pkg}/v#.#.#` From 4d121761b2acb463a1c1e29a30206a0b4b700d38 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 30 Jun 2023 20:15:46 -0700 Subject: [PATCH 09/27] cleanup, update validate script to use parseChannel --- .gitattributes | 1 + .vscode/settings.json | 3 +- RELEASE.md | 12 ---- tests/conformance/tsconfig.json | 6 +- typescript/CONTRIBUTING.md | 11 ++- typescript/examples/basicwriter/package.json | 2 +- typescript/examples/validate/package.json | 8 --- .../examples/validate/scripts/validate.ts | 68 ++++--------------- typescript/nodejs/LICENSE | 21 ++++++ typescript/support/.yarnrc | 2 - typescript/support/package.json | 2 +- yarn.lock | 12 +--- 12 files changed, 54 insertions(+), 94 deletions(-) delete mode 100644 RELEASE.md create mode 100644 typescript/nodejs/LICENSE delete mode 100644 typescript/support/.yarnrc diff --git a/.gitattributes b/.gitattributes index 965d9a40b3..90d0a6dc74 100644 --- a/.gitattributes +++ b/.gitattributes @@ -11,3 +11,4 @@ python/mcap-ros1-support/mcap_ros1/vendor/** linguist-vendored=true tests/conformance/data/** linguist-generated=true typescript/examples/flatbuffer/output/** linguist-generated=true go/ros/testdata/markers.bz2.bag filter=lfs diff=lfs merge=lfs -text +typescript/support/src/fixtures/byte-vector.ts linguist-generated=true diff --git a/.vscode/settings.json b/.vscode/settings.json index c4fbc14b68..4c5fc22e5c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,7 +24,8 @@ "python/**/build": true, "python/docs/*-apidoc": true, ".yarn/**": true, - "yarn.lock": true + "yarn.lock": true, + "**/dist": true }, "python.formatting.provider": "black", diff --git a/RELEASE.md b/RELEASE.md deleted file mode 100644 index 3313e435b0..0000000000 --- a/RELEASE.md +++ /dev/null @@ -1,12 +0,0 @@ -# Release - -How to make releases. - -## NPM - -- Checkout the version of the code you want to release -- Update package.json in typescript/{pkg}/package.json with the new version. -- Make a PR with your changes to package.json -- Wait for the PR to pass CI and merge -- Checkout main and tag the merged commit with `releases/typescript/{pkg}/v#.#.#` (replace #.#.# with the version you used in package.json) -- Push the new tag to the repo with `git push origin releases/typescript/{pkg}/v#.#.#` diff --git a/tests/conformance/tsconfig.json b/tests/conformance/tsconfig.json index 79e4f91da5..2deb6fae3e 100644 --- a/tests/conformance/tsconfig.json +++ b/tests/conformance/tsconfig.json @@ -7,6 +7,7 @@ "lib": ["es2020", "dom"], "paths": { "@mcap/core": ["../../typescript/core/src"], + "@mcap/nodejs": ["../../typescript/nodejs/src"], "@mcap/support": ["../../typescript/support/src"] }, @@ -15,11 +16,10 @@ }, "references": [ { "path": "../../typescript/core" }, - { "path": "../../typescript/support" }, - { "path": "../../typescript/nodejs" } + { "path": "../../typescript/nodejs" }, + { "path": "../../typescript/support" } ], "ts-node": { "require": ["tsconfig-paths/register"] - // "files": true } } diff --git a/typescript/CONTRIBUTING.md b/typescript/CONTRIBUTING.md index 276f4d904a..23052bd3e0 100644 --- a/typescript/CONTRIBUTING.md +++ b/typescript/CONTRIBUTING.md @@ -1,4 +1,4 @@ -## Development guide +# Development guide Install dependencies: @@ -31,3 +31,12 @@ Run benchmarks with Chrome debugger attached to use profiling tools: ``` yarn workspace @foxglove/mcap-benchmarks bench:debug ``` + +## Releasing to NPM + +- Check out the version of the code you want to release +- Update package.json in `typescript/{pkg}/package.json` with the new version. +- Make a PR with your changes to package.json +- Wait for the PR to pass CI and merge +- Checkout main and tag the merged commit with `releases/typescript/{pkg}/v#.#.#` (replace #.#.# with the version you used in package.json) +- Push the new tag to the repo with `git push origin releases/typescript/{pkg}/v#.#.#` diff --git a/typescript/examples/basicwriter/package.json b/typescript/examples/basicwriter/package.json index 7b4cd6d30a..e071a717a9 100644 --- a/typescript/examples/basicwriter/package.json +++ b/typescript/examples/basicwriter/package.json @@ -23,7 +23,7 @@ "@foxglove/eslint-plugin": "0.21.0", "@mcap/core": "workspace:*", "@mcap/nodejs": "workspace:*", - "@mcap/support": "*", + "@mcap/support": "workspace:*", "@types/node": "18.13.0", "eslint": "8.34.0", "eslint-config-prettier": "8.6.0", diff --git a/typescript/examples/validate/package.json b/typescript/examples/validate/package.json index 42e9836e9e..5605e949de 100644 --- a/typescript/examples/validate/package.json +++ b/typescript/examples/validate/package.json @@ -21,14 +21,7 @@ }, "devDependencies": { "@foxglove/eslint-plugin": "0.21.0", - "@foxglove/rosbag": "0.2.3", - "@foxglove/rosmsg": "3.1.0", - "@foxglove/rosmsg-serialization": "1.5.3", - "@foxglove/rosmsg2-serialization": "1.1.1", "@foxglove/tsconfig": "1.1.0", - "@foxglove/wasm-bz2": "0.1.1", - "@foxglove/wasm-lz4": "1.0.2", - "@foxglove/wasm-zstd": "1.0.1", "@mcap/core": "workspace:*", "@mcap/nodejs": "workspace:*", "@mcap/support": "workspace:*", @@ -45,7 +38,6 @@ "eslint-plugin-prettier": "4.2.1", "lodash": "4.17.21", "prettier": "2.8.4", - "protobufjs": "7.2.2", "ts-node": "10.9.1", "tsconfig-paths": "4.1.2", "typescript": "4.9.5" diff --git a/typescript/examples/validate/scripts/validate.ts b/typescript/examples/validate/scripts/validate.ts index 5585609ec8..98822c9e2f 100644 --- a/typescript/examples/validate/scripts/validate.ts +++ b/typescript/examples/validate/scripts/validate.ts @@ -1,6 +1,3 @@ -import { parse as parseMessageDefinition } from "@foxglove/rosmsg"; -import { LazyMessageReader as ROS1LazyMessageReader } from "@foxglove/rosmsg-serialization"; -import { MessageReader as ROS2MessageReader } from "@foxglove/rosmsg2-serialization"; import { hasMcapPrefix, McapConstants, @@ -9,14 +6,13 @@ import { McapTypes, } from "@mcap/core"; import { FileHandleReadable } from "@mcap/nodejs"; -import { loadDecompressHandlers, protobufFromBinaryDescriptor } from "@mcap/support"; +import { loadDecompressHandlers, parseChannel, ParsedChannel } from "@mcap/support"; import { program } from "commander"; import { createReadStream } from "fs"; import fs from "fs/promises"; import { isEqual } from "lodash"; import { performance } from "perf_hooks"; -type Channel = McapTypes.Channel; type TypedMcapRecord = McapTypes.TypedMcapRecord; function log(...data: unknown[]) { @@ -90,10 +86,7 @@ async function validate( const schemasById = new Map(); const channelInfoById = new Map< number, - { - info: Channel; - messageDeserializer?: (data: ArrayBufferView) => unknown; - } + { info: McapTypes.Channel; parsedChannel: ParsedChannel } >(); function processRecord(record: TypedMcapRecord) { @@ -123,47 +116,17 @@ async function validate( } break; } - if (record.schemaId === 0) { - throw new Error( - `Channel ${record.id} has no schema; channels without schemas are not supported`, - ); - } - const schema = schemasById.get(record.schemaId); - if (!schema) { - throw new Error(`Missing schema ${record.schemaId} for channel ${record.id}`); - } - let messageDeserializer: (data: ArrayBufferView) => unknown; - if (record.messageEncoding === "ros1") { - const reader = new ROS1LazyMessageReader( - parseMessageDefinition(new TextDecoder().decode(schema.data)), - ); - messageDeserializer = (data) => { - const size = reader.size(data); - if (size !== data.byteLength) { - throw new Error(`Message size ${size} should match buffer length ${data.byteLength}`); - } - return reader.readMessage(data).toJSON(); - }; - } else if (record.messageEncoding === "ros2") { - const reader = new ROS2MessageReader( - parseMessageDefinition(new TextDecoder().decode(schema.data), { - ros2: true, - }), - ); - messageDeserializer = (data) => reader.readMessage(data); - } else if (record.messageEncoding === "protobuf") { - const root = protobufFromBinaryDescriptor(schema.data); - const type = root.lookupType(schema.name); - - messageDeserializer = (data) => - type.decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength)); - } else if (record.messageEncoding === "json") { - const textDecoder = new TextDecoder(); - messageDeserializer = (data) => JSON.parse(textDecoder.decode(data)); - } else { - throw new Error(`unsupported encoding ${record.messageEncoding}`); + let schema: McapTypes.Schema | undefined; + if (record.schemaId !== 0) { + schema = schemasById.get(record.schemaId); + if (!schema) { + throw new Error(`Missing schema ${record.schemaId} for channel ${record.id}`); + } } - channelInfoById.set(record.id, { info: record, messageDeserializer }); + channelInfoById.set(record.id, { + info: record, + parsedChannel: parseChannel({ schema, messageEncoding: record.messageEncoding }), + }); break; } @@ -173,12 +136,7 @@ async function validate( throw new Error(`message for channel ${record.channelId} with no prior channel info`); } if (deserialize) { - if (channelInfo.messageDeserializer == undefined) { - throw new Error( - `No deserializer available for channel id: ${channelInfo.info.id} ${channelInfo.info.messageEncoding}`, - ); - } - const message = channelInfo.messageDeserializer(record.data); + const message = channelInfo.parsedChannel.deserialize(record.data); if (dump) { log(message); } diff --git a/typescript/nodejs/LICENSE b/typescript/nodejs/LICENSE new file mode 100644 index 0000000000..0aa593ea8c --- /dev/null +++ b/typescript/nodejs/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Foxglove Technologies Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/typescript/support/.yarnrc b/typescript/support/.yarnrc deleted file mode 100644 index 565db54d62..0000000000 --- a/typescript/support/.yarnrc +++ /dev/null @@ -1,2 +0,0 @@ -version-tag-prefix "releases/typescript/support/v" -version-git-message "TypeScript @mcap/support v%s" diff --git a/typescript/support/package.json b/typescript/support/package.json index a5da728e67..6cb091ebff 100644 --- a/typescript/support/package.json +++ b/typescript/support/package.json @@ -1,6 +1,6 @@ { "name": "@mcap/support", - "version": "0.1.0", + "version": "0.0.1", "description": "Common schema and message parsing logic for use with MCAP and related protocols", "license": "MIT", "repository": { diff --git a/yarn.lock b/yarn.lock index 8f8d664590..2fa42f4696 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2892,7 +2892,7 @@ __metadata: "@foxglove/eslint-plugin": 0.21.0 "@mcap/core": "workspace:*" "@mcap/nodejs": "workspace:*" - "@mcap/support": "*" + "@mcap/support": "workspace:*" "@types/node": 18.13.0 eslint: 8.34.0 eslint-config-prettier: 8.6.0 @@ -2933,14 +2933,7 @@ __metadata: resolution: "@foxglove/mcap-example-validate@workspace:typescript/examples/validate" dependencies: "@foxglove/eslint-plugin": 0.21.0 - "@foxglove/rosbag": 0.2.3 - "@foxglove/rosmsg": 3.1.0 - "@foxglove/rosmsg-serialization": 1.5.3 - "@foxglove/rosmsg2-serialization": 1.1.1 "@foxglove/tsconfig": 1.1.0 - "@foxglove/wasm-bz2": 0.1.1 - "@foxglove/wasm-lz4": 1.0.2 - "@foxglove/wasm-zstd": 1.0.1 "@mcap/core": "workspace:*" "@mcap/nodejs": "workspace:*" "@mcap/support": "workspace:*" @@ -2957,7 +2950,6 @@ __metadata: eslint-plugin-prettier: 4.2.1 lodash: 4.17.21 prettier: 2.8.4 - protobufjs: 7.2.2 ts-node: 10.9.1 tsconfig-paths: 4.1.2 typescript: 4.9.5 @@ -3572,7 +3564,7 @@ __metadata: languageName: unknown linkType: soft -"@mcap/support@*, @mcap/support@workspace:*, @mcap/support@workspace:typescript/support": +"@mcap/support@workspace:*, @mcap/support@workspace:typescript/support": version: 0.0.0-use.local resolution: "@mcap/support@workspace:typescript/support" dependencies: From 5589b01e8a2b77dead2b3b5ce2b696c82ee6329a Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 30 Jun 2023 20:22:43 -0700 Subject: [PATCH 10/27] cleanup & fixes --- .github/workflows/ci.yml | 6 ++++-- typescript/nodejs/package.json | 1 - typescript/support/package.json | 11 ----------- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0adca55d0c..89eeb7755a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -212,9 +212,11 @@ jobs: - run: yarn dedupe --check - run: yarn prettier:check - run: yarn workspace @mcap/core lint:ci - - run: yarn workspace @mcap/core typecheck + - run: yarn workspace @mcap/core build + - run: yarn workspace @mcap/nodejs lint:ci + - run: yarn workspace @mcap/nodejs build - run: yarn workspace @mcap/support lint:ci - - run: yarn workspace @mcap/support typecheck + - run: yarn workspace @mcap/support build - run: yarn typescript:test - name: Publish @mcap/core to NPM diff --git a/typescript/nodejs/package.json b/typescript/nodejs/package.json index a111d20f00..4595c092d4 100644 --- a/typescript/nodejs/package.json +++ b/typescript/nodejs/package.json @@ -28,7 +28,6 @@ "scripts": { "build": "tsc -b tsconfig.json tsconfig.cjs.json", "prepack": "yarn build", - "typecheck": "tsc -p tsconfig.json --noEmit", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", "test": "jest" diff --git a/typescript/support/package.json b/typescript/support/package.json index 6cb091ebff..2c7031de76 100644 --- a/typescript/support/package.json +++ b/typescript/support/package.json @@ -15,16 +15,6 @@ "module": "dist/esm/src/index.js", "main": "dist/cjs/src/index.js", "typings": "dist/esm/src/index.d.ts", - "exports": { - ".": { - "import": "./dist/esm/src/index.js", - "require": "./dist/cjs/src/index.js" - }, - "./nodejs": { - "import": "./dist/esm/src/nodejs/index.js", - "require": "./dist/cjs/src/nodejs/index.js" - } - }, "typedoc": { "entryPoint": "src/index.ts" }, @@ -37,7 +27,6 @@ "scripts": { "build": "tsc -b tsconfig.json tsconfig.cjs.json", "prepack": "yarn build", - "typecheck": "tsc -p tsconfig.json --noEmit", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", "test": "jest" From 145d6d42df4e287bcb04f5045d0eb57f2a7cb9df Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 30 Jun 2023 20:24:26 -0700 Subject: [PATCH 11/27] fix conformance-lint --- .github/workflows/ci.yml | 2 +- tests/conformance/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89eeb7755a..908d861a86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: cache: yarn - run: corepack enable && yarn install --immutable - run: yarn workspace @foxglove/mcap-conformance lint:ci - - run: yarn workspace @foxglove/mcap-conformance typecheck + - run: yarn workspace @foxglove/mcap-conformance build conformance-cpp: runs-on: ubuntu-latest diff --git a/tests/conformance/package.json b/tests/conformance/package.json index 00647c793a..93ea3d9c8c 100644 --- a/tests/conformance/package.json +++ b/tests/conformance/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "private": true, "scripts": { - "typecheck": "tsc -p tsconfig.json --noEmit", + "build": "tsc -b tsconfig.json tsconfig.cjs.json", "lint:ci": "eslint --report-unused-disable-directives .", "lint": "eslint --report-unused-disable-directives --fix .", "generate-inputs": "ts-node --files --project tsconfig.cjs.json scripts/generate-inputs", From 7dcef387be7b73f0c98d063405b12c269ea9a7ae Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 30 Jun 2023 20:48:56 -0700 Subject: [PATCH 12/27] remove links which are broken in typedoc rendered output --- typescript/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/typescript/README.md b/typescript/README.md index 8ed50bddd2..475dc8fe08 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -4,6 +4,6 @@ The following NPM packages are provided for use with JavaScript and TypeScript: -- [**@mcap/core**](./core) – low-level readers and writers -- [**@mcap/support**](./support) – support for well-known compression formats and encodings -- [**@mcap/nodejs**](./nodejs) – support for Node.js environment +- **@mcap/core** – low-level readers and writers +- **@mcap/support** – support for well-known compression formats and encodings +- **@mcap/nodejs** – support for Node.js environment From 5651518e21f0a8035a73d5d177e72d2a0ce0dda2 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 1 Sep 2023 19:27:14 -0700 Subject: [PATCH 13/27] Use @foxglove/protobufjs fork --- typescript/support/package.json | 4 +- typescript/support/src/parseChannel.test.ts | 2 +- .../support/src/parseJsonSchema.test.ts | 2 +- typescript/support/src/parseProtobufSchema.ts | 6 +-- .../src/protobufDefinitionsToDatatypes.ts | 2 +- typescript/support/src/protobufDescriptors.ts | 4 +- typescript/support/typings/protobufjs.d.ts | 6 +-- yarn.lock | 42 +++++++++---------- 8 files changed, 34 insertions(+), 34 deletions(-) diff --git a/typescript/support/package.json b/typescript/support/package.json index 2c7031de76..366926a77b 100644 --- a/typescript/support/package.json +++ b/typescript/support/package.json @@ -53,6 +53,7 @@ }, "dependencies": { "@foxglove/message-definition": "^0.2.0", + "@foxglove/protobufjs": "0.0.1-toobject-bigint.1", "@foxglove/rosmsg": "^4.2.2", "@foxglove/rosmsg-serialization": "^2.0.1", "@foxglove/rosmsg2-serialization": "^2.0.2", @@ -62,7 +63,6 @@ "@foxglove/wasm-zstd": "^1.0.1", "@protobufjs/base64": "^1.1.2", "flatbuffers": "^23.5.26", - "flatbuffers_reflection": "^0.0.6", - "protobufjs": "^7.2.2" + "flatbuffers_reflection": "^0.0.6" } } diff --git a/typescript/support/src/parseChannel.test.ts b/typescript/support/src/parseChannel.test.ts index 7f46003718..89259029f1 100644 --- a/typescript/support/src/parseChannel.test.ts +++ b/typescript/support/src/parseChannel.test.ts @@ -1,5 +1,5 @@ +import { FileDescriptorSet, IFileDescriptorSet } from "@foxglove/protobufjs/ext/descriptor"; import fs from "fs"; -import { FileDescriptorSet, IFileDescriptorSet } from "protobufjs/ext/descriptor"; import { parseChannel } from "./parseChannel"; diff --git a/typescript/support/src/parseJsonSchema.test.ts b/typescript/support/src/parseJsonSchema.test.ts index 105fcc01eb..ce2bd9bb2b 100644 --- a/typescript/support/src/parseJsonSchema.test.ts +++ b/typescript/support/src/parseJsonSchema.test.ts @@ -1,5 +1,5 @@ +import * as protobufjs from "@foxglove/protobufjs"; import { foxgloveMessageSchemas, generateJsonSchema } from "@foxglove/schemas/internal"; -import * as protobufjs from "protobufjs"; import { parseJsonSchema } from "./parseJsonSchema"; diff --git a/typescript/support/src/parseProtobufSchema.ts b/typescript/support/src/parseProtobufSchema.ts index 8726158f08..2d3b0f785c 100644 --- a/typescript/support/src/parseProtobufSchema.ts +++ b/typescript/support/src/parseProtobufSchema.ts @@ -1,5 +1,5 @@ -import protobufjs from "protobufjs"; -import { FileDescriptorSet } from "protobufjs/ext/descriptor"; +import protobufjs from "@foxglove/protobufjs"; +import { FileDescriptorSet } from "@foxglove/protobufjs/ext/descriptor"; import { protobufDefinitionsToDatatypes, stripLeadingDot } from "./protobufDefinitionsToDatatypes"; import { MessageDefinitionMap } from "./types"; @@ -61,7 +61,7 @@ export function parseProtobufSchema( const deserialize = (data: ArrayBufferView) => { return rootType.toObject( rootType.decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength)), - { defaults: true }, + { defaults: true, longs: BigInt }, ); }; diff --git a/typescript/support/src/protobufDefinitionsToDatatypes.ts b/typescript/support/src/protobufDefinitionsToDatatypes.ts index 2265f291e1..dc28239272 100644 --- a/typescript/support/src/protobufDefinitionsToDatatypes.ts +++ b/typescript/support/src/protobufDefinitionsToDatatypes.ts @@ -1,5 +1,5 @@ import { MessageDefinitionField } from "@foxglove/message-definition"; -import protobufjs from "protobufjs"; +import protobufjs from "@foxglove/protobufjs"; import { MessageDefinitionMap } from "./types"; diff --git a/typescript/support/src/protobufDescriptors.ts b/typescript/support/src/protobufDescriptors.ts index fb572a7a5b..42678d43dc 100644 --- a/typescript/support/src/protobufDescriptors.ts +++ b/typescript/support/src/protobufDescriptors.ts @@ -1,8 +1,8 @@ // eslint-disable-next-line @typescript-eslint/triple-slash-reference /// -import protobufjs from "protobufjs"; -import { FileDescriptorSet } from "protobufjs/ext/descriptor"; +import protobufjs from "@foxglove/protobufjs"; +import { FileDescriptorSet } from "@foxglove/protobufjs/ext/descriptor"; export type ProtobufDescriptor = ReturnType; diff --git a/typescript/support/typings/protobufjs.d.ts b/typescript/support/typings/protobufjs.d.ts index 8c0445835e..61be38e80a 100644 --- a/typescript/support/typings/protobufjs.d.ts +++ b/typescript/support/typings/protobufjs.d.ts @@ -1,8 +1,8 @@ -import protobufjs from "protobufjs"; -import descriptor from "protobufjs/ext/descriptor"; +import protobufjs from "@foxglove/protobufjs"; +import descriptor from "@foxglove/protobufjs/ext/descriptor"; // https://github.com/protobufjs/protobuf.js/issues/1499 -declare module "protobufjs" { +declare module "@foxglove/protobufjs" { interface ReflectionObject { toDescriptor( protoVersion: string, diff --git a/yarn.lock b/yarn.lock index 2fa42f4696..c296d630da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2963,6 +2963,26 @@ __metadata: languageName: node linkType: hard +"@foxglove/protobufjs@npm:0.0.1-toobject-bigint.1": + version: 0.0.1-toobject-bigint.1 + resolution: "@foxglove/protobufjs@npm:0.0.1-toobject-bigint.1" + dependencies: + "@protobufjs/aspromise": ^1.1.2 + "@protobufjs/base64": ^1.1.2 + "@protobufjs/codegen": ^2.0.4 + "@protobufjs/eventemitter": ^1.1.0 + "@protobufjs/fetch": ^1.1.0 + "@protobufjs/float": ^1.0.2 + "@protobufjs/inquire": ^1.1.0 + "@protobufjs/path": ^1.1.2 + "@protobufjs/pool": ^1.1.0 + "@protobufjs/utf8": ^1.1.0 + "@types/node": ">=13.7.0" + long: ^5.0.0 + checksum: bce02c9a255116927fe327a7fce1dde8682ff0e742064a0120e7944d346385e28aecc315030c5560afc0208eaf1448d2a36b87f9f424531b7dd06e1b18100808 + languageName: node + linkType: hard + "@foxglove/rosbag@npm:0.2.3": version: 0.2.3 resolution: "@foxglove/rosbag@npm:0.2.3" @@ -3570,6 +3590,7 @@ __metadata: dependencies: "@foxglove/eslint-plugin": 0.21.0 "@foxglove/message-definition": ^0.2.0 + "@foxglove/protobufjs": 0.0.1-toobject-bigint.1 "@foxglove/rosmsg": ^4.2.2 "@foxglove/rosmsg-serialization": ^2.0.1 "@foxglove/rosmsg2-serialization": ^2.0.2 @@ -3595,7 +3616,6 @@ __metadata: flatbuffers_reflection: ^0.0.6 jest: 29.4.3 prettier: 2.8.4 - protobufjs: ^7.2.2 ts-jest: 29.0.5 typescript: 4.9.5 languageName: unknown @@ -12662,26 +12682,6 @@ __metadata: languageName: node linkType: hard -"protobufjs@npm:^7.2.2": - version: 7.2.4 - resolution: "protobufjs@npm:7.2.4" - dependencies: - "@protobufjs/aspromise": ^1.1.2 - "@protobufjs/base64": ^1.1.2 - "@protobufjs/codegen": ^2.0.4 - "@protobufjs/eventemitter": ^1.1.0 - "@protobufjs/fetch": ^1.1.0 - "@protobufjs/float": ^1.0.2 - "@protobufjs/inquire": ^1.1.0 - "@protobufjs/path": ^1.1.2 - "@protobufjs/pool": ^1.1.0 - "@protobufjs/utf8": ^1.1.0 - "@types/node": ">=13.7.0" - long: ^5.0.0 - checksum: a952cdf2a5e5250c16ae651b570849b6f5b20a5475c3eef63ffb290ad239aa2916adfc1cc676f7fc93c69f48113df268761c0c246f7f023118c85bdd1a170044 - languageName: node - linkType: hard - "proxy-addr@npm:~2.0.7": version: 2.0.7 resolution: "proxy-addr@npm:2.0.7" From 69c8b72f0175c056168534d3e53a0991b3558517 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Tue, 12 Sep 2023 16:53:45 -0700 Subject: [PATCH 14/27] update from latest studio code --- cspell.config.yaml | 1 + typescript/support/package.json | 11 ++- typescript/support/src/TempBuffer.ts | 42 ++++++++++ typescript/support/src/decompressHandlers.ts | 2 +- typescript/support/src/index.ts | 1 + typescript/support/src/parseChannel.test.ts | 21 +++++ typescript/support/src/parseChannel.ts | 74 +++++++++++++---- .../support/src/parseFlatbufferSchema.test.ts | 8 +- .../support/src/parseFlatbufferSchema.ts | 2 +- .../support/src/parseJsonSchema.test.ts | 2 +- .../support/src/parseProtobufSchema.test.ts | 4 +- typescript/support/src/parseProtobufSchema.ts | 20 ++--- .../src/protobufDefinitionsToDatatypes.ts | 9 +++ yarn.lock | 79 +++++++++++++++---- 14 files changed, 220 insertions(+), 56 deletions(-) create mode 100644 typescript/support/src/TempBuffer.ts diff --git a/cspell.config.yaml b/cspell.config.yaml index afad2e584b..77b1b15ec8 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -103,6 +103,7 @@ words: - velodyne - waabi - webp + - xcdr - zstandard - zstd - zustand diff --git a/typescript/support/package.json b/typescript/support/package.json index 366926a77b..2bd6fe426e 100644 --- a/typescript/support/package.json +++ b/typescript/support/package.json @@ -52,17 +52,20 @@ "typescript": "4.9.5" }, "dependencies": { - "@foxglove/message-definition": "^0.2.0", + "@foxglove/message-definition": "^0.3.0", + "@foxglove/omgidl-parser": "^1.0.0", + "@foxglove/omgidl-serialization": "^1.0.0", "@foxglove/protobufjs": "0.0.1-toobject-bigint.1", + "@foxglove/ros2idl-parser": "^0.3.0", "@foxglove/rosmsg": "^4.2.2", - "@foxglove/rosmsg-serialization": "^2.0.1", + "@foxglove/rosmsg-serialization": "^2.0.2", "@foxglove/rosmsg2-serialization": "^2.0.2", - "@foxglove/schemas": "^1.3.1", + "@foxglove/schemas": "^1.5.1", "@foxglove/wasm-bz2": "^0.1.1", "@foxglove/wasm-lz4": "^1.0.2", "@foxglove/wasm-zstd": "^1.0.1", "@protobufjs/base64": "^1.1.2", "flatbuffers": "^23.5.26", - "flatbuffers_reflection": "^0.0.6" + "flatbuffers_reflection": "^0.0.7" } } diff --git a/typescript/support/src/TempBuffer.ts b/typescript/support/src/TempBuffer.ts new file mode 100644 index 0000000000..e6a93e7fd9 --- /dev/null +++ b/typescript/support/src/TempBuffer.ts @@ -0,0 +1,42 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/ + +import type { IWritable, McapTypes } from "@mcap/core"; + +/** + * In-memory buffer used for reading and writing MCAP files in tests. Can be used as both an IReadable and an IWritable. + */ +export class TempBuffer implements McapTypes.IReadable, IWritable { + #buffer = new ArrayBuffer(1024); + #size = 0; + + public position(): bigint { + return BigInt(this.#size); + } + + public async write(data: Uint8Array): Promise { + if (this.#size + data.byteLength > this.#buffer.byteLength) { + const newBuffer = new ArrayBuffer(this.#size + data.byteLength); + new Uint8Array(newBuffer).set(new Uint8Array(this.#buffer)); + this.#buffer = newBuffer; + } + new Uint8Array(this.#buffer, this.#size).set(data); + this.#size += data.byteLength; + } + + public async size(): Promise { + return BigInt(this.#size); + } + + public async read(offset: bigint, size: bigint): Promise { + if (offset < 0n || offset + size > BigInt(this.#buffer.byteLength)) { + throw new Error("read out of range"); + } + return new Uint8Array(this.#buffer, Number(offset), Number(size)); + } + + public get(): Uint8Array { + return new Uint8Array(this.#buffer, 0, this.#size); + } +} diff --git a/typescript/support/src/decompressHandlers.ts b/typescript/support/src/decompressHandlers.ts index 69f9d1031a..a71bab5112 100644 --- a/typescript/support/src/decompressHandlers.ts +++ b/typescript/support/src/decompressHandlers.ts @@ -1,4 +1,4 @@ -import { McapTypes } from "@mcap/core"; +import type { McapTypes } from "@mcap/core"; let handlersPromise: Promise | undefined; export async function loadDecompressHandlers(): Promise { diff --git a/typescript/support/src/index.ts b/typescript/support/src/index.ts index c8aee4c069..e0c22c89fd 100644 --- a/typescript/support/src/index.ts +++ b/typescript/support/src/index.ts @@ -3,3 +3,4 @@ export * from "./protobufDefinitionsToDatatypes"; export * from "./protobufDescriptors"; export * from "./parseChannel"; export * from "./decompressHandlers"; +export * from "./TempBuffer"; diff --git a/typescript/support/src/parseChannel.test.ts b/typescript/support/src/parseChannel.test.ts index 89259029f1..ca8044ab4b 100644 --- a/typescript/support/src/parseChannel.test.ts +++ b/typescript/support/src/parseChannel.test.ts @@ -88,4 +88,25 @@ describe("parseChannel", () => { const obj = channel.deserialize(new Uint8Array([0, 1, 0, 0, 5, 0, 0, 0, 65, 66, 67, 68, 0])); expect(obj).toEqual({ data: "ABCD" }); }); + it("works with omgidl xcdr2", () => { + const channel = parseChannel({ + messageEncoding: "cdr", + schema: { + name: "foo_msgs::Bar", + encoding: "omgidl", + data: new TextEncoder().encode(` + enum Color {RED, GREEN, BLUE}; + module foo_msgs { + struct NonRootBar {string data;}; + struct Bar {foo_msgs::NonRootBar data; Color color;}; + }; + `), + }, + }); + + const obj = channel.deserialize( + new Uint8Array([0, 1, 0, 0, 5, 0, 0, 0, 65, 66, 67, 68, 0, 0, 0, 0, 2, 0, 0, 0]), + ); + expect(obj).toEqual({ data: { data: "ABCD" }, color: 2 }); + }); }); diff --git a/typescript/support/src/parseChannel.ts b/typescript/support/src/parseChannel.ts index a3fc81aad8..e5fbe071c9 100644 --- a/typescript/support/src/parseChannel.ts +++ b/typescript/support/src/parseChannel.ts @@ -1,5 +1,8 @@ -import { MessageDefinition } from "@foxglove/message-definition"; -import { parse as parseMessageDefinition, parseRos2idl } from "@foxglove/rosmsg"; +import { MessageDefinition, MessageDefinitionField } from "@foxglove/message-definition"; +import { IDLMessageDefinition, parseIDL } from "@foxglove/omgidl-parser"; +import { MessageReader as OmgidlMessageReader } from "@foxglove/omgidl-serialization"; +import { parseRos2idl } from "@foxglove/ros2idl-parser"; +import { parse as parseMessageDefinition } from "@foxglove/rosmsg"; import { MessageReader } from "@foxglove/rosmsg-serialization"; import { MessageReader as ROS2MessageReader } from "@foxglove/rosmsg2-serialization"; @@ -18,13 +21,41 @@ export type ParsedChannel = { datatypes: MessageDefinitionMap; }; +function parseIDLDefinitionsToDatatypes( + parsedDefinitions: IDLMessageDefinition[], + rootName?: string, +) { + // The only IDL definition non-conformant-to-MessageDefinition is unions + const convertUnionToMessageDefinition = (definition: IDLMessageDefinition): MessageDefinition => { + if (definition.aggregatedKind === "union") { + const innerDefs: MessageDefinitionField[] = definition.cases.map((caseDefinition) => ({ + ...caseDefinition.type, + predicates: caseDefinition.predicates, + })); + + if (definition.defaultCase != undefined) { + innerDefs.push(definition.defaultCase); + } + const { name } = definition; + return { + name, + definitions: innerDefs, + }; + } + return definition; + }; + + const standardDefs: MessageDefinition[] = parsedDefinitions.map(convertUnionToMessageDefinition); + return parsedDefinitionsToDatatypes(standardDefs, rootName); +} + function parsedDefinitionsToDatatypes( parsedDefinitions: MessageDefinition[], - rootName: string, + rootName?: string, ): MessageDefinitionMap { const datatypes: MessageDefinitionMap = new Map(); parsedDefinitions.forEach(({ name, definitions }, index) => { - if (index === 0) { + if (rootName != undefined && index === 0) { datatypes.set(rootName, { name: rootName, definitions }); } else if (name != undefined) { datatypes.set(name, { name, definitions }); @@ -118,7 +149,11 @@ export function parseChannel(channel: Channel): ParsedChannel { } if (channel.messageEncoding === "cdr") { - if (channel.schema?.encoding !== "ros2msg" && channel.schema?.encoding !== "ros2idl") { + if ( + channel.schema?.encoding !== "ros2msg" && + channel.schema?.encoding !== "ros2idl" && + channel.schema?.encoding !== "omgidl" + ) { throw new Error( `Message encoding ${channel.messageEncoding} with ${ channel.schema == undefined @@ -128,17 +163,28 @@ export function parseChannel(channel: Channel): ParsedChannel { ); } const schema = new TextDecoder().decode(channel.schema.data); - const isIdl = channel.schema.encoding === "ros2idl"; + if (channel.schema.encoding === "omgidl") { + const parsedDefinitions = parseIDL(schema); + const reader = new OmgidlMessageReader(channel.schema.name, parsedDefinitions); + const datatypes = parseIDLDefinitionsToDatatypes(parsedDefinitions); + return { + datatypes, + deserialize: (data) => reader.readMessage(data), + }; + } else { + const isIdl = channel.schema.encoding === "ros2idl"; - const parsedDefinitions = isIdl - ? parseRos2idl(schema) - : parseMessageDefinition(schema, { ros2: true }); + const parsedDefinitions = isIdl + ? parseRos2idl(schema) + : parseMessageDefinition(schema, { ros2: true }); - const reader = new ROS2MessageReader(parsedDefinitions); - return { - datatypes: parsedDefinitionsToDatatypes(parsedDefinitions, channel.schema.name), - deserialize: (data) => reader.readMessage(data), - }; + const reader = new ROS2MessageReader(parsedDefinitions); + + return { + datatypes: parsedDefinitionsToDatatypes(parsedDefinitions, channel.schema.name), + deserialize: (data) => reader.readMessage(data), + }; + } } throw new Error(`Unsupported encoding ${channel.messageEncoding}`); diff --git a/typescript/support/src/parseFlatbufferSchema.test.ts b/typescript/support/src/parseFlatbufferSchema.test.ts index d7cba6298f..205ab52ca2 100644 --- a/typescript/support/src/parseFlatbufferSchema.test.ts +++ b/typescript/support/src/parseFlatbufferSchema.test.ts @@ -351,11 +351,11 @@ describe("parseFlatbufferSchema", () => { builder.finish(Type.endType(builder)); expect(deserialize(builder.asUint8Array())).toEqual({ - base_size: 4n, + base_size: 4, base_type: 7, - element: 0n, - element_size: 0n, - fixed_length: 0n, + element: 0, + element_size: 0, + fixed_length: 0, index: 123, }); }); diff --git a/typescript/support/src/parseFlatbufferSchema.ts b/typescript/support/src/parseFlatbufferSchema.ts index 60c5725ff8..6551c9ec09 100644 --- a/typescript/support/src/parseFlatbufferSchema.ts +++ b/typescript/support/src/parseFlatbufferSchema.ts @@ -140,7 +140,7 @@ function typeForField(schema: SchemaT, field: FieldT): MessageDefinitionField[] case BaseType.Array: case BaseType.MaxBaseType: case undefined: - throw new Error("Unions and Arrays are not supported in @mcap/support currently."); + throw new Error("Unions and Arrays are not currently supported"); } return fields; } diff --git a/typescript/support/src/parseJsonSchema.test.ts b/typescript/support/src/parseJsonSchema.test.ts index ce2bd9bb2b..105fcc01eb 100644 --- a/typescript/support/src/parseJsonSchema.test.ts +++ b/typescript/support/src/parseJsonSchema.test.ts @@ -1,5 +1,5 @@ -import * as protobufjs from "@foxglove/protobufjs"; import { foxgloveMessageSchemas, generateJsonSchema } from "@foxglove/schemas/internal"; +import * as protobufjs from "protobufjs"; import { parseJsonSchema } from "./parseJsonSchema"; diff --git a/typescript/support/src/parseProtobufSchema.test.ts b/typescript/support/src/parseProtobufSchema.test.ts index 97296c7d31..7f3de0e050 100644 --- a/typescript/support/src/parseProtobufSchema.test.ts +++ b/typescript/support/src/parseProtobufSchema.test.ts @@ -204,7 +204,7 @@ describe("parseProtobufSchema", () => { { "isArray": false, "name": "sec", - "type": "int64", + "type": "int32", }, { "isArray": false, @@ -220,7 +220,7 @@ describe("parseProtobufSchema", () => { { "isArray": false, "name": "sec", - "type": "int64", + "type": "int32", }, { "isArray": false, diff --git a/typescript/support/src/parseProtobufSchema.ts b/typescript/support/src/parseProtobufSchema.ts index 2d3b0f785c..cc312c67a8 100644 --- a/typescript/support/src/parseProtobufSchema.ts +++ b/typescript/support/src/parseProtobufSchema.ts @@ -25,32 +25,26 @@ export function parseProtobufSchema( // {sec: number, nsec: number}, compatible with the rest of Studio. The standard Protobuf types // use different names (`seconds` and `nanos`), and `seconds` is an `int64`, which would be // deserialized as a bigint by default. + // + // protobufDefinitionsToDatatypes also has matching logic to rename the fields. const fixTimeType = (type: protobufjs.ReflectionObject | null) => { if (!type || !(type instanceof protobufjs.Type)) { return; } - // Rename fields so that protobufDefinitionsToDatatypes uses the new names - for (const field of type.fieldsArray) { - if (field.name === "seconds") { - field.name = "sec"; - } else if (field.name === "nanos") { - field.name = "nsec"; - } - } type.setup(); // ensure the original optimized toObject has been created const prevToObject = type.toObject; // eslint-disable-line @typescript-eslint/unbound-method const newToObject: typeof prevToObject = (message, options) => { const result = prevToObject.call(type, message, options); - const { sec, nsec } = result as { sec: bigint; nsec: number }; - if (typeof sec !== "bigint" || typeof nsec !== "number") { + const { seconds, nanos } = result as { seconds: bigint; nanos: number }; + if (typeof seconds !== "bigint" || typeof nanos !== "number") { return result; } - if (sec > BigInt(Number.MAX_SAFE_INTEGER)) { + if (seconds > BigInt(Number.MAX_SAFE_INTEGER)) { throw new Error( - `Timestamps with seconds greater than 2^53-1 are not supported (found seconds=${sec}, nanos=${nsec})`, + `Timestamps with seconds greater than 2^53-1 are not supported (found seconds=${seconds}, nanos=${nanos})`, ); } - return { sec: Number(sec), nsec }; + return { sec: Number(seconds), nsec: nanos }; }; type.toObject = newToObject; }; diff --git a/typescript/support/src/protobufDefinitionsToDatatypes.ts b/typescript/support/src/protobufDefinitionsToDatatypes.ts index dc28239272..4eea790bdd 100644 --- a/typescript/support/src/protobufDefinitionsToDatatypes.ts +++ b/typescript/support/src/protobufDefinitionsToDatatypes.ts @@ -70,6 +70,15 @@ export function protobufDefinitionsToDatatypes( throw new Error("Repeated bytes are not currently supported"); } definitions.push({ type: "uint8", name: field.name, isArray: true }); + } else if ( + type.fullName === ".google.protobuf.Timestamp" || + type.fullName === ".google.protobuf.Duration" + ) { + definitions.push({ + type: "int32", + name: field.name === "seconds" ? "sec" : "nsec", + isArray: field.repeated, + }); } else { definitions.push({ type: protobufScalarToRosPrimitive(field.type), diff --git a/yarn.lock b/yarn.lock index c296d630da..78346ae163 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2756,6 +2756,13 @@ __metadata: languageName: node linkType: hard +"@foxglove/cdr@npm:3.1.0": + version: 3.1.0 + resolution: "@foxglove/cdr@npm:3.1.0" + checksum: df27e4d1f2f6f4dab2c61a6a8ead7140ae1fbe4007e43b6c6690022f45db3dee838c8323b8aaf1660c8a9cc5da831f71f93bfbd2f18d94c12d0d2a8716420e47 + languageName: node + linkType: hard + "@foxglove/cdr@npm:^2.0.0": version: 2.0.0 resolution: "@foxglove/cdr@npm:2.0.0" @@ -2963,6 +2970,32 @@ __metadata: languageName: node linkType: hard +"@foxglove/message-definition@npm:^0.3.0": + version: 0.3.0 + resolution: "@foxglove/message-definition@npm:0.3.0" + checksum: 48edc4e57b7b1152514fafe3eb0652ebd85032524d6a8c102db5e3a53b0311fde6200b5aeada6a76bafd6033d3d6ac1bd38e6aa331219f4421c356b8b6d7fd94 + languageName: node + linkType: hard + +"@foxglove/omgidl-parser@npm:1.0.0, @foxglove/omgidl-parser@npm:^1.0.0": + version: 1.0.0 + resolution: "@foxglove/omgidl-parser@npm:1.0.0" + dependencies: + tslib: ^2 + checksum: 449c138be6744782ef54c6dbf4c320e975acb8f53747097d0e7f55f40e91c315cc3b135899b5335b10f510061dd8b10d7c50952867b490ff6dfa0de420ac01f2 + languageName: node + linkType: hard + +"@foxglove/omgidl-serialization@npm:^1.0.0": + version: 1.0.0 + resolution: "@foxglove/omgidl-serialization@npm:1.0.0" + dependencies: + "@foxglove/cdr": 3.1.0 + "@foxglove/message-definition": ^0.2.0 + checksum: 6a85af96240c91e99ddd234b2f0da6c963fc74eff41e751349114ea2cd35cee01bfae5d72860bb268f3444add345a8c6eb381f2710a2724c0150ac98ff0c7ab0 + languageName: node + linkType: hard + "@foxglove/protobufjs@npm:0.0.1-toobject-bigint.1": version: 0.0.1-toobject-bigint.1 resolution: "@foxglove/protobufjs@npm:0.0.1-toobject-bigint.1" @@ -2983,6 +3016,17 @@ __metadata: languageName: node linkType: hard +"@foxglove/ros2idl-parser@npm:^0.3.0": + version: 0.3.0 + resolution: "@foxglove/ros2idl-parser@npm:0.3.0" + dependencies: + "@foxglove/message-definition": ^0.2.0 + "@foxglove/omgidl-parser": 1.0.0 + md5-typescript: ^1.0.5 + checksum: 564ef15ee804ba204c901d478a47d7e4f6d7efe1509849a25795ec0fec147fc50fb1a2c67c69474610378977e921f5ec3df402755f29dd7ad49d24c663e8a134 + languageName: node + linkType: hard + "@foxglove/rosbag@npm:0.2.3": version: 0.2.3 resolution: "@foxglove/rosbag@npm:0.2.3" @@ -3014,12 +3058,12 @@ __metadata: languageName: node linkType: hard -"@foxglove/rosmsg-serialization@npm:^2.0.1": - version: 2.0.1 - resolution: "@foxglove/rosmsg-serialization@npm:2.0.1" +"@foxglove/rosmsg-serialization@npm:^2.0.2": + version: 2.0.2 + resolution: "@foxglove/rosmsg-serialization@npm:2.0.2" dependencies: "@foxglove/message-definition": ^0.2.0 - checksum: 7591e9fdfaef60b7ae19fdbe11d74450d49ac7b6b27b56f931bded95ea4cc520c0c33ea2f24d7a077f6174fdf2dae54353065189c831beb65419a007d130e5c8 + checksum: 066172af1ab659379ce50735f4627ecb45f2d75313fe3bba4e19f1e7c9f3f17f265492558c80669aa60ce1aa76539e5be3433b1ce59c0f78743d773e21a0c947 languageName: node linkType: hard @@ -3083,13 +3127,13 @@ __metadata: languageName: node linkType: hard -"@foxglove/schemas@npm:^1.0.0, @foxglove/schemas@npm:^1.3.1": - version: 1.3.1 - resolution: "@foxglove/schemas@npm:1.3.1" +"@foxglove/schemas@npm:^1.0.0, @foxglove/schemas@npm:^1.5.1": + version: 1.5.1 + resolution: "@foxglove/schemas@npm:1.5.1" dependencies: "@foxglove/rosmsg-msgs-common": ^3.0.0 tslib: ^2.5.0 - checksum: 8824ccbe7a4b1cfe21efa3119e0041895dfd4db407c7f9f130abf134ade188d8cd905ada906c5d948c63a4461ebc1e74ba637ea107e686d33f085024b39bce41 + checksum: 4dd054a318ead308ce980e51eb50c3bcd1c59f9c94707234f8e293bc9b52f2727a21fdad4516f3c4ba0d19ec8058cd60315fe526b90ed9085d52519aa98d12ad languageName: node linkType: hard @@ -3589,12 +3633,15 @@ __metadata: resolution: "@mcap/support@workspace:typescript/support" dependencies: "@foxglove/eslint-plugin": 0.21.0 - "@foxglove/message-definition": ^0.2.0 + "@foxglove/message-definition": ^0.3.0 + "@foxglove/omgidl-parser": ^1.0.0 + "@foxglove/omgidl-serialization": ^1.0.0 "@foxglove/protobufjs": 0.0.1-toobject-bigint.1 + "@foxglove/ros2idl-parser": ^0.3.0 "@foxglove/rosmsg": ^4.2.2 - "@foxglove/rosmsg-serialization": ^2.0.1 + "@foxglove/rosmsg-serialization": ^2.0.2 "@foxglove/rosmsg2-serialization": ^2.0.2 - "@foxglove/schemas": ^1.3.1 + "@foxglove/schemas": ^1.5.1 "@foxglove/tsconfig": 1.1.0 "@foxglove/wasm-bz2": ^0.1.1 "@foxglove/wasm-lz4": ^1.0.2 @@ -3613,7 +3660,7 @@ __metadata: eslint-plugin-jest: 27.2.1 eslint-plugin-prettier: 4.2.1 flatbuffers: ^23.5.26 - flatbuffers_reflection: ^0.0.6 + flatbuffers_reflection: ^0.0.7 jest: 29.4.3 prettier: 2.8.4 ts-jest: 29.0.5 @@ -8286,10 +8333,10 @@ __metadata: languageName: node linkType: hard -"flatbuffers_reflection@npm:^0.0.6": - version: 0.0.6 - resolution: "flatbuffers_reflection@npm:0.0.6" - checksum: 65cc9aa9344705b50c106fed739b154b722f212e8af00663d035581914d5e7c229757410a54beb0d267bc6af0983032fe723d81af758d51c5c76cc8ed6c9f5de +"flatbuffers_reflection@npm:^0.0.7": + version: 0.0.7 + resolution: "flatbuffers_reflection@npm:0.0.7" + checksum: 09e075ddb655f76b910472392a501c47bb50840c9f921c5d1d567a9fb472d77166b584b62354bc8f4423fa2606040c87b8c6c09942068e5c08bc5a370ddbf891 languageName: node linkType: hard From 7a583ca9f2d7a632e59d3727cdefcaaaa2c2c080 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Tue, 12 Sep 2023 17:20:33 -0700 Subject: [PATCH 15/27] add @mcap/browser package with BlobReadable --- package.json | 1 + typescript/browser/.eslintrc.js | 28 +++++++++++++ typescript/browser/LICENSE | 21 ++++++++++ typescript/browser/README.md | 33 ++++++++++++++++ typescript/browser/jest.config.json | 19 +++++++++ typescript/browser/package.json | 55 ++++++++++++++++++++++++++ typescript/browser/src/BlobReadable.ts | 27 +++++++++++++ typescript/browser/src/index.test.ts | 46 +++++++++++++++++++++ typescript/browser/src/index.ts | 1 + typescript/browser/tsconfig.cjs.json | 7 ++++ typescript/browser/tsconfig.json | 11 ++++++ typescript/core/src/TempBuffer.ts | 41 ++++++++++--------- typescript/core/src/index.ts | 1 + typescript/nodejs/tsconfig.json | 2 +- typescript/support/README.md | 23 +++++++++-- typescript/support/src/TempBuffer.ts | 42 -------------------- typescript/support/src/index.ts | 1 - typescript/typedoc.json | 2 +- yarn.lock | 25 ++++++++++++ 19 files changed, 319 insertions(+), 67 deletions(-) create mode 100644 typescript/browser/.eslintrc.js create mode 100644 typescript/browser/LICENSE create mode 100644 typescript/browser/README.md create mode 100644 typescript/browser/jest.config.json create mode 100644 typescript/browser/package.json create mode 100644 typescript/browser/src/BlobReadable.ts create mode 100644 typescript/browser/src/index.test.ts create mode 100644 typescript/browser/src/index.ts create mode 100644 typescript/browser/tsconfig.cjs.json create mode 100644 typescript/browser/tsconfig.json delete mode 100644 typescript/support/src/TempBuffer.ts diff --git a/package.json b/package.json index 902fe5009d..6c406c08e9 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "typescript/examples/*", "typescript/support", "typescript/nodejs", + "typescript/browser", "website" ] }, diff --git a/typescript/browser/.eslintrc.js b/typescript/browser/.eslintrc.js new file mode 100644 index 0000000000..85bdbf92d7 --- /dev/null +++ b/typescript/browser/.eslintrc.js @@ -0,0 +1,28 @@ +/* eslint-env node */ +module.exports = { + env: { es2020: true }, + ignorePatterns: ["dist"], + extends: ["plugin:@foxglove/base", "plugin:@foxglove/jest", "plugin:import/recommended"], + overrides: [ + { + files: ["*.ts", "*.tsx"], + extends: ["plugin:@foxglove/typescript"], + parserOptions: { + project: "../*/tsconfig.json", + tsconfigRootDir: __dirname, + // Enable typescript-eslint to use `src` files for type information across project references + // + EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true, + }, + }, + ], + rules: { + "no-warning-comments": ["error", { terms: ["fixme"], location: "anywhere" }], + }, + settings: { + "import/resolver": { + typescript: true, + node: true, + }, + }, +}; diff --git a/typescript/browser/LICENSE b/typescript/browser/LICENSE new file mode 100644 index 0000000000..0aa593ea8c --- /dev/null +++ b/typescript/browser/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Foxglove Technologies Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/typescript/browser/README.md b/typescript/browser/README.md new file mode 100644 index 0000000000..d3050d47f4 --- /dev/null +++ b/typescript/browser/README.md @@ -0,0 +1,33 @@ +# @mcap/browser + +[MCAP](https://mcap.dev/) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. + +The `@mcap/browser` package provides utilities for working with MCAP files in browsers. + +## Usage examples + +### Reading MCAP files in a browser + +```ts +import { loadDecompressHandlers } from "@mcap/support"; +import { BlobReadable } from "@mcap/browser"; +import { McapIndexedReader } from "@mcap/core"; +import { open } from "fs/promises"; + +async function onInputOrDrop(event: InputEvent | DragEvent) { + const file = event.dataTransfer.files[0]; + const decompressHandlers = await loadDecompressHandlers(); + const reader = await McapIndexedReader.Initialize({ + readable: new BlobReadable(file), + decompressHandlers, + }); +} +``` + +## License + +`@mcap/browser` is licensed under the [MIT License](https://opensource.org/licenses/MIT). + +## Stay in touch + +Join our [Slack channel](https://foxglove.dev/slack) to ask questions, share feedback, and stay up to date on what our team is working on. diff --git a/typescript/browser/jest.config.json b/typescript/browser/jest.config.json new file mode 100644 index 0000000000..09337c2e52 --- /dev/null +++ b/typescript/browser/jest.config.json @@ -0,0 +1,19 @@ +{ + "testMatch": ["/src/**/*.test.ts"], + "transform": { + "^.+\\.ts$": [ + "ts-jest", + { + "diagnostics": { + "//": "add 6133 (unused variables) to default ignore codes", + "ignoreCodes": [6059, 18002, 18003, 6133] + } + } + ] + }, + "moduleNameMapper": { + "^@mcap/core$": "/../core/src" + }, + "//": "Native find is slow because it does not exclude files: https://github.com/facebook/jest/pull/11264#issuecomment-825377579", + "haste": { "forceNodeFilesystemAPI": true } +} diff --git a/typescript/browser/package.json b/typescript/browser/package.json new file mode 100644 index 0000000000..72e7f963c4 --- /dev/null +++ b/typescript/browser/package.json @@ -0,0 +1,55 @@ +{ + "name": "@mcap/browser", + "version": "0.1.0", + "description": "Support library for using MCAP in the browser", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/foxglove/mcap.git" + }, + "author": { + "name": "Foxglove Technologies", + "email": "support@foxglove.dev" + }, + "homepage": "https://foxglove.dev/", + "module": "dist/esm/src/index.js", + "main": "dist/cjs/src/index.js", + "typings": "dist/esm/src/index.d.ts", + "typedoc": { + "entryPoint": "src/index.ts" + }, + "files": [ + "dist", + "src" + ], + "engines": { + "node": ">=14.0.0" + }, + "scripts": { + "build": "tsc -b tsconfig.json tsconfig.cjs.json", + "prepack": "yarn build", + "lint:ci": "eslint --report-unused-disable-directives .", + "lint": "eslint --report-unused-disable-directives --fix .", + "test": "jest" + }, + "devDependencies": { + "@foxglove/eslint-plugin": "0.21.0", + "@foxglove/tsconfig": "1.1.0", + "@mcap/core": "workspace:*", + "@types/jest": "29.4.0", + "@typescript-eslint/eslint-plugin": "5.52.0", + "@typescript-eslint/parser": "5.52.0", + "eslint": "8.34.0", + "eslint-config-prettier": "8.6.0", + "eslint-import-resolver-typescript": "3.5.5", + "eslint-plugin-es": "4.1.0", + "eslint-plugin-filenames": "1.3.2", + "eslint-plugin-import": "2.27.5", + "eslint-plugin-jest": "27.2.1", + "eslint-plugin-prettier": "4.2.1", + "jest": "29.4.3", + "prettier": "2.8.4", + "ts-jest": "29.0.5", + "typescript": "4.9.5" + } +} diff --git a/typescript/browser/src/BlobReadable.ts b/typescript/browser/src/BlobReadable.ts new file mode 100644 index 0000000000..51a438d998 --- /dev/null +++ b/typescript/browser/src/BlobReadable.ts @@ -0,0 +1,27 @@ +import type { McapTypes } from "@mcap/core"; + +/** + * IReadable implementation for Blob (and File, which is a Blob). + */ +export class BlobReadable implements McapTypes.IReadable { + #blob: Blob; + + public constructor(blob: Blob) { + this.#blob = blob; + } + + public async size(): Promise { + return BigInt(this.#blob.size); + } + + public async read(offset: bigint, size: bigint): Promise { + if (offset + size > this.#blob.size) { + throw new Error( + `Read of ${size} bytes at offset ${offset} exceeds file size ${this.#blob.size}`, + ); + } + return new Uint8Array( + await this.#blob.slice(Number(offset), Number(offset + size)).arrayBuffer(), + ); + } +} diff --git a/typescript/browser/src/index.test.ts b/typescript/browser/src/index.test.ts new file mode 100644 index 0000000000..77e95f993e --- /dev/null +++ b/typescript/browser/src/index.test.ts @@ -0,0 +1,46 @@ +import { McapWriter, McapIndexedReader, TempBuffer } from "@mcap/core"; +import { Blob } from "buffer"; + +import { BlobReadable } from "./BlobReadable"; + +async function collect(iterable: AsyncIterable): Promise { + const result: T[] = []; + for await (const item of iterable) { + result.push(item); + } + return result; +} + +describe("BlobReadable", () => { + it("reads blob", async () => { + const tempBuffer = new TempBuffer(); + + const header = { library: "lib", profile: "prof" }; + const writer = new McapWriter({ writable: tempBuffer }); + await writer.start(header); + const channel = { + topic: "foo", + schemaId: 0, + messageEncoding: "enc", + metadata: new Map(), + }; + const channelId = await writer.registerChannel(channel); + const message = { + channelId, + sequence: 1, + logTime: 1n, + publishTime: 2n, + data: new Uint8Array([1, 2, 3]), + }; + await writer.addMessage(message); + await writer.end(); + + const blob = new Blob([tempBuffer.get()]); + + const reader = await McapIndexedReader.Initialize({ + readable: new BlobReadable(blob as unknown as ConstructorParameters[0]), + }); + expect(reader.header).toEqual({ ...header, type: "Header" }); + expect(await collect(reader.readMessages())).toEqual([{ ...message, type: "Message" }]); + }); +}); diff --git a/typescript/browser/src/index.ts b/typescript/browser/src/index.ts new file mode 100644 index 0000000000..cd3b52e1bd --- /dev/null +++ b/typescript/browser/src/index.ts @@ -0,0 +1 @@ +export * from "./BlobReadable"; diff --git a/typescript/browser/tsconfig.cjs.json b/typescript/browser/tsconfig.cjs.json new file mode 100644 index 0000000000..ac31cb3343 --- /dev/null +++ b/typescript/browser/tsconfig.cjs.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist/cjs", + "module": "commonjs" + } +} diff --git a/typescript/browser/tsconfig.json b/typescript/browser/tsconfig.json new file mode 100644 index 0000000000..6a6b398a2e --- /dev/null +++ b/typescript/browser/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "@foxglove/tsconfig/base", + "include": ["./src/**/*"], + "compilerOptions": { + "outDir": "./dist/esm", + "lib": ["es2020", "dom"], + "composite": true, + "incremental": true + }, + "references": [{ "path": "../core" }] +} diff --git a/typescript/core/src/TempBuffer.ts b/typescript/core/src/TempBuffer.ts index c88bacdf25..81ac9d3669 100644 --- a/typescript/core/src/TempBuffer.ts +++ b/typescript/core/src/TempBuffer.ts @@ -1,34 +1,39 @@ import { IWritable } from "./IWritable"; import { IReadable } from "./types"; +/** + * In-memory buffer used for reading and writing MCAP files in tests. Can be used as both an IReadable and an IWritable. + */ export class TempBuffer implements IReadable, IWritable { - private _buffer = new ArrayBuffer(1024); - private _size = 0; + #buffer = new ArrayBuffer(1024); + #size = 0; - position(): bigint { - return BigInt(this._size); + public position(): bigint { + return BigInt(this.#size); } - async write(data: Uint8Array): Promise { - if (this._size + data.byteLength > this._buffer.byteLength) { - const newBuffer = new ArrayBuffer(this._size + data.byteLength); - new Uint8Array(newBuffer).set(new Uint8Array(this._buffer)); - this._buffer = newBuffer; + + public async write(data: Uint8Array): Promise { + if (this.#size + data.byteLength > this.#buffer.byteLength) { + const newBuffer = new ArrayBuffer(this.#size + data.byteLength); + new Uint8Array(newBuffer).set(new Uint8Array(this.#buffer)); + this.#buffer = newBuffer; } - new Uint8Array(this._buffer, this._size).set(data); - this._size += data.byteLength; + new Uint8Array(this.#buffer, this.#size).set(data); + this.#size += data.byteLength; } - async size(): Promise { - return BigInt(this._size); + public async size(): Promise { + return BigInt(this.#size); } - async read(offset: bigint, size: bigint): Promise { - if (offset < 0n || offset + size > BigInt(this._buffer.byteLength)) { + + public async read(offset: bigint, size: bigint): Promise { + if (offset < 0n || offset + size > BigInt(this.#buffer.byteLength)) { throw new Error("read out of range"); } - return new Uint8Array(this._buffer, Number(offset), Number(size)); + return new Uint8Array(this.#buffer, Number(offset), Number(size)); } - get(): Uint8Array { - return new Uint8Array(this._buffer, 0, this._size); + public get(): Uint8Array { + return new Uint8Array(this.#buffer, 0, this.#size); } } diff --git a/typescript/core/src/index.ts b/typescript/core/src/index.ts index 8edfe0416a..051122da3b 100644 --- a/typescript/core/src/index.ts +++ b/typescript/core/src/index.ts @@ -10,3 +10,4 @@ export type { IWritable } from "./IWritable"; export * from "./hasMcapPrefix"; export * from "./parse"; +export * from "./TempBuffer"; diff --git a/typescript/nodejs/tsconfig.json b/typescript/nodejs/tsconfig.json index 6a6b398a2e..ff2ce52dc6 100644 --- a/typescript/nodejs/tsconfig.json +++ b/typescript/nodejs/tsconfig.json @@ -3,7 +3,7 @@ "include": ["./src/**/*"], "compilerOptions": { "outDir": "./dist/esm", - "lib": ["es2020", "dom"], + "lib": ["es2020"], "composite": true, "incremental": true }, diff --git a/typescript/support/README.md b/typescript/support/README.md index e598f235fc..51dccec9ca 100644 --- a/typescript/support/README.md +++ b/typescript/support/README.md @@ -8,17 +8,32 @@ The `@mcap/support` package provides utilities for working with MCAP files that ### Reading MCAP files in a browser -TODO +```ts +import { loadDecompressHandlers } from "@mcap/support"; +import { BlobReadable } from "@mcap/browser"; +import { McapIndexedReader } from "@mcap/core"; +import { open } from "fs/promises"; + +async function onInputOrDrop(event: InputEvent | DragEvent) { + const file = event.dataTransfer.files[0]; + const decompressHandlers = await loadDecompressHandlers(); + const reader = await McapIndexedReader.Initialize({ + readable: new BlobReadable(file), + decompressHandlers, + }); +} +``` ### Reading MCAP files in Node.js ```ts import { loadDecompressHandlers } from "@mcap/support"; -import { FileHandleReadable } from "@mcap/support/nodejs"; -const decompressHandlers = await loadDecompressHandlers(); +import { FileHandleReadable } from "@mcap/nodejs"; +import { McapIndexedReader } from "@mcap/core"; +import { open } from "fs/promises"; +const decompressHandlers = await loadDecompressHandlers(); const fileHandle = await open("file.mcap", "r"); - const reader = await McapIndexedReader.Initialize({ readable: new FileHandleReadable(fileHandle), decompressHandlers, diff --git a/typescript/support/src/TempBuffer.ts b/typescript/support/src/TempBuffer.ts deleted file mode 100644 index e6a93e7fd9..0000000000 --- a/typescript/support/src/TempBuffer.ts +++ /dev/null @@ -1,42 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/ - -import type { IWritable, McapTypes } from "@mcap/core"; - -/** - * In-memory buffer used for reading and writing MCAP files in tests. Can be used as both an IReadable and an IWritable. - */ -export class TempBuffer implements McapTypes.IReadable, IWritable { - #buffer = new ArrayBuffer(1024); - #size = 0; - - public position(): bigint { - return BigInt(this.#size); - } - - public async write(data: Uint8Array): Promise { - if (this.#size + data.byteLength > this.#buffer.byteLength) { - const newBuffer = new ArrayBuffer(this.#size + data.byteLength); - new Uint8Array(newBuffer).set(new Uint8Array(this.#buffer)); - this.#buffer = newBuffer; - } - new Uint8Array(this.#buffer, this.#size).set(data); - this.#size += data.byteLength; - } - - public async size(): Promise { - return BigInt(this.#size); - } - - public async read(offset: bigint, size: bigint): Promise { - if (offset < 0n || offset + size > BigInt(this.#buffer.byteLength)) { - throw new Error("read out of range"); - } - return new Uint8Array(this.#buffer, Number(offset), Number(size)); - } - - public get(): Uint8Array { - return new Uint8Array(this.#buffer, 0, this.#size); - } -} diff --git a/typescript/support/src/index.ts b/typescript/support/src/index.ts index e0c22c89fd..c8aee4c069 100644 --- a/typescript/support/src/index.ts +++ b/typescript/support/src/index.ts @@ -3,4 +3,3 @@ export * from "./protobufDefinitionsToDatatypes"; export * from "./protobufDescriptors"; export * from "./parseChannel"; export * from "./decompressHandlers"; -export * from "./TempBuffer"; diff --git a/typescript/typedoc.json b/typescript/typedoc.json index 0cc0fe0eef..461c973119 100644 --- a/typescript/typedoc.json +++ b/typescript/typedoc.json @@ -1,5 +1,5 @@ { "name": "MCAP TypeScript SDK", "entryPointStrategy": "packages", - "entryPoints": ["./core", "./support", "./nodejs"] + "entryPoints": ["./core", "./support", "./nodejs", "./browser"] } diff --git a/yarn.lock b/yarn.lock index 78346ae163..8fd7531620 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3572,6 +3572,31 @@ __metadata: languageName: node linkType: hard +"@mcap/browser@workspace:typescript/browser": + version: 0.0.0-use.local + resolution: "@mcap/browser@workspace:typescript/browser" + dependencies: + "@foxglove/eslint-plugin": 0.21.0 + "@foxglove/tsconfig": 1.1.0 + "@mcap/core": "workspace:*" + "@types/jest": 29.4.0 + "@typescript-eslint/eslint-plugin": 5.52.0 + "@typescript-eslint/parser": 5.52.0 + eslint: 8.34.0 + eslint-config-prettier: 8.6.0 + eslint-import-resolver-typescript: 3.5.5 + eslint-plugin-es: 4.1.0 + eslint-plugin-filenames: 1.3.2 + eslint-plugin-import: 2.27.5 + eslint-plugin-jest: 27.2.1 + eslint-plugin-prettier: 4.2.1 + jest: 29.4.3 + prettier: 2.8.4 + ts-jest: 29.0.5 + typescript: 4.9.5 + languageName: unknown + linkType: soft + "@mcap/core@*, @mcap/core@workspace:*, @mcap/core@workspace:typescript/core": version: 0.0.0-use.local resolution: "@mcap/core@workspace:typescript/core" From 5d89e9cc50b997db761e72dbd5b66e6b6040a67b Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Tue, 12 Sep 2023 17:48:10 -0700 Subject: [PATCH 16/27] Extra protobuf time/duration modifications so they can live in studio only --- typescript/support/src/parseChannel.test.ts | 20 +- typescript/support/src/parseChannel.ts | 10 +- .../support/src/parseProtobufSchema.test.ts | 216 ++++++------------ typescript/support/src/parseProtobufSchema.ts | 58 ++--- .../src/protobufDefinitionsToDatatypes.ts | 13 +- 5 files changed, 121 insertions(+), 196 deletions(-) diff --git a/typescript/support/src/parseChannel.test.ts b/typescript/support/src/parseChannel.test.ts index ca8044ab4b..d5a513e771 100644 --- a/typescript/support/src/parseChannel.test.ts +++ b/typescript/support/src/parseChannel.test.ts @@ -34,13 +34,25 @@ describe("parseChannel", () => { }); it("works with protobuf", () => { + const processRootType = jest.fn().mockImplementation((x: unknown) => x); + const processMessageDefinitions = jest.fn().mockImplementation((x: unknown) => x); const fds = FileDescriptorSet.encode(FileDescriptorSet.root.toDescriptor("proto3")).finish(); - const channel = parseChannel({ - messageEncoding: "protobuf", - schema: { name: "google.protobuf.FileDescriptorSet", encoding: "protobuf", data: fds }, - }); + const channel = parseChannel( + { + messageEncoding: "protobuf", + schema: { name: "google.protobuf.FileDescriptorSet", encoding: "protobuf", data: fds }, + }, + { + protobuf: { + processRootType, + processMessageDefinitions, + }, + }, + ); const deserialized = channel.deserialize(fds) as IFileDescriptorSet; expect(deserialized.file[0]!.name).toEqual("google_protobuf.proto"); + expect(processRootType).toHaveBeenCalled(); + expect(processMessageDefinitions).toHaveBeenCalled(); }); it("works with ros1", () => { diff --git a/typescript/support/src/parseChannel.ts b/typescript/support/src/parseChannel.ts index e5fbe071c9..214caaadb8 100644 --- a/typescript/support/src/parseChannel.ts +++ b/typescript/support/src/parseChannel.ts @@ -8,7 +8,7 @@ import { MessageReader as ROS2MessageReader } from "@foxglove/rosmsg2-serializat import { parseFlatbufferSchema } from "./parseFlatbufferSchema"; import { parseJsonSchema } from "./parseJsonSchema"; -import { parseProtobufSchema } from "./parseProtobufSchema"; +import { ParseProtobufSchemaOptions, parseProtobufSchema } from "./parseProtobufSchema"; import { MessageDefinitionMap } from "./types"; type Channel = { @@ -64,6 +64,10 @@ function parsedDefinitionsToDatatypes( return datatypes; } +export type ParseChannelOptions = { + protobuf?: ParseProtobufSchemaOptions; +}; + /** * Process a channel/schema and extract information that can be used to deserialize messages on the * channel, and schemas in the format expected by Studio's RosDatatypes. @@ -72,7 +76,7 @@ function parsedDefinitionsToDatatypes( * - https://github.com/foxglove/mcap/blob/main/docs/specification/well-known-message-encodings.md * - https://github.com/foxglove/mcap/blob/main/docs/specification/well-known-schema-encodings.md */ -export function parseChannel(channel: Channel): ParsedChannel { +export function parseChannel(channel: Channel, options?: ParseChannelOptions): ParsedChannel { if (channel.messageEncoding === "json") { if (channel.schema != undefined && channel.schema.encoding !== "jsonschema") { throw new Error( @@ -126,7 +130,7 @@ export function parseChannel(channel: Channel): ParsedChannel { } is not supported (expected protobuf)`, ); } - return parseProtobufSchema(channel.schema.name, channel.schema.data); + return parseProtobufSchema(channel.schema.name, channel.schema.data, options?.protobuf); } if (channel.messageEncoding === "ros1") { diff --git a/typescript/support/src/parseProtobufSchema.test.ts b/typescript/support/src/parseProtobufSchema.test.ts index 7f3de0e050..bb0bdbd59a 100644 --- a/typescript/support/src/parseProtobufSchema.test.ts +++ b/typescript/support/src/parseProtobufSchema.test.ts @@ -26,6 +26,77 @@ describe("parseProtobufSchema", () => { ), ); expect(channel.deserialize(Buffer.from("0A0101", "hex"))).toEqual({ data: [1] }); + expect(channel.datatypes).toEqual( + new Map([ + [ + "ExampleMessage", + { + definitions: [ + { isConstant: true, name: "UNKNOWN", type: "int32", value: 0 }, + { isConstant: true, name: "WHATEVER", type: "int32", value: 0 }, + { isConstant: true, name: "FOO", type: "int32", value: 1 }, + { isConstant: true, name: "BAR", type: "int32", value: 2 }, + { name: "data", type: "int32" }, + ], + }, + ], + ]), + ); + }); + + it("allows modifying deserialization and datatypes", () => { + /* + syntax = "proto3"; + + enum ExampleEnum { + option allow_alias = true; + UNKNOWN = 0; + WHATEVER = 0; + FOO = 1; + BAR = 2; + } + + message ExampleMessage { + repeated ExampleEnum data = 1; + } + */ + const channel = parseProtobufSchema( + "ExampleMessage", + Buffer.from( + // cspell:disable-next-line + "0A8D010A156578616D706C655F6D6573736167652E70726F746F222C0A0E4578616D706C654D657373616765121A0A046461746118012003280E320C2E4578616D706C65456E756D2A3E0A0B4578616D706C65456E756D120B0A07554E4B4E4F574E1000120C0A085748415445564552100012070A03464F4F100112070A0342415210021A021001620670726F746F33", + "hex", + ), + { + processRootType(rootType) { + rootType.fieldsById[1]!.name = "renamed_data"; + return rootType; + }, + processMessageDefinitions(definitions) { + definitions + .get("ExampleMessage")! + .definitions.find((def) => def.name === "renamed_data")!.type = "float64"; + return definitions; + }, + }, + ); + expect(channel.deserialize(Buffer.from("0A0101", "hex"))).toEqual({ renamed_data: [1] }); + expect(channel.datatypes).toEqual( + new Map([ + [ + "ExampleMessage", + { + definitions: [ + { isConstant: true, name: "UNKNOWN", type: "int32", value: 0 }, + { isConstant: true, name: "WHATEVER", type: "int32", value: 0 }, + { isConstant: true, name: "FOO", type: "int32", value: 1 }, + { isConstant: true, name: "BAR", type: "int32", value: 2 }, + { name: "renamed_data", type: "float64" }, + ], + }, + ], + ]), + ); }); it("handles protobuf int64 values", () => { @@ -102,149 +173,4 @@ describe("parseProtobufSchema", () => { ], }); }); - - it("converts protobuf time/duration int64 to number rather than bigint", () => { - const poseInFrameChannel = parseProtobufSchema( - "foxglove.PoseInFrame", - Buffer.from( - // cspell:disable-next-line - "CmcKGWZveGdsb3ZlL1F1YXRlcm5pb24ucHJvdG8SCGZveGdsb3ZlIjgKClF1YXRlcm5pb24SCQoBeBgBIAEoARIJCgF5GAIgASgBEgkKAXoYAyABKAESCQoBdxgEIAEoAWIGcHJvdG8zClYKFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8SCGZveGdsb3ZlIioKB1ZlY3RvcjMSCQoBeBgBIAEoARIJCgF5GAIgASgBEgkKAXoYAyABKAFiBnByb3RvMwqyAQoTZm94Z2xvdmUvUG9zZS5wcm90bxIIZm94Z2xvdmUaGWZveGdsb3ZlL1F1YXRlcm5pb24ucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8iVgoEUG9zZRIjCghwb3NpdGlvbhgBIAEoCzIRLmZveGdsb3ZlLlZlY3RvcjMSKQoLb3JpZW50YXRpb24YAiABKAsyFC5mb3hnbG92ZS5RdWF0ZXJuaW9uYgZwcm90bzMK7wEKH2dvb2dsZS9wcm90b2J1Zi90aW1lc3RhbXAucHJvdG8SD2dvb2dsZS5wcm90b2J1ZiIrCglUaW1lc3RhbXASDwoHc2Vjb25kcxgBIAEoAxINCgVuYW5vcxgCIAEoBUKFAQoTY29tLmdvb2dsZS5wcm90b2J1ZkIOVGltZXN0YW1wUHJvdG9QAVoyZ29vZ2xlLmdvbGFuZy5vcmcvcHJvdG9idWYvdHlwZXMva25vd24vdGltZXN0YW1wcGL4AQGiAgNHUEKqAh5Hb29nbGUuUHJvdG9idWYuV2VsbEtub3duVHlwZXNiBnByb3RvMwrSAQoaZm94Z2xvdmUvUG9zZUluRnJhbWUucHJvdG8SCGZveGdsb3ZlGhNmb3hnbG92ZS9Qb3NlLnByb3RvGh9nb29nbGUvcHJvdG9idWYvdGltZXN0YW1wLnByb3RvImwKC1Bvc2VJbkZyYW1lEi0KCXRpbWVzdGFtcBgBIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASEAoIZnJhbWVfaWQYAiABKAkSHAoEcG9zZRgDIAEoCzIOLmZveGdsb3ZlLlBvc2ViBnByb3RvMw==", - "base64", - ), - ); - - const poseInFrame = poseInFrameChannel.deserialize( - Buffer.from( - // cspell:disable-next-line - "CgwIx8LcoQYQuJzV6wESA2ZvbxooChsJAAAAAAAA8D8RAAAAAAAAAEAZAAAAAAAACEASCSEAAAAAAADwPw==", - "base64", - ), - ); - expect(poseInFrame).toMatchInlineSnapshot(` - { - "frame_id": "foo", - "pose": { - "orientation": { - "w": 1, - "x": 0, - "y": 0, - "z": 0, - }, - "position": { - "x": 1, - "y": 2, - "z": 3, - }, - }, - "timestamp": { - "nsec": 494227000, - "sec": 1681334599, - }, - } - `); - - expect(() => - poseInFrameChannel.deserialize( - Buffer.from( - // cspell:disable-next-line - "CgsIgICAgICAgBAQARIDZm9vGigKGwkAAAAAAADwPxEAAAAAAAAAQBkAAAAAAAAIQBIJIQAAAAAAAPA/", - "base64", - ), - ), - ).toThrow( - "Timestamps with seconds greater than 2^53-1 are not supported (found seconds=9007199254740992, nanos=1)", - ); - - const sceneUpdateChannel = parseProtobufSchema( - "foxglove.SceneUpdate", - Buffer.from( - // cspell:disable-next-line - "Cl0KFGZveGdsb3ZlL0NvbG9yLnByb3RvEghmb3hnbG92ZSIzCgVDb2xvchIJCgFyGAEgASgBEgkKAWcYAiABKAESCQoBYhgDIAEoARIJCgFhGAQgASgBYgZwcm90bzMKZwoZZm94Z2xvdmUvUXVhdGVybmlvbi5wcm90bxIIZm94Z2xvdmUiOAoKUXVhdGVybmlvbhIJCgF4GAEgASgBEgkKAXkYAiABKAESCQoBehgDIAEoARIJCgF3GAQgASgBYgZwcm90bzMKVgoWZm94Z2xvdmUvVmVjdG9yMy5wcm90bxIIZm94Z2xvdmUiKgoHVmVjdG9yMxIJCgF4GAEgASgBEgkKAXkYAiABKAESCQoBehgDIAEoAWIGcHJvdG8zCrIBChNmb3hnbG92ZS9Qb3NlLnByb3RvEghmb3hnbG92ZRoZZm94Z2xvdmUvUXVhdGVybmlvbi5wcm90bxoWZm94Z2xvdmUvVmVjdG9yMy5wcm90byJWCgRQb3NlEiMKCHBvc2l0aW9uGAEgASgLMhEuZm94Z2xvdmUuVmVjdG9yMxIpCgtvcmllbnRhdGlvbhgCIAEoCzIULmZveGdsb3ZlLlF1YXRlcm5pb25iBnByb3RvMwqHAgodZm94Z2xvdmUvQXJyb3dQcmltaXRpdmUucHJvdG8SCGZveGdsb3ZlGhRmb3hnbG92ZS9Db2xvci5wcm90bxoTZm94Z2xvdmUvUG9zZS5wcm90byKoAQoOQXJyb3dQcmltaXRpdmUSHAoEcG9zZRgBIAEoCzIOLmZveGdsb3ZlLlBvc2USFAoMc2hhZnRfbGVuZ3RoGAIgASgBEhYKDnNoYWZ0X2RpYW1ldGVyGAMgASgBEhMKC2hlYWRfbGVuZ3RoGAQgASgBEhUKDWhlYWRfZGlhbWV0ZXIYBSABKAESHgoFY29sb3IYBiABKAsyDy5mb3hnbG92ZS5Db2xvcmIGcHJvdG8zCuMBChxmb3hnbG92ZS9DdWJlUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8ibgoNQ3ViZVByaW1pdGl2ZRIcCgRwb3NlGAEgASgLMg4uZm94Z2xvdmUuUG9zZRIfCgRzaXplGAIgASgLMhEuZm94Z2xvdmUuVmVjdG9yMxIeCgVjb2xvchgDIAEoCzIPLmZveGdsb3ZlLkNvbG9yYgZwcm90bzMKlQIKIGZveGdsb3ZlL0N5bGluZGVyUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8imwEKEUN5bGluZGVyUHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEh8KBHNpemUYAiABKAsyES5mb3hnbG92ZS5WZWN0b3IzEhQKDGJvdHRvbV9zY2FsZRgDIAEoARIRCgl0b3Bfc2NhbGUYBCABKAESHgoFY29sb3IYBSABKAsyDy5mb3hnbG92ZS5Db2xvcmIGcHJvdG8zClsKG2ZveGdsb3ZlL0tleVZhbHVlUGFpci5wcm90bxIIZm94Z2xvdmUiKgoMS2V5VmFsdWVQYWlyEgsKA2tleRgBIAEoCRINCgV2YWx1ZRgCIAEoCWIGcHJvdG8zClQKFWZveGdsb3ZlL1BvaW50My5wcm90bxIIZm94Z2xvdmUiKQoGUG9pbnQzEgkKAXgYASABKAESCQoBeRgCIAEoARIJCgF6GAMgASgBYgZwcm90bzMKpAMKHGZveGdsb3ZlL0xpbmVQcmltaXRpdmUucHJvdG8SCGZveGdsb3ZlGhRmb3hnbG92ZS9Db2xvci5wcm90bxoVZm94Z2xvdmUvUG9pbnQzLnByb3RvGhNmb3hnbG92ZS9Qb3NlLnByb3RvIq8CCg1MaW5lUHJpbWl0aXZlEioKBHR5cGUYASABKA4yHC5mb3hnbG92ZS5MaW5lUHJpbWl0aXZlLlR5cGUSHAoEcG9zZRgCIAEoCzIOLmZveGdsb3ZlLlBvc2USEQoJdGhpY2tuZXNzGAMgASgBEhcKD3NjYWxlX2ludmFyaWFudBgEIAEoCBIgCgZwb2ludHMYBSADKAsyEC5mb3hnbG92ZS5Qb2ludDMSHgoFY29sb3IYBiABKAsyDy5mb3hnbG92ZS5Db2xvchIfCgZjb2xvcnMYByADKAsyDy5mb3hnbG92ZS5Db2xvchIPCgdpbmRpY2VzGAggAygHIjQKBFR5cGUSDgoKTElORV9TVFJJUBAAEg0KCUxJTkVfTE9PUBABEg0KCUxJTkVfTElTVBACYgZwcm90bzMKrgIKHWZveGdsb3ZlL01vZGVsUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8itwEKDk1vZGVsUHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEiAKBXNjYWxlGAIgASgLMhEuZm94Z2xvdmUuVmVjdG9yMxIeCgVjb2xvchgDIAEoCzIPLmZveGdsb3ZlLkNvbG9yEhYKDm92ZXJyaWRlX2NvbG9yGAQgASgIEgsKA3VybBgFIAEoCRISCgptZWRpYV90eXBlGAYgASgJEgwKBGRhdGEYByABKAxiBnByb3RvMwrnAQoeZm94Z2xvdmUvU3BoZXJlUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8icAoPU3BoZXJlUHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEh8KBHNpemUYAiABKAsyES5mb3hnbG92ZS5WZWN0b3IzEh4KBWNvbG9yGAMgASgLMg8uZm94Z2xvdmUuQ29sb3JiBnByb3RvMwr4AQocZm94Z2xvdmUvVGV4dFByaW1pdGl2ZS5wcm90bxIIZm94Z2xvdmUaFGZveGdsb3ZlL0NvbG9yLnByb3RvGhNmb3hnbG92ZS9Qb3NlLnByb3RvIpoBCg1UZXh0UHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEhEKCWJpbGxib2FyZBgCIAEoCBIRCglmb250X3NpemUYAyABKAESFwoPc2NhbGVfaW52YXJpYW50GAQgASgIEh4KBWNvbG9yGAUgASgLMg8uZm94Z2xvdmUuQ29sb3ISDAoEdGV4dBgGIAEoCWIGcHJvdG8zCqYCCiRmb3hnbG92ZS9UcmlhbmdsZUxpc3RQcmltaXRpdmUucHJvdG8SCGZveGdsb3ZlGhRmb3hnbG92ZS9Db2xvci5wcm90bxoVZm94Z2xvdmUvUG9pbnQzLnByb3RvGhNmb3hnbG92ZS9Qb3NlLnByb3RvIqkBChVUcmlhbmdsZUxpc3RQcmltaXRpdmUSHAoEcG9zZRgBIAEoCzIOLmZveGdsb3ZlLlBvc2USIAoGcG9pbnRzGAIgAygLMhAuZm94Z2xvdmUuUG9pbnQzEh4KBWNvbG9yGAMgASgLMg8uZm94Z2xvdmUuQ29sb3ISHwoGY29sb3JzGAQgAygLMg8uZm94Z2xvdmUuQ29sb3ISDwoHaW5kaWNlcxgFIAMoB2IGcHJvdG8zCusBCh5nb29nbGUvcHJvdG9idWYvZHVyYXRpb24ucHJvdG8SD2dvb2dsZS5wcm90b2J1ZiIqCghEdXJhdGlvbhIPCgdzZWNvbmRzGAEgASgDEg0KBW5hbm9zGAIgASgFQoMBChNjb20uZ29vZ2xlLnByb3RvYnVmQg1EdXJhdGlvblByb3RvUAFaMWdvb2dsZS5nb2xhbmcub3JnL3Byb3RvYnVmL3R5cGVzL2tub3duL2R1cmF0aW9ucGL4AQGiAgNHUEKqAh5Hb29nbGUuUHJvdG9idWYuV2VsbEtub3duVHlwZXNiBnByb3RvMwrvAQofZ29vZ2xlL3Byb3RvYnVmL3RpbWVzdGFtcC5wcm90bxIPZ29vZ2xlLnByb3RvYnVmIisKCVRpbWVzdGFtcBIPCgdzZWNvbmRzGAEgASgDEg0KBW5hbm9zGAIgASgFQoUBChNjb20uZ29vZ2xlLnByb3RvYnVmQg5UaW1lc3RhbXBQcm90b1ABWjJnb29nbGUuZ29sYW5nLm9yZy9wcm90b2J1Zi90eXBlcy9rbm93bi90aW1lc3RhbXBwYvgBAaICA0dQQqoCHkdvb2dsZS5Qcm90b2J1Zi5XZWxsS25vd25UeXBlc2IGcHJvdG8zCrIHChpmb3hnbG92ZS9TY2VuZUVudGl0eS5wcm90bxIIZm94Z2xvdmUaHWZveGdsb3ZlL0Fycm93UHJpbWl0aXZlLnByb3RvGhxmb3hnbG92ZS9DdWJlUHJpbWl0aXZlLnByb3RvGiBmb3hnbG92ZS9DeWxpbmRlclByaW1pdGl2ZS5wcm90bxobZm94Z2xvdmUvS2V5VmFsdWVQYWlyLnByb3RvGhxmb3hnbG92ZS9MaW5lUHJpbWl0aXZlLnByb3RvGh1mb3hnbG92ZS9Nb2RlbFByaW1pdGl2ZS5wcm90bxoeZm94Z2xvdmUvU3BoZXJlUHJpbWl0aXZlLnByb3RvGhxmb3hnbG92ZS9UZXh0UHJpbWl0aXZlLnByb3RvGiRmb3hnbG92ZS9UcmlhbmdsZUxpc3RQcmltaXRpdmUucHJvdG8aHmdvb2dsZS9wcm90b2J1Zi9kdXJhdGlvbi5wcm90bxofZ29vZ2xlL3Byb3RvYnVmL3RpbWVzdGFtcC5wcm90byKjBAoLU2NlbmVFbnRpdHkSLQoJdGltZXN0YW1wGAEgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIQCghmcmFtZV9pZBgCIAEoCRIKCgJpZBgDIAEoCRIrCghsaWZldGltZRgEIAEoCzIZLmdvb2dsZS5wcm90b2J1Zi5EdXJhdGlvbhIUCgxmcmFtZV9sb2NrZWQYBSABKAgSKAoIbWV0YWRhdGEYBiADKAsyFi5mb3hnbG92ZS5LZXlWYWx1ZVBhaXISKAoGYXJyb3dzGAcgAygLMhguZm94Z2xvdmUuQXJyb3dQcmltaXRpdmUSJgoFY3ViZXMYCCADKAsyFy5mb3hnbG92ZS5DdWJlUHJpbWl0aXZlEioKB3NwaGVyZXMYCSADKAsyGS5mb3hnbG92ZS5TcGhlcmVQcmltaXRpdmUSLgoJY3lsaW5kZXJzGAogAygLMhsuZm94Z2xvdmUuQ3lsaW5kZXJQcmltaXRpdmUSJgoFbGluZXMYCyADKAsyFy5mb3hnbG92ZS5MaW5lUHJpbWl0aXZlEjIKCXRyaWFuZ2xlcxgMIAMoCzIfLmZveGdsb3ZlLlRyaWFuZ2xlTGlzdFByaW1pdGl2ZRImCgV0ZXh0cxgNIAMoCzIXLmZveGdsb3ZlLlRleHRQcmltaXRpdmUSKAoGbW9kZWxzGA4gAygLMhguZm94Z2xvdmUuTW9kZWxQcmltaXRpdmViBnByb3RvMwr+AQoiZm94Z2xvdmUvU2NlbmVFbnRpdHlEZWxldGlvbi5wcm90bxIIZm94Z2xvdmUaH2dvb2dsZS9wcm90b2J1Zi90aW1lc3RhbXAucHJvdG8ipAEKE1NjZW5lRW50aXR5RGVsZXRpb24SLQoJdGltZXN0YW1wGAEgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIwCgR0eXBlGAIgASgOMiIuZm94Z2xvdmUuU2NlbmVFbnRpdHlEZWxldGlvbi5UeXBlEgoKAmlkGAMgASgJIiAKBFR5cGUSDwoLTUFUQ0hJTkdfSUQQABIHCgNBTEwQAWIGcHJvdG8zCtgBChpmb3hnbG92ZS9TY2VuZVVwZGF0ZS5wcm90bxIIZm94Z2xvdmUaGmZveGdsb3ZlL1NjZW5lRW50aXR5LnByb3RvGiJmb3hnbG92ZS9TY2VuZUVudGl0eURlbGV0aW9uLnByb3RvImgKC1NjZW5lVXBkYXRlEjAKCWRlbGV0aW9ucxgBIAMoCzIdLmZveGdsb3ZlLlNjZW5lRW50aXR5RGVsZXRpb24SJwoIZW50aXRpZXMYAiADKAsyFS5mb3hnbG92ZS5TY2VuZUVudGl0eWIGcHJvdG8z", - "base64", - ), - ); - - const sceneUpdate = sceneUpdateChannel.deserialize( - // cspell:disable-next-line - Buffer.from("EhwKDAjHwtyhBhC4nNXrASIMCMfC3KEGELic1esB", "base64"), - ); - expect(sceneUpdate).toMatchInlineSnapshot(` - { - "deletions": [], - "entities": [ - { - "arrows": [], - "cubes": [], - "cylinders": [], - "frame_id": "", - "frame_locked": false, - "id": "", - "lifetime": { - "nsec": 494227000, - "sec": 1681334599, - }, - "lines": [], - "metadata": [], - "models": [], - "spheres": [], - "texts": [], - "timestamp": { - "nsec": 494227000, - "sec": 1681334599, - }, - "triangles": [], - }, - ], - } - `); - - expect(sceneUpdateChannel.datatypes.get("google.protobuf.Timestamp")).toMatchInlineSnapshot(` - { - "definitions": [ - { - "isArray": false, - "name": "sec", - "type": "int32", - }, - { - "isArray": false, - "name": "nsec", - "type": "int32", - }, - ], - } - `); - expect(sceneUpdateChannel.datatypes.get("google.protobuf.Duration")).toMatchInlineSnapshot(` - { - "definitions": [ - { - "isArray": false, - "name": "sec", - "type": "int32", - }, - { - "isArray": false, - "name": "nsec", - "type": "int32", - }, - ], - } - `); - - // Duration too large - expect(() => - // cspell:disable-next-line - sceneUpdateChannel.deserialize(Buffer.from("EhMKBAgCEAMiCwiAgICAgICAEBAB", "base64")), - ).toThrow( - "Timestamps with seconds greater than 2^53-1 are not supported (found seconds=9007199254740992, nanos=1)", - ); - - // Timestamp too large - expect(() => - // cspell:disable-next-line - sceneUpdateChannel.deserialize(Buffer.from("EhMKCwiAgICAgICAEBABIgQIAhAD", "base64")), - ).toThrow( - "Timestamps with seconds greater than 2^53-1 are not supported (found seconds=9007199254740992, nanos=1)", - ); - }); }); diff --git a/typescript/support/src/parseProtobufSchema.ts b/typescript/support/src/parseProtobufSchema.ts index cc312c67a8..bd8d8dfd4d 100644 --- a/typescript/support/src/parseProtobufSchema.ts +++ b/typescript/support/src/parseProtobufSchema.ts @@ -4,6 +4,22 @@ import { FileDescriptorSet } from "@foxglove/protobufjs/ext/descriptor"; import { protobufDefinitionsToDatatypes, stripLeadingDot } from "./protobufDefinitionsToDatatypes"; import { MessageDefinitionMap } from "./types"; +export type ParseProtobufSchemaOptions = { + /** + * A function that will be called with the root type after parsing the FileDescriptorSet. Used by + * Foxglove Studio to modify the deserialization behavior of google.protobuf.Timestamp & + * google.protobuf.Duration. + */ + processRootType?: (rootType: protobufjs.Type) => protobufjs.Type; + + /** + * A function that will be called after producing message definitions from the schema. Used by + * Foxglove Studio to modify the field name of google.protobuf.Timestamp & + * google.protobuf.Duration. + */ + processMessageDefinitions?: (definitions: MessageDefinitionMap) => MessageDefinitionMap; +}; + /** * Parse a Protobuf binary schema (FileDescriptorSet) and produce datatypes and a deserializer * function. @@ -11,6 +27,7 @@ import { MessageDefinitionMap } from "./types"; export function parseProtobufSchema( schemaName: string, schemaData: Uint8Array, + options?: ParseProtobufSchemaOptions, ): { datatypes: MessageDefinitionMap; deserialize: (buffer: ArrayBufferView) => unknown; @@ -19,38 +36,10 @@ export function parseProtobufSchema( const root = protobufjs.Root.fromDescriptor(descriptorSet); root.resolveAll(); - const rootType = root.lookupType(schemaName); - - // Modify the definition of google.protobuf.Timestamp and Duration so they are interpreted as - // {sec: number, nsec: number}, compatible with the rest of Studio. The standard Protobuf types - // use different names (`seconds` and `nanos`), and `seconds` is an `int64`, which would be - // deserialized as a bigint by default. - // - // protobufDefinitionsToDatatypes also has matching logic to rename the fields. - const fixTimeType = (type: protobufjs.ReflectionObject | null) => { - if (!type || !(type instanceof protobufjs.Type)) { - return; - } - type.setup(); // ensure the original optimized toObject has been created - const prevToObject = type.toObject; // eslint-disable-line @typescript-eslint/unbound-method - const newToObject: typeof prevToObject = (message, options) => { - const result = prevToObject.call(type, message, options); - const { seconds, nanos } = result as { seconds: bigint; nanos: number }; - if (typeof seconds !== "bigint" || typeof nanos !== "number") { - return result; - } - if (seconds > BigInt(Number.MAX_SAFE_INTEGER)) { - throw new Error( - `Timestamps with seconds greater than 2^53-1 are not supported (found seconds=${seconds}, nanos=${nanos})`, - ); - } - return { sec: Number(seconds), nsec: nanos }; - }; - type.toObject = newToObject; - }; - - fixTimeType(root.lookup(".google.protobuf.Timestamp")); - fixTimeType(root.lookup(".google.protobuf.Duration")); + let rootType = root.lookupType(schemaName); + if (options?.processRootType) { + rootType = options.processRootType(rootType); + } const deserialize = (data: ArrayBufferView) => { return rootType.toObject( @@ -59,8 +48,11 @@ export function parseProtobufSchema( ); }; - const datatypes: MessageDefinitionMap = new Map(); + let datatypes: MessageDefinitionMap = new Map(); protobufDefinitionsToDatatypes(datatypes, rootType); + if (options?.processMessageDefinitions) { + datatypes = options.processMessageDefinitions(datatypes); + } if (!datatypes.has(schemaName)) { throw new Error( diff --git a/typescript/support/src/protobufDefinitionsToDatatypes.ts b/typescript/support/src/protobufDefinitionsToDatatypes.ts index 4eea790bdd..9b48da07e0 100644 --- a/typescript/support/src/protobufDefinitionsToDatatypes.ts +++ b/typescript/support/src/protobufDefinitionsToDatatypes.ts @@ -45,8 +45,8 @@ export function protobufDefinitionsToDatatypes( for (const field of type.fieldsArray) { if (field.resolvedType instanceof protobufjs.Enum) { for (const [name, value] of Object.entries(field.resolvedType.values)) { - // Note: names from different enums might conflict. The player API will need to be updated - // to associate fields with enums (similar to the __foxglove_enum annotation hack). + // Note: names from different enums might conflict. The @foxglove/message-definition API + // will need to be updated to associate fields with enums. // https://github.com/foxglove/studio/issues/2214 definitions.push({ name, type: "int32", isConstant: true, value }); } @@ -70,15 +70,6 @@ export function protobufDefinitionsToDatatypes( throw new Error("Repeated bytes are not currently supported"); } definitions.push({ type: "uint8", name: field.name, isArray: true }); - } else if ( - type.fullName === ".google.protobuf.Timestamp" || - type.fullName === ".google.protobuf.Duration" - ) { - definitions.push({ - type: "int32", - name: field.name === "seconds" ? "sec" : "nsec", - isArray: field.repeated, - }); } else { definitions.push({ type: protobufScalarToRosPrimitive(field.type), From 811f45e1d64f3e5ed006ad26163ebac3ea92048e Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Tue, 12 Sep 2023 17:52:34 -0700 Subject: [PATCH 17/27] exports & comments --- typescript/support/src/index.ts | 6 ++++-- typescript/support/src/parseChannel.ts | 6 ++++++ typescript/support/src/parseProtobufSchema.ts | 3 +++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/typescript/support/src/index.ts b/typescript/support/src/index.ts index c8aee4c069..3ab38c8fa3 100644 --- a/typescript/support/src/index.ts +++ b/typescript/support/src/index.ts @@ -1,5 +1,7 @@ +export * from "./decompressHandlers"; +export * from "./parseChannel"; +export * from "./parseFlatbufferSchema"; export * from "./parseJsonSchema"; +export * from "./parseProtobufSchema"; export * from "./protobufDefinitionsToDatatypes"; export * from "./protobufDescriptors"; -export * from "./parseChannel"; -export * from "./decompressHandlers"; diff --git a/typescript/support/src/parseChannel.ts b/typescript/support/src/parseChannel.ts index 214caaadb8..f6d9912ad0 100644 --- a/typescript/support/src/parseChannel.ts +++ b/typescript/support/src/parseChannel.ts @@ -64,7 +64,13 @@ function parsedDefinitionsToDatatypes( return datatypes; } +/** + * Options to configure the behavior of {@link parseChannel}. + */ export type ParseChannelOptions = { + /** + * Options to configure the behavior of {@link parseProtobufSchema}. + */ protobuf?: ParseProtobufSchemaOptions; }; diff --git a/typescript/support/src/parseProtobufSchema.ts b/typescript/support/src/parseProtobufSchema.ts index bd8d8dfd4d..1e93d57062 100644 --- a/typescript/support/src/parseProtobufSchema.ts +++ b/typescript/support/src/parseProtobufSchema.ts @@ -4,6 +4,9 @@ import { FileDescriptorSet } from "@foxglove/protobufjs/ext/descriptor"; import { protobufDefinitionsToDatatypes, stripLeadingDot } from "./protobufDefinitionsToDatatypes"; import { MessageDefinitionMap } from "./types"; +/** + * Options to configure the behavior of {@link parseProtobufSchema}. + */ export type ParseProtobufSchemaOptions = { /** * A function that will be called with the root type after parsing the FileDescriptorSet. Used by From 80b98d6e592948df27a9a1325b28e040f952646c Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Tue, 12 Sep 2023 17:53:44 -0700 Subject: [PATCH 18/27] readme update --- typescript/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/typescript/README.md b/typescript/README.md index 475dc8fe08..e2b8e6469d 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -7,3 +7,4 @@ The following NPM packages are provided for use with JavaScript and TypeScript: - **@mcap/core** – low-level readers and writers - **@mcap/support** – support for well-known compression formats and encodings - **@mcap/nodejs** – support for Node.js environment +- **@mcap/browser** – support for browser environment From 350e423d27cb5fadc1ac04b57e7a0547b85a5c06 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Tue, 12 Sep 2023 17:56:09 -0700 Subject: [PATCH 19/27] update comment to remove RosDatatypes reference --- typescript/support/src/parseChannel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typescript/support/src/parseChannel.ts b/typescript/support/src/parseChannel.ts index f6d9912ad0..8381f8fed4 100644 --- a/typescript/support/src/parseChannel.ts +++ b/typescript/support/src/parseChannel.ts @@ -76,7 +76,7 @@ export type ParseChannelOptions = { /** * Process a channel/schema and extract information that can be used to deserialize messages on the - * channel, and schemas in the format expected by Studio's RosDatatypes. + * channel, and schemas in the format compatible with `@foxglove/message-definition`. * * See: * - https://github.com/foxglove/mcap/blob/main/docs/specification/well-known-message-encodings.md From c95de40593a624f8427d34bf1b64bab204d33df2 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Tue, 12 Sep 2023 18:32:46 -0700 Subject: [PATCH 20/27] update (disabled) publish workflow --- .github/workflows/ci.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e168e06934..13b04a7e7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -224,15 +224,21 @@ jobs: YARN_NPM_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} # To be enabled once API has settled + # - name: Publish @mcap/support to NPM + # if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/support/v') }} + # run: yarn workspace @mcap/support publish --access public + # env: + # NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + # - name: Publish @mcap/nodejs to NPM # if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/nodejs/v') }} # run: yarn workspace @mcap/nodejs publish --access public # env: # NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} - # - name: Publish @mcap/support to NPM - # if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/support/v') }} - # run: yarn workspace @mcap/support publish --access public + # - name: Publish @mcap/browser to NPM + # if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/browser/v') }} + # run: yarn workspace @mcap/browser publish --access public # env: # NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} From d6b5b390c529514ea1c258cde06db3d6f6f56f28 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Tue, 12 Sep 2023 18:37:28 -0700 Subject: [PATCH 21/27] more CI & package.json updates --- .github/workflows/ci.yml | 6 ++++-- package.json | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13b04a7e7f..e0d2788ed6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -211,10 +211,12 @@ jobs: - run: yarn prettier:check - run: yarn workspace @mcap/core lint:ci - run: yarn workspace @mcap/core build - - run: yarn workspace @mcap/nodejs lint:ci - - run: yarn workspace @mcap/nodejs build - run: yarn workspace @mcap/support lint:ci - run: yarn workspace @mcap/support build + - run: yarn workspace @mcap/nodejs lint:ci + - run: yarn workspace @mcap/nodejs build + - run: yarn workspace @mcap/browser lint:ci + - run: yarn workspace @mcap/browser build - run: yarn typescript:test - name: Publish @mcap/core to NPM diff --git a/package.json b/package.json index 6c406c08e9..143a80e364 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,8 @@ "start": "yarn workspace website start", "spellcheck": "cspell --relative '**'", "typescript:test": "yarn jest --config typescript/jest.config.json", - "typescript:build": "yarn workspace @mcap/core build && yarn workspace @mcap/support build && yarn workspace @mcap/nodejs build", - "typescript:clean": "yarn workspace @mcap/core build --clean && yarn workspace @mcap/support build --clean && yarn workspace @mcap/nodejs build --clean", + "typescript:build": "yarn workspace @mcap/core build && yarn workspace @mcap/support build && yarn workspace @mcap/nodejs build && yarn workspace @mcap/browser build", + "typescript:clean": "yarn workspace @mcap/core build --clean && yarn workspace @mcap/support build --clean && yarn workspace @mcap/nodejs build --clean && yarn workspace @mcap/browser build --clean", "test:conformance:generate-inputs": "yarn workspace @foxglove/mcap-conformance generate-inputs --data-dir \"$(pwd)/tests/conformance/data\"", "test:conformance": "yarn workspace @foxglove/mcap-conformance run-tests --data-dir \"$(pwd)/tests/conformance/data\"" }, From fb906704fba3c6e18aedbca263e426406582e154 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 17 Nov 2023 18:30:57 -0800 Subject: [PATCH 22/27] remove parseChannel stuff --- .vscode/extensions.json | 3 +- .vscode/settings.json | 8 +- typescript/examples/bag2mcap/package.json | 2 +- typescript/examples/validate/package.json | 6 +- .../examples/validate/scripts/validate.ts | 70 +++- typescript/support/package.json | 4 +- .../support/src/fixtures/ByteVector.bfbs | 3 - .../support/src/fixtures/byte-vector.ts | 91 ---- .../support/src/fixtures/reflection.bfbs | 3 - typescript/support/src/index.ts | 5 - typescript/support/src/parseChannel.test.ts | 124 ------ typescript/support/src/parseChannel.ts | 201 --------- .../support/src/parseFlatbufferSchema.test.ts | 389 ------------------ .../support/src/parseFlatbufferSchema.ts | 208 ---------- .../support/src/parseJsonSchema.test.ts | 275 ------------- typescript/support/src/parseJsonSchema.ts | 185 --------- .../support/src/parseProtobufSchema.test.ts | 176 -------- typescript/support/src/parseProtobufSchema.ts | 69 ---- .../src/protobufDefinitionsToDatatypes.ts | 81 ---- typescript/support/src/protobufDescriptors.ts | 4 +- typescript/support/src/types.ts | 4 - typescript/support/typings/protobufjs.d.ts | 6 +- website/package.json | 2 +- yarn.lock | 60 +-- 24 files changed, 87 insertions(+), 1892 deletions(-) delete mode 100644 typescript/support/src/fixtures/ByteVector.bfbs delete mode 100644 typescript/support/src/fixtures/byte-vector.ts delete mode 100644 typescript/support/src/fixtures/reflection.bfbs delete mode 100644 typescript/support/src/parseChannel.test.ts delete mode 100644 typescript/support/src/parseChannel.ts delete mode 100644 typescript/support/src/parseFlatbufferSchema.test.ts delete mode 100644 typescript/support/src/parseFlatbufferSchema.ts delete mode 100644 typescript/support/src/parseJsonSchema.test.ts delete mode 100644 typescript/support/src/parseJsonSchema.ts delete mode 100644 typescript/support/src/parseProtobufSchema.test.ts delete mode 100644 typescript/support/src/parseProtobufSchema.ts delete mode 100644 typescript/support/src/protobufDefinitionsToDatatypes.ts delete mode 100644 typescript/support/src/types.ts diff --git a/.vscode/extensions.json b/.vscode/extensions.json index bb94d4d388..6741286f2e 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -6,6 +6,7 @@ "orta.vscode-jest", "ms-vscode.cpptools-extension-pack", "ms-vscode-remote.vscode-remote-extensionpack", - "streetsidesoftware.code-spell-checker" + "streetsidesoftware.code-spell-checker", + "ms-python.flake8" ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 36ac11b5c4..141330e2a1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -27,17 +27,17 @@ "**/dist": true }, - "python.formatting.provider": "black", + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter" + }, "python.analysis.typeCheckingMode": "strict", - "python.linting.flake8Enabled": true, - "python.linting.enabled": true, - "python.linting.flake8Args": ["--config", "python/.flake8"], "python.analysis.extraPaths": [ "./python/mcap", "./python/mcap-protobuf-support", "./python/mcap-ros1-support", "./python/mcap-ros2-support" ], + "flake8.args": ["--config", "python/.flake8"], // https://github.com/microsoft/vscode-cpptools/issues/722 "C_Cpp.autoAddFileAssociations": false, diff --git a/typescript/examples/bag2mcap/package.json b/typescript/examples/bag2mcap/package.json index 9b5c13d5f5..44acec6e6b 100644 --- a/typescript/examples/bag2mcap/package.json +++ b/typescript/examples/bag2mcap/package.json @@ -45,7 +45,7 @@ "eslint-plugin-prettier": "5.0.1", "lodash": "4.17.21", "prettier": "3.1.0", - "protobufjs": "7.2.2", + "protobufjs": "7.2.5", "ts-node": "10.9.1", "tsconfig-paths": "4.1.2", "typescript": "5.2.2" diff --git a/typescript/examples/validate/package.json b/typescript/examples/validate/package.json index 74c997da48..f43ebac538 100644 --- a/typescript/examples/validate/package.json +++ b/typescript/examples/validate/package.json @@ -29,7 +29,9 @@ "@foxglove/wasm-bz2": "0.1.1", "@foxglove/wasm-lz4": "1.0.2", "@foxglove/wasm-zstd": "1.0.1", - "@mcap/core": "*", + "@mcap/core": "workspace:*", + "@mcap/nodejs": "workspace:*", + "@mcap/support": "workspace:*", "@types/jest": "29.5.8", "@types/lodash": "4.14.191", "@types/node": "18.13.0", @@ -46,7 +48,7 @@ "jest": "29.7.0", "lodash": "4.17.21", "prettier": "3.1.0", - "protobufjs": "7.2.2", + "protobufjs": "7.2.5", "ts-jest": "29.1.1", "ts-node": "10.9.1", "tsconfig-paths": "4.1.2", diff --git a/typescript/examples/validate/scripts/validate.ts b/typescript/examples/validate/scripts/validate.ts index 7d93c03a28..150186f794 100644 --- a/typescript/examples/validate/scripts/validate.ts +++ b/typescript/examples/validate/scripts/validate.ts @@ -1,3 +1,6 @@ +import { parse as parseMessageDefinition } from "@foxglove/rosmsg"; +import { LazyMessageReader as ROS1LazyMessageReader } from "@foxglove/rosmsg-serialization"; +import { MessageReader as ROS2MessageReader } from "@foxglove/rosmsg2-serialization"; import { hasMcapPrefix, McapConstants, @@ -6,13 +9,16 @@ import { McapTypes, } from "@mcap/core"; import { FileHandleReadable } from "@mcap/nodejs"; -import { loadDecompressHandlers, parseChannel, ParsedChannel } from "@mcap/support"; +import { loadDecompressHandlers } from "@mcap/support"; import { program } from "commander"; import { createReadStream } from "fs"; import fs from "fs/promises"; import { isEqual } from "lodash"; import { performance } from "perf_hooks"; +import protobufjs from "protobufjs"; +import { FileDescriptorSet } from "protobufjs/ext/descriptor"; +type Channel = McapTypes.Channel; type TypedMcapRecord = McapTypes.TypedMcapRecord; function log(...data: unknown[]) { @@ -90,7 +96,10 @@ async function validate( const schemasById = new Map(); const channelInfoById = new Map< number, - { info: McapTypes.Channel; parsedChannel: ParsedChannel } + { + info: Channel; + messageDeserializer?: (data: ArrayBufferView) => unknown; + } >(); function processRecord(record: TypedMcapRecord) { @@ -120,17 +129,47 @@ async function validate( } break; } - let schema: McapTypes.Schema | undefined; - if (record.schemaId !== 0) { - schema = schemasById.get(record.schemaId); - if (!schema) { - throw new Error(`Missing schema ${record.schemaId} for channel ${record.id}`); - } + if (record.schemaId === 0) { + throw new Error( + `Channel ${record.id} has no schema; channels without schemas are not supported`, + ); + } + const schema = schemasById.get(record.schemaId); + if (!schema) { + throw new Error(`Missing schema ${record.schemaId} for channel ${record.id}`); } - channelInfoById.set(record.id, { - info: record, - parsedChannel: parseChannel({ schema, messageEncoding: record.messageEncoding }), - }); + let messageDeserializer: (data: ArrayBufferView) => unknown; + if (record.messageEncoding === "ros1") { + const reader = new ROS1LazyMessageReader( + parseMessageDefinition(new TextDecoder().decode(schema.data)), + ); + messageDeserializer = (data) => { + const size = reader.size(data); + if (size !== data.byteLength) { + throw new Error(`Message size ${size} should match buffer length ${data.byteLength}`); + } + return reader.readMessage(data).toJSON(); + }; + } else if (record.messageEncoding === "ros2") { + const reader = new ROS2MessageReader( + parseMessageDefinition(new TextDecoder().decode(schema.data), { + ros2: true, + }), + ); + messageDeserializer = (data) => reader.readMessage(data); + } else if (record.messageEncoding === "protobuf") { + const root = protobufjs.Root.fromDescriptor(FileDescriptorSet.decode(schema.data)); + const type = root.lookupType(schema.name); + + messageDeserializer = (data) => + type.decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength)); + } else if (record.messageEncoding === "json") { + const textDecoder = new TextDecoder(); + messageDeserializer = (data) => JSON.parse(textDecoder.decode(data)); + } else { + throw new Error(`unsupported encoding ${record.messageEncoding}`); + } + channelInfoById.set(record.id, { info: record, messageDeserializer }); break; } @@ -140,7 +179,12 @@ async function validate( throw new Error(`message for channel ${record.channelId} with no prior channel info`); } if (deserialize) { - const message = channelInfo.parsedChannel.deserialize(record.data); + if (channelInfo.messageDeserializer == undefined) { + throw new Error( + `No deserializer available for channel id: ${channelInfo.info.id} ${channelInfo.info.messageEncoding}`, + ); + } + const message = channelInfo.messageDeserializer(record.data); if (dump) { log(message); } diff --git a/typescript/support/package.json b/typescript/support/package.json index b36e6c7675..a71afbbb6d 100644 --- a/typescript/support/package.json +++ b/typescript/support/package.json @@ -55,7 +55,6 @@ "@foxglove/message-definition": "^0.3.0", "@foxglove/omgidl-parser": "^1.0.0", "@foxglove/omgidl-serialization": "^1.0.0", - "@foxglove/protobufjs": "0.0.1-toobject-bigint.1", "@foxglove/ros2idl-parser": "^0.3.0", "@foxglove/rosmsg": "^4.2.2", "@foxglove/rosmsg-serialization": "^2.0.2", @@ -66,6 +65,7 @@ "@foxglove/wasm-zstd": "^1.0.1", "@protobufjs/base64": "^1.1.2", "flatbuffers": "^23.5.26", - "flatbuffers_reflection": "^0.0.7" + "flatbuffers_reflection": "^0.0.7", + "protobufjs": "7.2.5" } } diff --git a/typescript/support/src/fixtures/ByteVector.bfbs b/typescript/support/src/fixtures/ByteVector.bfbs deleted file mode 100644 index 9c57006f12..0000000000 --- a/typescript/support/src/fixtures/ByteVector.bfbs +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5b800c9b0754f148c25ebd86ba9457923a6dd1099d55e5e9f2bd064e85001ac2 -size 296 diff --git a/typescript/support/src/fixtures/byte-vector.ts b/typescript/support/src/fixtures/byte-vector.ts deleted file mode 100644 index 1abb175805..0000000000 --- a/typescript/support/src/fixtures/byte-vector.ts +++ /dev/null @@ -1,91 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify -/* eslint-disable */ -import * as flatbuffers from "flatbuffers"; - -export class ByteVector { - bb: flatbuffers.ByteBuffer | null = null; - bb_pos = 0; - __init(i: number, bb: flatbuffers.ByteBuffer): ByteVector { - this.bb_pos = i; - this.bb = bb; - return this; - } - - static getRootAsByteVector(bb: flatbuffers.ByteBuffer, obj?: ByteVector): ByteVector { - return (obj || new ByteVector()).__init(bb.readInt32(bb.position()) + bb.position(), bb); - } - - static getSizePrefixedRootAsByteVector(bb: flatbuffers.ByteBuffer, obj?: ByteVector): ByteVector { - bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); - return (obj || new ByteVector()).__init(bb.readInt32(bb.position()) + bb.position(), bb); - } - - data(index: number): number | null { - const offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? this.bb!.readUint8(this.bb!.__vector(this.bb_pos + offset) + index) : 0; - } - - dataLength(): number { - const offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; - } - - dataArray(): Uint8Array | null { - const offset = this.bb!.__offset(this.bb_pos, 4); - return offset - ? new Uint8Array( - this.bb!.bytes().buffer, - this.bb!.bytes().byteOffset + this.bb!.__vector(this.bb_pos + offset), - this.bb!.__vector_len(this.bb_pos + offset), - ) - : null; - } - - static startByteVector(builder: flatbuffers.Builder) { - builder.startObject(1); - } - - static addData(builder: flatbuffers.Builder, dataOffset: flatbuffers.Offset) { - builder.addFieldOffset(0, dataOffset, 0); - } - - static createDataVector( - builder: flatbuffers.Builder, - data: number[] | Uint8Array, - ): flatbuffers.Offset { - builder.startVector(1, data.length, 1); - for (let i = data.length - 1; i >= 0; i--) { - builder.addInt8(data[i]!); - } - return builder.endVector(); - } - - static startDataVector(builder: flatbuffers.Builder, numElems: number) { - builder.startVector(1, numElems, 1); - } - - static endByteVector(builder: flatbuffers.Builder): flatbuffers.Offset { - const offset = builder.endObject(); - return offset; - } - - static finishByteVectorBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset) { - builder.finish(offset); - } - - static finishSizePrefixedByteVectorBuffer( - builder: flatbuffers.Builder, - offset: flatbuffers.Offset, - ) { - builder.finish(offset, undefined, true); - } - - static createByteVector( - builder: flatbuffers.Builder, - dataOffset: flatbuffers.Offset, - ): flatbuffers.Offset { - ByteVector.startByteVector(builder); - ByteVector.addData(builder, dataOffset); - return ByteVector.endByteVector(builder); - } -} diff --git a/typescript/support/src/fixtures/reflection.bfbs b/typescript/support/src/fixtures/reflection.bfbs deleted file mode 100644 index f5b1e43d5a..0000000000 --- a/typescript/support/src/fixtures/reflection.bfbs +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8a345ca0456791076519c32a8c5f3a95b8fc8dd680efddc659a7bc723c01128f -size 5608 diff --git a/typescript/support/src/index.ts b/typescript/support/src/index.ts index 3ab38c8fa3..f469258d50 100644 --- a/typescript/support/src/index.ts +++ b/typescript/support/src/index.ts @@ -1,7 +1,2 @@ export * from "./decompressHandlers"; -export * from "./parseChannel"; -export * from "./parseFlatbufferSchema"; -export * from "./parseJsonSchema"; -export * from "./parseProtobufSchema"; -export * from "./protobufDefinitionsToDatatypes"; export * from "./protobufDescriptors"; diff --git a/typescript/support/src/parseChannel.test.ts b/typescript/support/src/parseChannel.test.ts deleted file mode 100644 index d5a513e771..0000000000 --- a/typescript/support/src/parseChannel.test.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { FileDescriptorSet, IFileDescriptorSet } from "@foxglove/protobufjs/ext/descriptor"; -import fs from "fs"; - -import { parseChannel } from "./parseChannel"; - -describe("parseChannel", () => { - it("works with json/jsonschema", () => { - const channel = parseChannel({ - messageEncoding: "json", - schema: { - name: "X", - encoding: "jsonschema", - data: new TextEncoder().encode( - JSON.stringify({ type: "object", properties: { value: { type: "string" } } }), - ), - }, - }); - expect(channel.deserialize(new TextEncoder().encode(JSON.stringify({ value: "hi" })))).toEqual({ - value: "hi", - }); - }); - - it("works with flatbuffer", () => { - const reflectionSchema = fs.readFileSync(`${__dirname}/fixtures/reflection.bfbs`); - const channel = parseChannel({ - messageEncoding: "flatbuffer", - schema: { name: "reflection.Schema", encoding: "flatbuffer", data: reflectionSchema }, - }); - const deserialized = channel.deserialize(reflectionSchema) as { - objects: Record[]; - }; - expect(deserialized.objects.length).toEqual(10); - expect(deserialized.objects[0]!.name).toEqual("reflection.Enum"); - }); - - it("works with protobuf", () => { - const processRootType = jest.fn().mockImplementation((x: unknown) => x); - const processMessageDefinitions = jest.fn().mockImplementation((x: unknown) => x); - const fds = FileDescriptorSet.encode(FileDescriptorSet.root.toDescriptor("proto3")).finish(); - const channel = parseChannel( - { - messageEncoding: "protobuf", - schema: { name: "google.protobuf.FileDescriptorSet", encoding: "protobuf", data: fds }, - }, - { - protobuf: { - processRootType, - processMessageDefinitions, - }, - }, - ); - const deserialized = channel.deserialize(fds) as IFileDescriptorSet; - expect(deserialized.file[0]!.name).toEqual("google_protobuf.proto"); - expect(processRootType).toHaveBeenCalled(); - expect(processMessageDefinitions).toHaveBeenCalled(); - }); - - it("works with ros1", () => { - const channel = parseChannel({ - messageEncoding: "ros1", - schema: { - name: "foo_msgs/Bar", - encoding: "ros1msg", - data: new TextEncoder().encode("string data"), - }, - }); - - const obj = channel.deserialize(new Uint8Array([4, 0, 0, 0, 65, 66, 67, 68])); - expect(obj).toEqual({ data: "ABCD" }); - }); - - it("works with ros2", () => { - const channel = parseChannel({ - messageEncoding: "cdr", - schema: { - name: "foo_msgs/Bar", - encoding: "ros2msg", - data: new TextEncoder().encode("string data"), - }, - }); - - const obj = channel.deserialize(new Uint8Array([0, 1, 0, 0, 5, 0, 0, 0, 65, 66, 67, 68, 0])); - expect(obj).toEqual({ data: "ABCD" }); - }); - - it("works with ros2idl", () => { - const channel = parseChannel({ - messageEncoding: "cdr", - schema: { - name: "foo_msgs/Bar", - encoding: "ros2idl", - data: new TextEncoder().encode(` - module foo_msgs { - struct Bar {string data;}; - }; - `), - }, - }); - - const obj = channel.deserialize(new Uint8Array([0, 1, 0, 0, 5, 0, 0, 0, 65, 66, 67, 68, 0])); - expect(obj).toEqual({ data: "ABCD" }); - }); - it("works with omgidl xcdr2", () => { - const channel = parseChannel({ - messageEncoding: "cdr", - schema: { - name: "foo_msgs::Bar", - encoding: "omgidl", - data: new TextEncoder().encode(` - enum Color {RED, GREEN, BLUE}; - module foo_msgs { - struct NonRootBar {string data;}; - struct Bar {foo_msgs::NonRootBar data; Color color;}; - }; - `), - }, - }); - - const obj = channel.deserialize( - new Uint8Array([0, 1, 0, 0, 5, 0, 0, 0, 65, 66, 67, 68, 0, 0, 0, 0, 2, 0, 0, 0]), - ); - expect(obj).toEqual({ data: { data: "ABCD" }, color: 2 }); - }); -}); diff --git a/typescript/support/src/parseChannel.ts b/typescript/support/src/parseChannel.ts deleted file mode 100644 index 8381f8fed4..0000000000 --- a/typescript/support/src/parseChannel.ts +++ /dev/null @@ -1,201 +0,0 @@ -import { MessageDefinition, MessageDefinitionField } from "@foxglove/message-definition"; -import { IDLMessageDefinition, parseIDL } from "@foxglove/omgidl-parser"; -import { MessageReader as OmgidlMessageReader } from "@foxglove/omgidl-serialization"; -import { parseRos2idl } from "@foxglove/ros2idl-parser"; -import { parse as parseMessageDefinition } from "@foxglove/rosmsg"; -import { MessageReader } from "@foxglove/rosmsg-serialization"; -import { MessageReader as ROS2MessageReader } from "@foxglove/rosmsg2-serialization"; - -import { parseFlatbufferSchema } from "./parseFlatbufferSchema"; -import { parseJsonSchema } from "./parseJsonSchema"; -import { ParseProtobufSchemaOptions, parseProtobufSchema } from "./parseProtobufSchema"; -import { MessageDefinitionMap } from "./types"; - -type Channel = { - messageEncoding: string; - schema: { name: string; encoding: string; data: Uint8Array } | undefined; -}; - -export type ParsedChannel = { - deserialize: (data: ArrayBufferView) => unknown; - datatypes: MessageDefinitionMap; -}; - -function parseIDLDefinitionsToDatatypes( - parsedDefinitions: IDLMessageDefinition[], - rootName?: string, -) { - // The only IDL definition non-conformant-to-MessageDefinition is unions - const convertUnionToMessageDefinition = (definition: IDLMessageDefinition): MessageDefinition => { - if (definition.aggregatedKind === "union") { - const innerDefs: MessageDefinitionField[] = definition.cases.map((caseDefinition) => ({ - ...caseDefinition.type, - predicates: caseDefinition.predicates, - })); - - if (definition.defaultCase != undefined) { - innerDefs.push(definition.defaultCase); - } - const { name } = definition; - return { - name, - definitions: innerDefs, - }; - } - return definition; - }; - - const standardDefs: MessageDefinition[] = parsedDefinitions.map(convertUnionToMessageDefinition); - return parsedDefinitionsToDatatypes(standardDefs, rootName); -} - -function parsedDefinitionsToDatatypes( - parsedDefinitions: MessageDefinition[], - rootName?: string, -): MessageDefinitionMap { - const datatypes: MessageDefinitionMap = new Map(); - parsedDefinitions.forEach(({ name, definitions }, index) => { - if (rootName != undefined && index === 0) { - datatypes.set(rootName, { name: rootName, definitions }); - } else if (name != undefined) { - datatypes.set(name, { name, definitions }); - } - }); - return datatypes; -} - -/** - * Options to configure the behavior of {@link parseChannel}. - */ -export type ParseChannelOptions = { - /** - * Options to configure the behavior of {@link parseProtobufSchema}. - */ - protobuf?: ParseProtobufSchemaOptions; -}; - -/** - * Process a channel/schema and extract information that can be used to deserialize messages on the - * channel, and schemas in the format compatible with `@foxglove/message-definition`. - * - * See: - * - https://github.com/foxglove/mcap/blob/main/docs/specification/well-known-message-encodings.md - * - https://github.com/foxglove/mcap/blob/main/docs/specification/well-known-schema-encodings.md - */ -export function parseChannel(channel: Channel, options?: ParseChannelOptions): ParsedChannel { - if (channel.messageEncoding === "json") { - if (channel.schema != undefined && channel.schema.encoding !== "jsonschema") { - throw new Error( - `Message encoding ${channel.messageEncoding} with schema encoding '${channel.schema.encoding}' is not supported (expected jsonschema or no schema)`, - ); - } - const textDecoder = new TextDecoder(); - let datatypes: MessageDefinitionMap = new Map(); - let deserialize = (data: ArrayBufferView) => JSON.parse(textDecoder.decode(data)) as unknown; - if (channel.schema != undefined) { - const schema = - channel.schema.data.length > 0 - ? (JSON.parse(textDecoder.decode(channel.schema.data)) as unknown) - : undefined; - if (schema != undefined) { - if (typeof schema !== "object") { - throw new Error(`Invalid schema, expected JSON object, got ${typeof schema}`); - } - const { datatypes: parsedDatatypes, postprocessValue } = parseJsonSchema( - schema as Record, - channel.schema.name, - ); - datatypes = parsedDatatypes; - deserialize = (data) => - postprocessValue(JSON.parse(textDecoder.decode(data)) as Record); - } - } - return { deserialize, datatypes }; - } - - if (channel.messageEncoding === "flatbuffer") { - if (channel.schema?.encoding !== "flatbuffer") { - throw new Error( - `Message encoding ${channel.messageEncoding} with ${ - channel.schema == undefined - ? "no encoding" - : `schema encoding '${channel.schema.encoding}'` - } is not supported (expected flatbuffer)`, - ); - } - return parseFlatbufferSchema(channel.schema.name, channel.schema.data); - } - - if (channel.messageEncoding === "protobuf") { - if (channel.schema?.encoding !== "protobuf") { - throw new Error( - `Message encoding ${channel.messageEncoding} with ${ - channel.schema == undefined - ? "no encoding" - : `schema encoding '${channel.schema.encoding}'` - } is not supported (expected protobuf)`, - ); - } - return parseProtobufSchema(channel.schema.name, channel.schema.data, options?.protobuf); - } - - if (channel.messageEncoding === "ros1") { - if (channel.schema?.encoding !== "ros1msg") { - throw new Error( - `Message encoding ${channel.messageEncoding} with ${ - channel.schema == undefined - ? "no encoding" - : `schema encoding '${channel.schema.encoding}'` - } is not supported (expected ros1msg)`, - ); - } - const schema = new TextDecoder().decode(channel.schema.data); - const parsedDefinitions = parseMessageDefinition(schema); - const reader = new MessageReader(parsedDefinitions); - return { - datatypes: parsedDefinitionsToDatatypes(parsedDefinitions, channel.schema.name), - deserialize: (data) => reader.readMessage(data), - }; - } - - if (channel.messageEncoding === "cdr") { - if ( - channel.schema?.encoding !== "ros2msg" && - channel.schema?.encoding !== "ros2idl" && - channel.schema?.encoding !== "omgidl" - ) { - throw new Error( - `Message encoding ${channel.messageEncoding} with ${ - channel.schema == undefined - ? "no encoding" - : `schema encoding '${channel.schema.encoding}'` - } is not supported (expected "ros2msg" or "ros2idl")`, - ); - } - const schema = new TextDecoder().decode(channel.schema.data); - if (channel.schema.encoding === "omgidl") { - const parsedDefinitions = parseIDL(schema); - const reader = new OmgidlMessageReader(channel.schema.name, parsedDefinitions); - const datatypes = parseIDLDefinitionsToDatatypes(parsedDefinitions); - return { - datatypes, - deserialize: (data) => reader.readMessage(data), - }; - } else { - const isIdl = channel.schema.encoding === "ros2idl"; - - const parsedDefinitions = isIdl - ? parseRos2idl(schema) - : parseMessageDefinition(schema, { ros2: true }); - - const reader = new ROS2MessageReader(parsedDefinitions); - - return { - datatypes: parsedDefinitionsToDatatypes(parsedDefinitions, channel.schema.name), - deserialize: (data) => reader.readMessage(data), - }; - } - } - - throw new Error(`Unsupported encoding ${channel.messageEncoding}`); -} diff --git a/typescript/support/src/parseFlatbufferSchema.test.ts b/typescript/support/src/parseFlatbufferSchema.test.ts deleted file mode 100644 index 205ab52ca2..0000000000 --- a/typescript/support/src/parseFlatbufferSchema.test.ts +++ /dev/null @@ -1,389 +0,0 @@ -import { ByteBuffer, Builder } from "flatbuffers"; -import { Schema, BaseType, Type } from "flatbuffers_reflection"; -import fs from "fs"; - -import { ByteVector } from "./fixtures/byte-vector"; -import { parseFlatbufferSchema } from "./parseFlatbufferSchema"; - -const enumSchema = { - definitions: [ - { - isArray: true, - isComplex: true, - name: "attributes", - type: "reflection.KeyValue", - }, - { - name: "declaration_file", - type: "string", - }, - { - isArray: true, - name: "documentation", - type: "string", - }, - { - name: "is_union", - type: "bool", - }, - { - name: "name", - type: "string", - }, - { - isComplex: true, - name: "underlying_type", - type: "reflection.Type", - }, - { - isArray: true, - isComplex: true, - name: "values", - type: "reflection.EnumVal", - }, - ], -}; -const typeSchema = { - definitions: [ - { - name: "base_size", - type: "uint32", - }, - { - isConstant: true, - name: "None", - type: "int8", - value: 0n, - }, - { - isConstant: true, - name: "UType", - type: "int8", - value: 1n, - }, - { - isConstant: true, - name: "Bool", - type: "int8", - value: 2n, - }, - { - isConstant: true, - name: "Byte", - type: "int8", - value: 3n, - }, - { - isConstant: true, - name: "UByte", - type: "int8", - value: 4n, - }, - { - isConstant: true, - name: "Short", - type: "int8", - value: 5n, - }, - { - isConstant: true, - name: "UShort", - type: "int8", - value: 6n, - }, - { - isConstant: true, - name: "Int", - type: "int8", - value: 7n, - }, - { - isConstant: true, - name: "UInt", - type: "int8", - value: 8n, - }, - { - isConstant: true, - name: "Long", - type: "int8", - value: 9n, - }, - { - isConstant: true, - name: "ULong", - type: "int8", - value: 10n, - }, - { - isConstant: true, - name: "Float", - type: "int8", - value: 11n, - }, - { - isConstant: true, - name: "Double", - type: "int8", - value: 12n, - }, - { - isConstant: true, - name: "String", - type: "int8", - value: 13n, - }, - { - isConstant: true, - name: "Vector", - type: "int8", - value: 14n, - }, - { - isConstant: true, - name: "Obj", - type: "int8", - value: 15n, - }, - { - isConstant: true, - name: "Union", - type: "int8", - value: 16n, - }, - { - isConstant: true, - name: "Array", - type: "int8", - value: 17n, - }, - { - isConstant: true, - name: "MaxBaseType", - type: "int8", - value: 18n, - }, - { - name: "base_type", - type: "int8", - }, - { - isConstant: true, - name: "None", - type: "int8", - value: 0n, - }, - { - isConstant: true, - name: "UType", - type: "int8", - value: 1n, - }, - { - isConstant: true, - name: "Bool", - type: "int8", - value: 2n, - }, - { - isConstant: true, - name: "Byte", - type: "int8", - value: 3n, - }, - { - isConstant: true, - name: "UByte", - type: "int8", - value: 4n, - }, - { - isConstant: true, - name: "Short", - type: "int8", - value: 5n, - }, - { - isConstant: true, - name: "UShort", - type: "int8", - value: 6n, - }, - { - isConstant: true, - name: "Int", - type: "int8", - value: 7n, - }, - { - isConstant: true, - name: "UInt", - type: "int8", - value: 8n, - }, - { - isConstant: true, - name: "Long", - type: "int8", - value: 9n, - }, - { - isConstant: true, - name: "ULong", - type: "int8", - value: 10n, - }, - { - isConstant: true, - name: "Float", - type: "int8", - value: 11n, - }, - { - isConstant: true, - name: "Double", - type: "int8", - value: 12n, - }, - { - isConstant: true, - name: "String", - type: "int8", - value: 13n, - }, - { - isConstant: true, - name: "Vector", - type: "int8", - value: 14n, - }, - { - isConstant: true, - name: "Obj", - type: "int8", - value: 15n, - }, - { - isConstant: true, - name: "Union", - type: "int8", - value: 16n, - }, - { - isConstant: true, - name: "Array", - type: "int8", - value: 17n, - }, - { - isConstant: true, - name: "MaxBaseType", - type: "int8", - value: 18n, - }, - { - name: "element", - type: "int8", - }, - { - name: "element_size", - type: "uint32", - }, - { - name: "fixed_length", - type: "uint16", - }, - { - name: "index", - type: "int32", - }, - ], -}; - -describe("parseFlatbufferSchema", () => { - it("rejects invalid schema", () => { - expect(() => parseFlatbufferSchema("test", new Uint8Array([1]))).toThrow(); - }); - it("parses root table schema", () => { - // Use the reflection Schema itself to read the reflection Schema (this is - // actually a pretty good test case for coverage, since the Schema message - // includes almost all the various flatbuffer features). - // The .bfbs file in question is generated from running - // $ flatc -b --schema reflection/reflection.fbs - // In https://github.com/google/flatbuffers - const reflectionSchemaBuffer: Buffer = fs.readFileSync(`${__dirname}/fixtures/reflection.bfbs`); - const { datatypes, deserialize } = parseFlatbufferSchema( - "reflection.Schema", - reflectionSchemaBuffer, - ); - /* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call */ - const deserialized: any = deserialize(reflectionSchemaBuffer); - const reflectionSchemaByteBuffer: ByteBuffer = new ByteBuffer(reflectionSchemaBuffer); - const schema = Schema.getRootAsSchema(reflectionSchemaByteBuffer); - // Spot check individual components to ensure that they got deserialized correctly. - expect(deserialized.objects.length).toEqual(schema.objectsLength()); - expect(deserialized.objects.length).toEqual(10); - expect(deserialized.objects[0].name).toEqual("reflection.Enum"); - expect(deserialized.file_ident).toEqual("BFBS"); - expect(deserialized.file_ext).toEqual("bfbs"); - expect(deserialized.fbs_files[0].filename.substr(-14)).toEqual("reflection.fbs"); - // Spot check the datatypes list. - expect(datatypes.keys()).toContain("reflection.Enum"); - expect(datatypes.keys()).toContain("reflection.Object"); - expect(datatypes.get("reflection.Enum")).toEqual(enumSchema); - /* eslint-enable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call */ - }); - it("parses non-root table schema", () => { - const reflectionSchemaBuffer: Buffer = fs.readFileSync(`${__dirname}/fixtures/reflection.bfbs`); - const { datatypes, deserialize } = parseFlatbufferSchema( - "reflection.Type", - reflectionSchemaBuffer, - ); - expect(datatypes.keys()).toContain("reflection.Type"); - expect(datatypes.get("reflection.Type")).toEqual(typeSchema); - - // Construct a reflection.Type object from scratch and confirm that we get - // exactly the correct result. - const builder = new Builder(); - Type.startType(builder); - Type.addBaseType(builder, BaseType.Int); - Type.addIndex(builder, 123); - builder.finish(Type.endType(builder)); - - expect(deserialize(builder.asUint8Array())).toEqual({ - base_size: 4, - base_type: 7, - element: 0, - element_size: 0, - fixed_length: 0, - index: 123, - }); - }); - it("converts uint8 vectors to uint8arrays", () => { - const builder = new Builder(); - - /** - * Byte Vector Schema (.fbs file not included in this repo) - * table ByteVector { - * data:[uint8]; - * } - * root_type ByteVector; - */ - const data = ByteVector.createDataVector(builder, [1, 2, 3]); - ByteVector.startByteVector(builder); - ByteVector.addData(builder, data); - const byteVector = ByteVector.endByteVector(builder); - builder.finish(byteVector); - /** the underlying buffer for the builder is larger than the uint8array of the data - * this needs to be cleared so that the reading from the buffer by the parser doesn't use the wrong offsets - * normally when this is written to a file, only the contents of the Uint8Array are written, not the underlying buffer - * so this replicates that - * essentially need to make sure byteVectorBin.buffer !== builder.asUint8Array().buffer - */ - const byteVectorBin = Uint8Array.from(builder.asUint8Array()); - - const byteVectorSchemaArray = fs.readFileSync(`${__dirname}/fixtures/ByteVector.bfbs`); - const { deserialize } = parseFlatbufferSchema("ByteVector", byteVectorSchemaArray); - expect(deserialize(byteVectorBin)).toEqual({ data: new Uint8Array([1, 2, 3]) }); - }); -}); diff --git a/typescript/support/src/parseFlatbufferSchema.ts b/typescript/support/src/parseFlatbufferSchema.ts deleted file mode 100644 index 6551c9ec09..0000000000 --- a/typescript/support/src/parseFlatbufferSchema.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { MessageDefinitionField } from "@foxglove/message-definition"; -import { ByteBuffer } from "flatbuffers"; -import { BaseType, Schema, SchemaT, FieldT, Parser, Table } from "flatbuffers_reflection"; - -import { MessageDefinitionMap } from "./types"; - -function typeForSimpleField(type: BaseType): string { - switch (type) { - case BaseType.Bool: - return "bool"; - case BaseType.Byte: - return "int8"; - case BaseType.UType: - case BaseType.UByte: - return "uint8"; - case BaseType.Short: - return "int16"; - case BaseType.UShort: - return "uint16"; - case BaseType.Int: - return "int32"; - case BaseType.UInt: - return "uint32"; - case BaseType.Long: - return "int64"; - case BaseType.ULong: - return "uint64"; - case BaseType.Float: - return "float32"; - case BaseType.Double: - return "float64"; - case BaseType.String: - return "string"; - case BaseType.Vector: - case BaseType.Obj: - case BaseType.Union: - case BaseType.Array: - throw new Error(`${type} is not a simple type.`); - case BaseType.None: - case BaseType.MaxBaseType: - throw new Error("None is not a valid type."); - } -} - -function flatbufferString(unchecked: string | Uint8Array | null | undefined): string { - if (typeof unchecked === "string") { - return unchecked; - } - throw new Error(`Expected string, found ${typeof unchecked}`); -} - -function typeForField(schema: SchemaT, field: FieldT): MessageDefinitionField[] { - const fields: MessageDefinitionField[] = []; - switch (field.type?.baseType) { - case BaseType.UType: - case BaseType.Bool: - case BaseType.Byte: - case BaseType.UByte: - case BaseType.Short: - case BaseType.UShort: - case BaseType.Int: - case BaseType.UInt: - case BaseType.Long: - case BaseType.ULong: - case BaseType.Float: - case BaseType.Double: - case BaseType.String: - case BaseType.None: { - const simpleType = typeForSimpleField(field.type.baseType); - // Enums have magic logic--the constants definitions for the enum values - // have to go right before the enum itself. - if (field.type.index !== -1) { - const enums = schema.enums[field.type.index]?.values; - if (enums == undefined) { - throw new Error( - `Invalid schema, missing enum values for field type ${ - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - schema.enums[field.type.index]?.name - }`, - ); - } - for (const enumVal of enums) { - fields.push({ - name: flatbufferString(enumVal.name), - type: simpleType, - isConstant: true, - value: enumVal.value, - }); - } - } - fields.push({ name: flatbufferString(field.name), type: simpleType }); - break; - } - case BaseType.Vector: - switch (field.type.element) { - case BaseType.Vector: - case BaseType.Union: - case BaseType.Array: - case BaseType.None: - throw new Error("Vectors of vectors, unions, arrays, and None's are unsupported."); - case BaseType.Obj: - fields.push({ - name: flatbufferString(field.name), - type: flatbufferString(schema.objects[field.type.index]?.name), - isComplex: true, - isArray: true, - }); - break; - default: { - const type = typeForSimpleField(field.type.element); - // Enums have magic logic--the constants definitions for the enum - // values have to go right before the enum itself. - if (field.type.index !== -1) { - const enums = schema.enums[field.type.index]?.values; - if (enums == undefined) { - throw new Error("Invalid schema"); - } - for (const enumVal of enums) { - fields.push({ - name: flatbufferString(enumVal.name), - type, - isConstant: true, - value: enumVal.value, - }); - } - } - fields.push({ name: flatbufferString(field.name), type, isArray: true }); - break; - } - } - break; - case BaseType.Obj: - fields.push({ - name: flatbufferString(field.name), - type: flatbufferString(schema.objects[field.type.index]?.name), - isComplex: true, - }); - break; - case BaseType.Union: - case BaseType.Array: - case BaseType.MaxBaseType: - case undefined: - throw new Error("Unions and Arrays are not currently supported"); - } - return fields; -} - -/** - * Parse a flatbuffer binary schema and produce datatypes and a deserializer function. - * - * Note: Currently this does not support "lazy" message reading in the style of the ros1 message - * reader, and so will relatively inefficiently deserialize the entire flatbuffer message. - */ -export function parseFlatbufferSchema( - schemaName: string, - schemaArray: Uint8Array, -): { - datatypes: MessageDefinitionMap; - deserialize: (buffer: ArrayBufferView) => unknown; -} { - const datatypes: MessageDefinitionMap = new Map(); - const schemaBuffer = new ByteBuffer(schemaArray); - const rawSchema = Schema.getRootAsSchema(schemaBuffer); - const schema = rawSchema.unpack(); - - let typeIndex = -1; - for (let schemaIndex = 0; schemaIndex < schema.objects.length; ++schemaIndex) { - const object = schema.objects[schemaIndex]; - if (object?.name === schemaName) { - typeIndex = schemaIndex; - } - let fields: MessageDefinitionField[] = []; - if (object?.fields == undefined) { - continue; - } - for (const field of object.fields) { - fields = fields.concat(typeForField(schema, field)); - } - datatypes.set(flatbufferString(object.name), { definitions: fields }); - } - if (typeIndex === -1) { - if (schema.rootTable?.name !== schemaName) { - throw new Error( - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - `Type "${schemaName}" is not available in the schema for "${schema.rootTable?.name}".`, - ); - } - } - const parser = new Parser(rawSchema); - // We set readDefaults=true to ensure that the reader receives default values for unset fields, or - // fields that were explicitly set but with ForceDefaults(false) on the writer side. This is - // necessary because `datatypes` does not include information about default values from the - // schema. See discussion: - const toObject = parser.toObjectLambda(typeIndex, /*readDefaults=*/ true); - const deserialize = (buffer: ArrayBufferView) => { - const byteBuffer = new ByteBuffer( - new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength), - ); - const table = new Table( - byteBuffer, - typeIndex, - byteBuffer.readInt32(byteBuffer.position()) + byteBuffer.position(), - false, - ); - return toObject(table); - }; - return { datatypes, deserialize }; -} diff --git a/typescript/support/src/parseJsonSchema.test.ts b/typescript/support/src/parseJsonSchema.test.ts deleted file mode 100644 index 105fcc01eb..0000000000 --- a/typescript/support/src/parseJsonSchema.test.ts +++ /dev/null @@ -1,275 +0,0 @@ -import { foxgloveMessageSchemas, generateJsonSchema } from "@foxglove/schemas/internal"; -import * as protobufjs from "protobufjs"; - -import { parseJsonSchema } from "./parseJsonSchema"; - -describe("parseJsonSchema", () => { - it("rejects invalid schema", () => { - expect(() => parseJsonSchema({}, "X")).toThrow(`Expected "type": "object"`); - expect(() => parseJsonSchema({ type: 3 }, "X")).toThrow(`Expected "type": "object"`); - expect(() => parseJsonSchema({ type: "string" }, "X")).toThrow(`Expected "type": "object"`); - expect(() => - parseJsonSchema({ type: "object", properties: { x: { type: "null" } } }, "X"), - ).toThrow(`Unsupported type "null" for x`); - expect(() => - parseJsonSchema( - { type: "object", properties: { x: { type: "string", contentEncoding: "X" } } }, - "X", - ), - ).toThrow(`Unsupported contentEncoding "X"`); - expect(() => - parseJsonSchema( - { - type: "object", - properties: { - x: { type: "array", items: { type: "string", contentEncoding: "base64" } }, - }, - }, - "X", - ), - ).toThrow(`Unsupported contentEncoding "base64" for array item`); - }); - - it.each([ - { - name: "Foo", - schema: { type: "object", properties: {} }, - expectedDatatypes: new Map([["Foo", { definitions: [] }]]), - value: {}, - expectedValue: {}, - }, - - { - name: "Foo", - schema: { - type: "object", - properties: { - str: { type: "string" }, - bin: { type: "string", contentEncoding: "base64" }, - num: { type: "number" }, - int: { type: "integer" }, - bool: { type: "boolean" }, - }, - }, - expectedDatatypes: new Map([ - [ - "Foo", - { - definitions: [ - { name: "str", type: "string" }, - { name: "bin", type: "uint8", isArray: true }, - { name: "num", type: "float64" }, - { name: "int", type: "float64" }, - { name: "bool", type: "bool" }, - ], - }, - ], - ]), - value: { - str: "str", - bin: protobufjs.util.base64.encode(new Uint8Array([0, 1, 0xfe, 0xff]), 0, 4), - num: 1.5, - int: 1.5, - bool: true, - }, - expectedValue: { - str: "str", - bin: new Uint8Array([0, 1, 0xfe, 0xff]), - num: 1.5, - int: 1.5, - bool: true, - }, - }, - - { - name: "Foo", - schema: { - type: "object", - properties: { - str: { type: "array", items: { type: "string" } }, - num: { type: "array", items: { type: "number" } }, - int: { type: "array", items: { type: "integer" } }, - bool: { type: "array", items: { type: "boolean" } }, - }, - }, - expectedDatatypes: new Map([ - [ - "Foo", - { - definitions: [ - { name: "str", type: "string", isArray: true }, - { name: "num", type: "float64", isArray: true }, - { name: "int", type: "float64", isArray: true }, - { name: "bool", type: "bool", isArray: true }, - ], - }, - ], - ]), - value: { str: ["str"], num: [1.5], int: [1.5], bool: [true] }, - expectedValue: { str: ["str"], num: [1.5], int: [1.5], bool: [true] }, - }, - - { - name: "Foo", - schema: { - type: "object", - properties: { - bar: { type: "object", properties: { str: { type: "string" } } }, - }, - }, - expectedDatatypes: new Map([ - ["Foo", { definitions: [{ name: "bar", type: "Foo.bar", isComplex: true }] }], - ["Foo.bar", { definitions: [{ name: "str", type: "string" }] }], - ]), - value: { bar: { str: "str" } }, - expectedValue: { bar: { str: "str" } }, - }, - - { - name: "Foo", - schema: { - type: "object", - properties: { - bin: { type: "string", contentEncoding: "base64" }, - nested: { - type: "object", - properties: { - bin2: { type: "string", contentEncoding: "base64" }, - bar: { - type: "array", - items: { - type: "object", - properties: { bin3: { type: "string", contentEncoding: "base64" } }, - }, - }, - }, - }, - }, - }, - expectedDatatypes: new Map([ - [ - "Foo", - { - definitions: [ - { name: "bin", type: "uint8", isArray: true }, - { name: "nested", type: "Foo.nested", isComplex: true }, - ], - }, - ], - [ - "Foo.nested", - { - definitions: [ - { name: "bin2", type: "uint8", isArray: true }, - { name: "bar", type: "Foo.nested.bar", isComplex: true, isArray: true }, - ], - }, - ], - ["Foo.nested.bar", { definitions: [{ name: "bin3", type: "uint8", isArray: true }] }], - ]), - value: { - bin: protobufjs.util.base64.encode(new Uint8Array([0xa1, 0xb2, 0xc3]), 0, 3), - nested: { - bin2: protobufjs.util.base64.encode(new Uint8Array([0xd4, 0xe5, 0xf6]), 0, 3), - bar: [ - { bin3: protobufjs.util.base64.encode(new Uint8Array([0, 1, 0xfe, 0xff]), 0, 4) }, - { bin3: protobufjs.util.base64.encode(new Uint8Array([2, 3, 0xfe, 0xff]), 0, 4) }, - ], - }, - }, - expectedValue: { - bin: new Uint8Array([0xa1, 0xb2, 0xc3]), - nested: { - bin2: new Uint8Array([0xd4, 0xe5, 0xf6]), - bar: [ - { bin3: new Uint8Array([0, 1, 0xfe, 0xff]) }, - { bin3: new Uint8Array([2, 3, 0xfe, 0xff]) }, - ], - }, - }, - }, - ])( - "converts schema to datatypes and decodes base64", - ({ name, schema, expectedDatatypes, value, expectedValue }) => { - const { datatypes, postprocessValue } = parseJsonSchema(schema, name); - expect(datatypes).toEqual(expectedDatatypes); - expect(postprocessValue(value)).toEqual(expectedValue); - }, - ); - - it("allows missing sub-properties", () => { - const { postprocessValue } = parseJsonSchema( - { - type: "object", - properties: { - foo: { type: "number" }, - bar: { - type: "object", - properties: { baz: { type: "object", properties: { quux: { type: "number" } } } }, - }, - }, - }, - "Root", - ); - expect(postprocessValue({ foo: 3 })).toEqual({ foo: 3 }); - }); - it("allows missing sub-properties in arrays", () => { - const { postprocessValue } = parseJsonSchema( - { - type: "object", - properties: { - arr: { - type: "array", - items: { - type: "object", - properties: { - foo: { type: "number" }, - bar: { - type: "object", - properties: { baz: { type: "object", properties: { quux: { type: "number" } } } }, - }, - }, - }, - }, - }, - }, - "Root", - ); - expect(postprocessValue({ arr: [{ foo: 3 }] })).toEqual({ arr: [{ foo: 3 }] }); - }); - - it("converts oneOf to enum", () => { - const { datatypes } = parseJsonSchema( - { - type: "object", - properties: { - level: { - oneOf: [ - { title: "X", const: 1 }, - { title: "Y", const: 2 }, - ], - }, - }, - }, - "Log", - ); - expect(datatypes).toEqual( - new Map([ - [ - "Log", - { - definitions: [ - { name: "X", type: "uint32", isConstant: true, value: 1 }, - { name: "Y", type: "uint32", isConstant: true, value: 2 }, - { name: "level", type: "uint32" }, - ], - }, - ], - ]), - ); - }); - - it.each(Object.values(foxgloveMessageSchemas))("handles Foxglove schema '$name'", (schema) => { - expect(() => parseJsonSchema(generateJsonSchema(schema), schema.name)).not.toThrow(); - }); -}); diff --git a/typescript/support/src/parseJsonSchema.ts b/typescript/support/src/parseJsonSchema.ts deleted file mode 100644 index 575ceea4b6..0000000000 --- a/typescript/support/src/parseJsonSchema.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { MessageDefinitionField } from "@foxglove/message-definition"; -import * as base64 from "@protobufjs/base64"; - -import { MessageDefinitionMap } from "./types"; - -/** - * Parse a JSON Schema and produce datatypes and a deserializer function. - */ -export function parseJsonSchema( - rootJsonSchema: Record, - rootTypeName: string, -): { - datatypes: MessageDefinitionMap; - - /** - * A function that should be called after parsing a value from a JSON string to do any necessary - * post-processing (e.g. base64 decoding) - */ - postprocessValue: (value: Record) => unknown; -} { - const datatypes: MessageDefinitionMap = new Map(); - - function addFieldsRecursive( - schema: Record, - typeName: string, - keyPath: string[], - ): (value: Record) => unknown { - let postprocessObject: (value: Record) => unknown = (value) => value; - const fields: MessageDefinitionField[] = []; - if (schema.type !== "object") { - throw new Error( - `Expected "type": "object" for schema ${typeName}, got ${JSON.stringify(schema.type)}`, - ); - } - for (const [fieldName, fieldSchema] of Object.entries( - schema.properties as Record>, - )) { - if (Array.isArray(fieldSchema.oneOf)) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - if (fieldSchema.oneOf.every((alternative) => typeof alternative.const === "number")) { - for (const alternative of fieldSchema.oneOf) { - fields.push({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - name: alternative.title, - type: "uint32", - isConstant: true, - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - value: alternative.const, - }); - } - fields.push({ name: fieldName, type: "uint32" }); - continue; - } else { - throw new Error( - `Unsupported type for ${keyPath - .concat(fieldName) - .join(".")}: oneOf alternatives must have number values`, - ); - } - } - switch (fieldSchema.type) { - case "boolean": - fields.push({ name: fieldName, type: "bool" }); - break; - case "string": - switch (fieldSchema.contentEncoding) { - case undefined: - fields.push({ name: fieldName, type: "string" }); - break; - case "base64": { - fields.push({ name: fieldName, type: "uint8", isArray: true }); - const prevPostprocess = postprocessObject; - postprocessObject = (value) => { - const str = value[fieldName]; - if (typeof str === "string") { - const decoded = new Uint8Array(base64.length(str)); - if (base64.decode(str, decoded, 0) !== decoded.byteLength) { - throw new Error( - `Failed to decode base64 data for ${keyPath.concat(fieldName).join(".")}`, - ); - } - value[fieldName] = decoded; - } - return prevPostprocess(value); - }; - break; - } - default: - throw new Error( - `Unsupported contentEncoding ${JSON.stringify( - fieldSchema.contentEncoding, - )} in ${keyPath.concat(fieldName).join(".")}`, - ); - } - break; - case "number": - case "integer": - fields.push({ name: fieldName, type: "float64" }); - break; - case "object": { - const nestedTypeName = `${typeName}.${fieldName}`; - const postprocessNestedObject = addFieldsRecursive( - fieldSchema, - nestedTypeName, - keyPath.concat(fieldName), - ); - const prevPostprocess = postprocessObject; - postprocessObject = (value) => { - const fieldValue = value[fieldName]; - if (fieldValue != undefined && typeof fieldValue === "object") { - value[fieldName] = postprocessNestedObject(fieldValue as Record); - } - return prevPostprocess(value); - }; - fields.push({ name: fieldName, type: nestedTypeName, isComplex: true }); - break; - } - case "array": { - const itemSchema = fieldSchema.items as Record; - switch (itemSchema.type) { - case "boolean": - fields.push({ name: fieldName, type: "bool", isArray: true }); - break; - case "string": - if (itemSchema.contentEncoding != undefined) { - throw new Error( - `Unsupported contentEncoding ${JSON.stringify( - itemSchema.contentEncoding, - )} for array item ${keyPath.concat(fieldName).join(".")}`, - ); - } - fields.push({ name: fieldName, type: "string", isArray: true }); - break; - case "number": - case "integer": - fields.push({ name: fieldName, type: "float64", isArray: true }); - break; - case "object": { - const nestedTypeName = `${typeName}.${fieldName}`; - const postprocessArrayItem = addFieldsRecursive( - fieldSchema.items as Record, - nestedTypeName, - keyPath.concat(fieldName), - ); - const prevPostprocess = postprocessObject; - postprocessObject = (value) => { - const arr = value[fieldName]; - if (Array.isArray(arr)) { - value[fieldName] = arr.map(postprocessArrayItem); - } - return prevPostprocess(value); - }; - fields.push({ - name: fieldName, - type: nestedTypeName, - isComplex: true, - isArray: true, - }); - break; - } - default: - throw new Error( - `Unsupported type ${JSON.stringify(itemSchema.type)} for array item ${keyPath - .concat(fieldName) - .join(".")}`, - ); - } - break; - } - case "null": - default: - throw new Error( - `Unsupported type ${JSON.stringify(fieldSchema.type)} for ${keyPath - .concat(fieldName) - .join(".")}`, - ); - } - } - datatypes.set(typeName, { definitions: fields }); - return postprocessObject; - } - - const postprocessValue = addFieldsRecursive(rootJsonSchema, rootTypeName, []); - return { datatypes, postprocessValue }; -} diff --git a/typescript/support/src/parseProtobufSchema.test.ts b/typescript/support/src/parseProtobufSchema.test.ts deleted file mode 100644 index bb0bdbd59a..0000000000 --- a/typescript/support/src/parseProtobufSchema.test.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { parseProtobufSchema } from "./parseProtobufSchema"; - -describe("parseProtobufSchema", () => { - it("handles protobuf repeated enum having multiple default value aliases", () => { - /* - syntax = "proto3"; - - enum ExampleEnum { - option allow_alias = true; - UNKNOWN = 0; - WHATEVER = 0; - FOO = 1; - BAR = 2; - } - - message ExampleMessage { - repeated ExampleEnum data = 1; - } - */ - const channel = parseProtobufSchema( - "ExampleMessage", - Buffer.from( - // cspell:disable-next-line - "0A8D010A156578616D706C655F6D6573736167652E70726F746F222C0A0E4578616D706C654D657373616765121A0A046461746118012003280E320C2E4578616D706C65456E756D2A3E0A0B4578616D706C65456E756D120B0A07554E4B4E4F574E1000120C0A085748415445564552100012070A03464F4F100112070A0342415210021A021001620670726F746F33", - "hex", - ), - ); - expect(channel.deserialize(Buffer.from("0A0101", "hex"))).toEqual({ data: [1] }); - expect(channel.datatypes).toEqual( - new Map([ - [ - "ExampleMessage", - { - definitions: [ - { isConstant: true, name: "UNKNOWN", type: "int32", value: 0 }, - { isConstant: true, name: "WHATEVER", type: "int32", value: 0 }, - { isConstant: true, name: "FOO", type: "int32", value: 1 }, - { isConstant: true, name: "BAR", type: "int32", value: 2 }, - { name: "data", type: "int32" }, - ], - }, - ], - ]), - ); - }); - - it("allows modifying deserialization and datatypes", () => { - /* - syntax = "proto3"; - - enum ExampleEnum { - option allow_alias = true; - UNKNOWN = 0; - WHATEVER = 0; - FOO = 1; - BAR = 2; - } - - message ExampleMessage { - repeated ExampleEnum data = 1; - } - */ - const channel = parseProtobufSchema( - "ExampleMessage", - Buffer.from( - // cspell:disable-next-line - "0A8D010A156578616D706C655F6D6573736167652E70726F746F222C0A0E4578616D706C654D657373616765121A0A046461746118012003280E320C2E4578616D706C65456E756D2A3E0A0B4578616D706C65456E756D120B0A07554E4B4E4F574E1000120C0A085748415445564552100012070A03464F4F100112070A0342415210021A021001620670726F746F33", - "hex", - ), - { - processRootType(rootType) { - rootType.fieldsById[1]!.name = "renamed_data"; - return rootType; - }, - processMessageDefinitions(definitions) { - definitions - .get("ExampleMessage")! - .definitions.find((def) => def.name === "renamed_data")!.type = "float64"; - return definitions; - }, - }, - ); - expect(channel.deserialize(Buffer.from("0A0101", "hex"))).toEqual({ renamed_data: [1] }); - expect(channel.datatypes).toEqual( - new Map([ - [ - "ExampleMessage", - { - definitions: [ - { isConstant: true, name: "UNKNOWN", type: "int32", value: 0 }, - { isConstant: true, name: "WHATEVER", type: "int32", value: 0 }, - { isConstant: true, name: "FOO", type: "int32", value: 1 }, - { isConstant: true, name: "BAR", type: "int32", value: 2 }, - { name: "renamed_data", type: "float64" }, - ], - }, - ], - ]), - ); - }); - - it("handles protobuf int64 values", () => { - /* - syntax = "proto3"; - - message Int64Test { - int64 int64 = 1; - uint64 uint64 = 2; - sint64 sint64 = 3; - fixed64 fixed64 = 4; - sfixed64 sfixed64 = 5; - map int64map = 6; - map uint64map = 7; - map sint64map = 8; - map fixed64map = 9; - map sfixed64map = 10; - repeated Nested nested = 11; - } - - message Nested { - int64 int64 = 1; - uint64 uint64 = 2; - sint64 sint64 = 3; - fixed64 fixed64 = 4; - sfixed64 sfixed64 = 5; - map int64map = 6; - map uint64map = 7; - map sint64map = 8; - map fixed64map = 9; - map sfixed64map = 10; - } - */ - const channel = parseProtobufSchema( - "Int64Test", - Buffer.from( - // cspell:disable-next-line - "CvILCg9JbnQ2NFRlc3QucHJvdG8igwYKCUludDY0VGVzdBIUCgVpbnQ2NBgBIAEoA1IFaW50NjQSFgoGdWludDY0GAIgASgEUgZ1aW50NjQSFgoGc2ludDY0GAMgASgSUgZzaW50NjQSGAoHZml4ZWQ2NBgEIAEoBlIHZml4ZWQ2NBIaCghzZml4ZWQ2NBgFIAEoEFIIc2ZpeGVkNjQSNAoIaW50NjRtYXAYBiADKAsyGC5JbnQ2NFRlc3QuSW50NjRtYXBFbnRyeVIIaW50NjRtYXASNwoJdWludDY0bWFwGAcgAygLMhkuSW50NjRUZXN0LlVpbnQ2NG1hcEVudHJ5Ugl1aW50NjRtYXASNwoJc2ludDY0bWFwGAggAygLMhkuSW50NjRUZXN0LlNpbnQ2NG1hcEVudHJ5UglzaW50NjRtYXASOgoKZml4ZWQ2NG1hcBgJIAMoCzIaLkludDY0VGVzdC5GaXhlZDY0bWFwRW50cnlSCmZpeGVkNjRtYXASPQoLc2ZpeGVkNjRtYXAYCiADKAsyGy5JbnQ2NFRlc3QuU2ZpeGVkNjRtYXBFbnRyeVILc2ZpeGVkNjRtYXASHwoGbmVzdGVkGAsgAygLMgcuTmVzdGVkUgZuZXN0ZWQaOwoNSW50NjRtYXBFbnRyeRIQCgNrZXkYASABKANSA2tleRIUCgV2YWx1ZRgCIAEoA1IFdmFsdWU6AjgBGjwKDlVpbnQ2NG1hcEVudHJ5EhAKA2tleRgBIAEoBFIDa2V5EhQKBXZhbHVlGAIgASgEUgV2YWx1ZToCOAEaPAoOU2ludDY0bWFwRW50cnkSEAoDa2V5GAEgASgSUgNrZXkSFAoFdmFsdWUYAiABKBJSBXZhbHVlOgI4ARo9Cg9GaXhlZDY0bWFwRW50cnkSEAoDa2V5GAEgASgGUgNrZXkSFAoFdmFsdWUYAiABKAZSBXZhbHVlOgI4ARo+ChBTZml4ZWQ2NG1hcEVudHJ5EhAKA2tleRgBIAEoEFIDa2V5EhQKBXZhbHVlGAIgASgQUgV2YWx1ZToCOAEi0AUKBk5lc3RlZBIUCgVpbnQ2NBgBIAEoA1IFaW50NjQSFgoGdWludDY0GAIgASgEUgZ1aW50NjQSFgoGc2ludDY0GAMgASgSUgZzaW50NjQSGAoHZml4ZWQ2NBgEIAEoBlIHZml4ZWQ2NBIaCghzZml4ZWQ2NBgFIAEoEFIIc2ZpeGVkNjQSMQoIaW50NjRtYXAYBiADKAsyFS5OZXN0ZWQuSW50NjRtYXBFbnRyeVIIaW50NjRtYXASNAoJdWludDY0bWFwGAcgAygLMhYuTmVzdGVkLlVpbnQ2NG1hcEVudHJ5Ugl1aW50NjRtYXASNAoJc2ludDY0bWFwGAggAygLMhYuTmVzdGVkLlNpbnQ2NG1hcEVudHJ5UglzaW50NjRtYXASNwoKZml4ZWQ2NG1hcBgJIAMoCzIXLk5lc3RlZC5GaXhlZDY0bWFwRW50cnlSCmZpeGVkNjRtYXASOgoLc2ZpeGVkNjRtYXAYCiADKAsyGC5OZXN0ZWQuU2ZpeGVkNjRtYXBFbnRyeVILc2ZpeGVkNjRtYXAaOwoNSW50NjRtYXBFbnRyeRIQCgNrZXkYASABKANSA2tleRIUCgV2YWx1ZRgCIAEoA1IFdmFsdWU6AjgBGjwKDlVpbnQ2NG1hcEVudHJ5EhAKA2tleRgBIAEoBFIDa2V5EhQKBXZhbHVlGAIgASgEUgV2YWx1ZToCOAEaPAoOU2ludDY0bWFwRW50cnkSEAoDa2V5GAEgASgSUgNrZXkSFAoFdmFsdWUYAiABKBJSBXZhbHVlOgI4ARo9Cg9GaXhlZDY0bWFwRW50cnkSEAoDa2V5GAEgASgGUgNrZXkSFAoFdmFsdWUYAiABKAZSBXZhbHVlOgI4ARo+ChBTZml4ZWQ2NG1hcEVudHJ5EhAKA2tleRgBIAEoEFIDa2V5EhQKBXZhbHVlGAIgASgQUgV2YWx1ZToCOAFiBnByb3RvMw==", - "base64", - ), - ); - expect( - channel.deserialize( - Buffer.from( - // cspell:disable-next-line - "088c95f0c4c5a9d28ff20110bc85a0cfc8e0c8e38a0118e7d59ff6f4acdbe01b21bc02e8890423c78a298c0a9c584c491ff23216088c95f0c4c5a9d28ff201108c95f0c4c5a9d28ff2013a1608bc85a0cfc8e0c8e38a0110bc85a0cfc8e0c8e38a01421408e7d59ff6f4acdbe01b10e7d59ff6f4acdbe01b4a1209bc02e8890423c78a11bc02e8890423c78a5212098c0a9c584c491ff2118c0a9c584c491ff25aa001088c95f0c4c5a9d28ff20110bc85a0cfc8e0c8e38a0118e7d59ff6f4acdbe01b21bc02e8890423c78a298c0a9c584c491ff23216088c95f0c4c5a9d28ff201108c95f0c4c5a9d28ff2013a1608bc85a0cfc8e0c8e38a0110bc85a0cfc8e0c8e38a01421408e7d59ff6f4acdbe01b10e7d59ff6f4acdbe01b4a1209bc02e8890423c78a11bc02e8890423c78a5212098c0a9c584c491ff2118c0a9c584c491ff2", - "hex", - ), - ), - ).toEqual({ - int64: -999999999999997300n, - uint64: 10000000000000000700n, - sint64: -999999999999997300n, - fixed64: 10000000000000000700n, - sfixed64: -999999999999997300n, - int64map: [{ key: -999999999999997300n, value: -999999999999997300n }], - uint64map: [{ key: 10000000000000000700n, value: 10000000000000000700n }], - sint64map: [{ key: -999999999999997300n, value: -999999999999997300n }], - fixed64map: [{ key: 10000000000000000700n, value: 10000000000000000700n }], - sfixed64map: [{ key: -999999999999997300n, value: -999999999999997300n }], - nested: [ - { - int64: -999999999999997300n, - uint64: 10000000000000000700n, - sint64: -999999999999997300n, - fixed64: 10000000000000000700n, - sfixed64: -999999999999997300n, - int64map: [{ key: -999999999999997300n, value: -999999999999997300n }], - uint64map: [{ key: 10000000000000000700n, value: 10000000000000000700n }], - sint64map: [{ key: -999999999999997300n, value: -999999999999997300n }], - fixed64map: [{ key: 10000000000000000700n, value: 10000000000000000700n }], - sfixed64map: [{ key: -999999999999997300n, value: -999999999999997300n }], - }, - ], - }); - }); -}); diff --git a/typescript/support/src/parseProtobufSchema.ts b/typescript/support/src/parseProtobufSchema.ts deleted file mode 100644 index 1e93d57062..0000000000 --- a/typescript/support/src/parseProtobufSchema.ts +++ /dev/null @@ -1,69 +0,0 @@ -import protobufjs from "@foxglove/protobufjs"; -import { FileDescriptorSet } from "@foxglove/protobufjs/ext/descriptor"; - -import { protobufDefinitionsToDatatypes, stripLeadingDot } from "./protobufDefinitionsToDatatypes"; -import { MessageDefinitionMap } from "./types"; - -/** - * Options to configure the behavior of {@link parseProtobufSchema}. - */ -export type ParseProtobufSchemaOptions = { - /** - * A function that will be called with the root type after parsing the FileDescriptorSet. Used by - * Foxglove Studio to modify the deserialization behavior of google.protobuf.Timestamp & - * google.protobuf.Duration. - */ - processRootType?: (rootType: protobufjs.Type) => protobufjs.Type; - - /** - * A function that will be called after producing message definitions from the schema. Used by - * Foxglove Studio to modify the field name of google.protobuf.Timestamp & - * google.protobuf.Duration. - */ - processMessageDefinitions?: (definitions: MessageDefinitionMap) => MessageDefinitionMap; -}; - -/** - * Parse a Protobuf binary schema (FileDescriptorSet) and produce datatypes and a deserializer - * function. - */ -export function parseProtobufSchema( - schemaName: string, - schemaData: Uint8Array, - options?: ParseProtobufSchemaOptions, -): { - datatypes: MessageDefinitionMap; - deserialize: (buffer: ArrayBufferView) => unknown; -} { - const descriptorSet = FileDescriptorSet.decode(schemaData); - - const root = protobufjs.Root.fromDescriptor(descriptorSet); - root.resolveAll(); - let rootType = root.lookupType(schemaName); - if (options?.processRootType) { - rootType = options.processRootType(rootType); - } - - const deserialize = (data: ArrayBufferView) => { - return rootType.toObject( - rootType.decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength)), - { defaults: true, longs: BigInt }, - ); - }; - - let datatypes: MessageDefinitionMap = new Map(); - protobufDefinitionsToDatatypes(datatypes, rootType); - if (options?.processMessageDefinitions) { - datatypes = options.processMessageDefinitions(datatypes); - } - - if (!datatypes.has(schemaName)) { - throw new Error( - `Protobuf schema does not contain an entry for '${schemaName}'. The schema name should be fully-qualified, e.g. '${stripLeadingDot( - rootType.fullName, - )}'.`, - ); - } - - return { deserialize, datatypes }; -} diff --git a/typescript/support/src/protobufDefinitionsToDatatypes.ts b/typescript/support/src/protobufDefinitionsToDatatypes.ts deleted file mode 100644 index 9b48da07e0..0000000000 --- a/typescript/support/src/protobufDefinitionsToDatatypes.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { MessageDefinitionField } from "@foxglove/message-definition"; -import protobufjs from "@foxglove/protobufjs"; - -import { MessageDefinitionMap } from "./types"; - -function protobufScalarToRosPrimitive(type: string): string { - switch (type) { - case "double": - return "float64"; - case "float": - return "float32"; - case "int32": - case "sint32": - case "sfixed32": - return "int32"; - case "uint32": - case "fixed32": - return "uint32"; - case "int64": - case "sint64": - case "sfixed64": - return "int64"; - case "uint64": - case "fixed64": - return "uint64"; - case "bool": - return "bool"; - case "string": - return "string"; - } - throw new Error(`Expected protobuf scalar type, got ${type}`); -} - -export function stripLeadingDot(typeName: string): string { - return typeName.replace(/^\./, ""); -} - -export function protobufDefinitionsToDatatypes( - datatypes: MessageDefinitionMap, - type: protobufjs.Type, -): void { - const definitions: MessageDefinitionField[] = []; - // The empty list reference is added to the map so a `.has` lookup below can prevent infinite recursion on cyclical types - datatypes.set(stripLeadingDot(type.fullName), { definitions }); - for (const field of type.fieldsArray) { - if (field.resolvedType instanceof protobufjs.Enum) { - for (const [name, value] of Object.entries(field.resolvedType.values)) { - // Note: names from different enums might conflict. The @foxglove/message-definition API - // will need to be updated to associate fields with enums. - // https://github.com/foxglove/studio/issues/2214 - definitions.push({ name, type: "int32", isConstant: true, value }); - } - definitions.push({ type: "int32", name: field.name }); - } else if (field.resolvedType) { - const fullName = stripLeadingDot(field.resolvedType.fullName); - definitions.push({ - type: fullName, - name: field.name, - isComplex: true, - isArray: field.repeated, - }); - - // If we've already processed this datatype we should skip it. - // This avoid infinite recursion with datatypes that reference themselves. - if (!datatypes.has(fullName)) { - protobufDefinitionsToDatatypes(datatypes, field.resolvedType); - } - } else if (field.type === "bytes") { - if (field.repeated) { - throw new Error("Repeated bytes are not currently supported"); - } - definitions.push({ type: "uint8", name: field.name, isArray: true }); - } else { - definitions.push({ - type: protobufScalarToRosPrimitive(field.type), - name: field.name, - isArray: field.repeated, - }); - } - } -} diff --git a/typescript/support/src/protobufDescriptors.ts b/typescript/support/src/protobufDescriptors.ts index 42678d43dc..fb572a7a5b 100644 --- a/typescript/support/src/protobufDescriptors.ts +++ b/typescript/support/src/protobufDescriptors.ts @@ -1,8 +1,8 @@ // eslint-disable-next-line @typescript-eslint/triple-slash-reference /// -import protobufjs from "@foxglove/protobufjs"; -import { FileDescriptorSet } from "@foxglove/protobufjs/ext/descriptor"; +import protobufjs from "protobufjs"; +import { FileDescriptorSet } from "protobufjs/ext/descriptor"; export type ProtobufDescriptor = ReturnType; diff --git a/typescript/support/src/types.ts b/typescript/support/src/types.ts deleted file mode 100644 index 7a540949b4..0000000000 --- a/typescript/support/src/types.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { MessageDefinition } from "@foxglove/message-definition"; - -/** A map of schema name to the schema message definition */ -export type MessageDefinitionMap = Map; diff --git a/typescript/support/typings/protobufjs.d.ts b/typescript/support/typings/protobufjs.d.ts index 61be38e80a..8c0445835e 100644 --- a/typescript/support/typings/protobufjs.d.ts +++ b/typescript/support/typings/protobufjs.d.ts @@ -1,8 +1,8 @@ -import protobufjs from "@foxglove/protobufjs"; -import descriptor from "@foxglove/protobufjs/ext/descriptor"; +import protobufjs from "protobufjs"; +import descriptor from "protobufjs/ext/descriptor"; // https://github.com/protobufjs/protobuf.js/issues/1499 -declare module "@foxglove/protobufjs" { +declare module "protobufjs" { interface ReflectionObject { toDescriptor( protoVersion: string, diff --git a/website/package.json b/website/package.json index d0c5f68edb..b16f3c508b 100644 --- a/website/package.json +++ b/website/package.json @@ -44,7 +44,7 @@ "prism-react-renderer": "1.3.5", "process": "0.11.10", "promise-queue": "2.2.5", - "protobufjs": "7.2.3", + "protobufjs": "7.2.5", "react": "17.0.2", "react-async": "10.0.1", "react-dom": "17.0.2", diff --git a/yarn.lock b/yarn.lock index ab5f4407cf..c44c1296e6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2918,7 +2918,7 @@ __metadata: eslint-plugin-prettier: 5.0.1 lodash: 4.17.21 prettier: 3.1.0 - protobufjs: 7.2.2 + protobufjs: 7.2.5 ts-node: 10.9.1 tsconfig-paths: 4.1.2 typescript: 5.2.2 @@ -3002,7 +3002,9 @@ __metadata: "@foxglove/wasm-bz2": 0.1.1 "@foxglove/wasm-lz4": 1.0.2 "@foxglove/wasm-zstd": 1.0.1 - "@mcap/core": "*" + "@mcap/core": "workspace:*" + "@mcap/nodejs": "workspace:*" + "@mcap/support": "workspace:*" "@types/jest": 29.5.8 "@types/lodash": 4.14.191 "@types/node": 18.13.0 @@ -3019,7 +3021,7 @@ __metadata: jest: 29.7.0 lodash: 4.17.21 prettier: 3.1.0 - protobufjs: 7.2.2 + protobufjs: 7.2.5 ts-jest: 29.1.1 ts-node: 10.9.1 tsconfig-paths: 4.1.2 @@ -3069,26 +3071,6 @@ __metadata: languageName: node linkType: hard -"@foxglove/protobufjs@npm:0.0.1-toobject-bigint.1": - version: 0.0.1-toobject-bigint.1 - resolution: "@foxglove/protobufjs@npm:0.0.1-toobject-bigint.1" - dependencies: - "@protobufjs/aspromise": ^1.1.2 - "@protobufjs/base64": ^1.1.2 - "@protobufjs/codegen": ^2.0.4 - "@protobufjs/eventemitter": ^1.1.0 - "@protobufjs/fetch": ^1.1.0 - "@protobufjs/float": ^1.0.2 - "@protobufjs/inquire": ^1.1.0 - "@protobufjs/path": ^1.1.2 - "@protobufjs/pool": ^1.1.0 - "@protobufjs/utf8": ^1.1.0 - "@types/node": ">=13.7.0" - long: ^5.0.0 - checksum: bce02c9a255116927fe327a7fce1dde8682ff0e742064a0120e7944d346385e28aecc315030c5560afc0208eaf1448d2a36b87f9f424531b7dd06e1b18100808 - languageName: node - linkType: hard - "@foxglove/ros2idl-parser@npm:^0.3.0": version: 0.3.1 resolution: "@foxglove/ros2idl-parser@npm:0.3.1" @@ -3713,7 +3695,6 @@ __metadata: "@foxglove/message-definition": ^0.3.0 "@foxglove/omgidl-parser": ^1.0.0 "@foxglove/omgidl-serialization": ^1.0.0 - "@foxglove/protobufjs": 0.0.1-toobject-bigint.1 "@foxglove/ros2idl-parser": ^0.3.0 "@foxglove/rosmsg": ^4.2.2 "@foxglove/rosmsg-serialization": ^2.0.2 @@ -3740,6 +3721,7 @@ __metadata: flatbuffers_reflection: ^0.0.7 jest: 29.7.0 prettier: 3.1.0 + protobufjs: 7.2.5 ts-jest: 29.1.1 typescript: 5.2.2 languageName: unknown @@ -13009,29 +12991,9 @@ __metadata: languageName: node linkType: hard -"protobufjs@npm:7.2.2": - version: 7.2.2 - resolution: "protobufjs@npm:7.2.2" - dependencies: - "@protobufjs/aspromise": ^1.1.2 - "@protobufjs/base64": ^1.1.2 - "@protobufjs/codegen": ^2.0.4 - "@protobufjs/eventemitter": ^1.1.0 - "@protobufjs/fetch": ^1.1.0 - "@protobufjs/float": ^1.0.2 - "@protobufjs/inquire": ^1.1.0 - "@protobufjs/path": ^1.1.2 - "@protobufjs/pool": ^1.1.0 - "@protobufjs/utf8": ^1.1.0 - "@types/node": ">=13.7.0" - long: ^5.0.0 - checksum: 86166e8f3e46789fa4d117ae72c17356db36bf87c0e0710d224be32e543b1c378a94e66dc2b1de5af45edfc25f56acfc7e688c50c956426a3ae97bc474f4445c - languageName: node - linkType: hard - -"protobufjs@npm:7.2.3": - version: 7.2.3 - resolution: "protobufjs@npm:7.2.3" +"protobufjs@npm:7.2.5": + version: 7.2.5 + resolution: "protobufjs@npm:7.2.5" dependencies: "@protobufjs/aspromise": ^1.1.2 "@protobufjs/base64": ^1.1.2 @@ -13045,7 +13007,7 @@ __metadata: "@protobufjs/utf8": ^1.1.0 "@types/node": ">=13.7.0" long: ^5.0.0 - checksum: 9afa6de5fced0139a5180c063718508fac3ea734a9f1aceb99712367b15473a83327f91193f16b63540f9112b09a40912f5f0441a9b0d3f3c6a1c7f707d78249 + checksum: 3770a072114061faebbb17cfd135bc4e187b66bc6f40cd8bac624368b0270871ec0cfb43a02b9fb4f029c8335808a840f1afba3c2e7ede7063b98ae6b98a703f languageName: node linkType: hard @@ -15929,7 +15891,7 @@ __metadata: prism-react-renderer: 1.3.5 process: 0.11.10 promise-queue: 2.2.5 - protobufjs: 7.2.3 + protobufjs: 7.2.5 react: 17.0.2 react-async: 10.0.1 react-dom: 17.0.2 From 8fdfa6634bb772b72c24c2af5cc42dbd97380696 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 17 Nov 2023 18:41:17 -0800 Subject: [PATCH 23/27] lint & yarn fixes --- .github/workflows/ci.yml | 33 +++++--- .github/workflows/website.yml | 3 +- .vscode/settings.json | 1 + tests/conformance/package.json | 2 +- typescript/benchmarks/package.json | 2 +- typescript/browser/package.json | 6 +- typescript/core/package.json | 2 +- typescript/examples/bag2mcap/package.json | 2 +- typescript/examples/basicwriter/package.json | 2 +- .../examples/flatbufferswriter/package.json | 2 +- .../text-annotation-demo/package.json | 2 +- typescript/examples/validate/package.json | 2 +- typescript/nodejs/package.json | 6 +- typescript/nodejs/src/FileHandleReadable.ts | 16 ++-- typescript/nodejs/src/FileHandleWritable.ts | 12 +-- typescript/support/package.json | 4 +- website/package.json | 2 +- yarn.lock | 78 ++++++++----------- 18 files changed, 89 insertions(+), 88 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 186d00c0c0..44003f4c9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,11 +14,12 @@ jobs: - uses: actions/checkout@v3 with: lfs: true + - run: corepack enable - uses: actions/setup-node@v4 with: node-version: 20.x cache: yarn - - run: corepack enable && yarn install --immutable + - run: yarn install --immutable - run: echo '::add-matcher::.github/cspell-problem-matcher.json' - run: yarn spellcheck --no-progress @@ -28,11 +29,12 @@ jobs: - uses: actions/checkout@v3 with: lfs: true + - run: corepack enable - uses: actions/setup-node@v4 with: node-version: 20.x cache: yarn - - run: corepack enable && yarn install --immutable + - run: yarn install --immutable - run: yarn workspace @foxglove/mcap-conformance lint:ci - run: yarn workspace @foxglove/mcap-conformance build @@ -42,6 +44,7 @@ jobs: - uses: actions/checkout@v3 with: lfs: true + - run: corepack enable - uses: actions/setup-node@v4 with: node-version: 20.x @@ -51,7 +54,7 @@ jobs: path: ~/.conan/data key: ${{ runner.os }}-${{ hashFiles('cpp/**/conanfile.py') }} - run: cd cpp && make ci - - run: corepack enable && yarn install --immutable + - run: yarn install --immutable - run: yarn test:conformance:generate-inputs --verify - run: yarn test:conformance --runner cpp- @@ -61,6 +64,7 @@ jobs: - uses: actions/checkout@v3 with: lfs: true + - run: corepack enable - uses: actions/setup-node@v4 with: node-version: 20.x @@ -69,7 +73,7 @@ jobs: with: go-version-file: go/go.work - run: cd go && make build-conformance-binaries - - run: corepack enable && yarn install --immutable + - run: yarn install --immutable - run: yarn test:conformance:generate-inputs --verify - run: yarn test:conformance --runner go- @@ -79,6 +83,7 @@ jobs: - uses: actions/checkout@v3 with: lfs: true + - run: corepack enable - uses: actions/setup-node@v4 with: node-version: 20.x @@ -87,7 +92,7 @@ jobs: with: python-version: 3.7 - run: cd python && pip install -e mcap - - run: corepack enable && yarn install --immutable + - run: yarn install --immutable - run: yarn test:conformance:generate-inputs --verify - run: yarn test:conformance --runner py- @@ -97,11 +102,12 @@ jobs: - uses: actions/checkout@v3 with: lfs: true + - run: corepack enable - uses: actions/setup-node@v4 with: node-version: 20.x cache: yarn - - run: corepack enable && yarn install --immutable + - run: yarn install --immutable - run: yarn test:conformance:generate-inputs --verify - run: yarn test:conformance --runner ts- @@ -111,11 +117,12 @@ jobs: - uses: actions/checkout@v3 with: lfs: true + - run: corepack enable - uses: actions/setup-node@v4 with: node-version: 20.x cache: yarn - - run: corepack enable && yarn install --immutable + - run: yarn install --immutable - run: yarn test:conformance:generate-inputs --verify - run: yarn test:conformance --runner ksy- @@ -125,6 +132,7 @@ jobs: - uses: actions/checkout@v3 with: lfs: true + - run: corepack enable - uses: actions/setup-node@v4 with: node-version: 20.x @@ -133,7 +141,7 @@ jobs: with: swift-version: "5.7" - run: swift build - - run: corepack enable && yarn install --immutable + - run: yarn install --immutable - run: yarn test:conformance:generate-inputs --verify - run: yarn test:conformance --runner swift- @@ -143,6 +151,7 @@ jobs: - uses: actions/checkout@v3 with: lfs: true + - run: corepack enable - uses: actions/setup-node@v4 with: node-version: 20.x @@ -152,7 +161,7 @@ jobs: toolchain: stable default: true - run: cd rust && cargo build --example=conformance_reader - - run: corepack enable && yarn install --immutable + - run: yarn install --immutable - run: yarn test:conformance:generate-inputs --verify - run: yarn test:conformance --runner rust- @@ -201,12 +210,13 @@ jobs: - uses: actions/checkout@v3 with: lfs: true + - run: corepack enable - uses: actions/setup-node@v4 with: node-version: 20.x cache: yarn registry-url: https://registry.npmjs.org - - run: corepack enable && yarn install --immutable + - run: yarn install --immutable - run: yarn dedupe --check - run: yarn prettier:check - run: yarn workspace @mcap/core lint:ci @@ -250,11 +260,12 @@ jobs: - uses: actions/checkout@v3 with: lfs: true + - run: corepack enable - uses: actions/setup-node@v4 with: node-version: 20.x cache: yarn - - run: corepack enable && yarn install --immutable + - run: yarn install --immutable - run: yarn workspace @foxglove/mcap-benchmarks lint:ci - run: yarn workspace @foxglove/mcap-benchmarks typecheck - run: yarn workspace @foxglove/mcap-example-validate lint:ci diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index 70728449ea..c37b9a7260 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -14,12 +14,13 @@ jobs: with: lfs: true + - run: corepack enable - uses: actions/setup-node@v4 with: node-version: 20.x cache: yarn - - run: corepack enable && yarn install --immutable + - run: yarn install --immutable - run: yarn workspace website lint:ci - run: yarn workspace website typecheck diff --git a/.vscode/settings.json b/.vscode/settings.json index 141330e2a1..de4b464e0c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ // -*- jsonc -*- { + "editor.codeActionsOnSave": { "source.fixAll.eslint": true }, "editor.formatOnSave": true, "editor.defaultFormatter": "esbenp.prettier-vscode", diff --git a/tests/conformance/package.json b/tests/conformance/package.json index 694d964339..19a4f98943 100644 --- a/tests/conformance/package.json +++ b/tests/conformance/package.json @@ -23,7 +23,7 @@ "colors": "1.4.0", "commander": "11.1.0", "diff": "^5.1.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/typescript/benchmarks/package.json b/typescript/benchmarks/package.json index 2129ea854b..be989ae627 100644 --- a/typescript/benchmarks/package.json +++ b/typescript/benchmarks/package.json @@ -28,7 +28,7 @@ "@typescript-eslint/eslint-plugin": "6.11.0", "@typescript-eslint/parser": "6.11.0", "benny": "^3.7.1", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/typescript/browser/package.json b/typescript/browser/package.json index e09e20a61d..b21ec918ac 100644 --- a/typescript/browser/package.json +++ b/typescript/browser/package.json @@ -39,9 +39,9 @@ "@types/jest": "29.5.8", "@typescript-eslint/eslint-plugin": "6.11.0", "@typescript-eslint/parser": "6.11.0", - "eslint": "8.53.0", - "eslint-config-prettier": "8.6.0", - "eslint-import-resolver-typescript": "3.5.5", + "eslint": "8.54.0", + "eslint-config-prettier": "9.0.0", + "eslint-import-resolver-typescript": "3.6.1", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", "eslint-plugin-import": "2.29.0", diff --git a/typescript/core/package.json b/typescript/core/package.json index fefff70cad..11885bb567 100644 --- a/typescript/core/package.json +++ b/typescript/core/package.json @@ -38,7 +38,7 @@ "@types/node": "18.13.0", "@typescript-eslint/eslint-plugin": "6.11.0", "@typescript-eslint/parser": "6.11.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/typescript/examples/bag2mcap/package.json b/typescript/examples/bag2mcap/package.json index 44acec6e6b..b89565afea 100644 --- a/typescript/examples/bag2mcap/package.json +++ b/typescript/examples/bag2mcap/package.json @@ -37,7 +37,7 @@ "@typescript-eslint/eslint-plugin": "6.11.0", "@typescript-eslint/parser": "6.11.0", "commander": "11.1.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/typescript/examples/basicwriter/package.json b/typescript/examples/basicwriter/package.json index 0424c67227..9d53d8492d 100644 --- a/typescript/examples/basicwriter/package.json +++ b/typescript/examples/basicwriter/package.json @@ -25,7 +25,7 @@ "@mcap/nodejs": "workspace:*", "@mcap/support": "workspace:*", "@types/node": "18.13.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/typescript/examples/flatbufferswriter/package.json b/typescript/examples/flatbufferswriter/package.json index 27553db0cf..6292a27321 100644 --- a/typescript/examples/flatbufferswriter/package.json +++ b/typescript/examples/flatbufferswriter/package.json @@ -25,7 +25,7 @@ "@mcap/core": "workspace:*", "@mcap/nodejs": "workspace:*", "@types/node": "18.13.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/typescript/examples/text-annotation-demo/package.json b/typescript/examples/text-annotation-demo/package.json index 36434f65aa..5c15bf0c4b 100644 --- a/typescript/examples/text-annotation-demo/package.json +++ b/typescript/examples/text-annotation-demo/package.json @@ -24,7 +24,7 @@ "@mcap/core": "workspace:*", "@mcap/nodejs": "workspace:*", "@types/node": "18.13.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/typescript/examples/validate/package.json b/typescript/examples/validate/package.json index f43ebac538..2dfb970a9c 100644 --- a/typescript/examples/validate/package.json +++ b/typescript/examples/validate/package.json @@ -38,7 +38,7 @@ "@typescript-eslint/eslint-plugin": "6.11.0", "@typescript-eslint/parser": "6.11.0", "commander": "11.1.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/typescript/nodejs/package.json b/typescript/nodejs/package.json index 253ddddad8..a3e463aabb 100644 --- a/typescript/nodejs/package.json +++ b/typescript/nodejs/package.json @@ -40,9 +40,9 @@ "@types/node": "18.13.0", "@typescript-eslint/eslint-plugin": "6.11.0", "@typescript-eslint/parser": "6.11.0", - "eslint": "8.53.0", - "eslint-config-prettier": "8.6.0", - "eslint-import-resolver-typescript": "3.5.5", + "eslint": "8.54.0", + "eslint-config-prettier": "9.0.0", + "eslint-import-resolver-typescript": "3.6.1", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", "eslint-plugin-import": "2.29.0", diff --git a/typescript/nodejs/src/FileHandleReadable.ts b/typescript/nodejs/src/FileHandleReadable.ts index e09e2cda8d..9de82e3e79 100644 --- a/typescript/nodejs/src/FileHandleReadable.ts +++ b/typescript/nodejs/src/FileHandleReadable.ts @@ -5,26 +5,26 @@ import { FileHandle } from "fs/promises"; * IReadable implementation for FileHandle. */ export class FileHandleReadable implements McapTypes.IReadable { - private handle: FileHandle; - private buffer = new ArrayBuffer(4096); + #handle: FileHandle; + #buffer = new ArrayBuffer(4096); constructor(handle: FileHandle) { - this.handle = handle; + this.#handle = handle; } async size(): Promise { - return BigInt((await this.handle.stat()).size); + return BigInt((await this.#handle.stat()).size); } async read(offset: bigint, length: bigint): Promise { if (offset > Number.MAX_SAFE_INTEGER || length > Number.MAX_SAFE_INTEGER) { throw new Error(`Read too large: offset ${offset}, length ${length}`); } - if (length > this.buffer.byteLength) { - this.buffer = new ArrayBuffer(Number(length * 2n)); + if (length > this.#buffer.byteLength) { + this.#buffer = new ArrayBuffer(Number(length * 2n)); } - const result = await this.handle.read({ - buffer: new DataView(this.buffer, 0, Number(length)), + const result = await this.#handle.read({ + buffer: new DataView(this.#buffer, 0, Number(length)), position: Number(offset), }); if (result.bytesRead !== Number(length)) { diff --git a/typescript/nodejs/src/FileHandleWritable.ts b/typescript/nodejs/src/FileHandleWritable.ts index 4efb7a9588..6b8a7f095b 100644 --- a/typescript/nodejs/src/FileHandleWritable.ts +++ b/typescript/nodejs/src/FileHandleWritable.ts @@ -5,19 +5,19 @@ import { FileHandle } from "fs/promises"; * IWritable implementation for FileHandle. */ export class FileHandleWritable implements IWritable { - private handle: FileHandle; - private totalBytesWritten = 0; + #handle: FileHandle; + #totalBytesWritten = 0; constructor(handle: FileHandle) { - this.handle = handle; + this.#handle = handle; } async write(buffer: Uint8Array): Promise { - const written = await this.handle.write(buffer); - this.totalBytesWritten += written.bytesWritten; + const written = await this.#handle.write(buffer); + this.#totalBytesWritten += written.bytesWritten; } position(): bigint { - return BigInt(this.totalBytesWritten); + return BigInt(this.#totalBytesWritten); } } diff --git a/typescript/support/package.json b/typescript/support/package.json index a71afbbb6d..7da5929633 100644 --- a/typescript/support/package.json +++ b/typescript/support/package.json @@ -39,8 +39,8 @@ "@types/node": "18.13.0", "@typescript-eslint/eslint-plugin": "6.11.0", "@typescript-eslint/parser": "6.11.0", - "eslint": "8.53.0", - "eslint-config-prettier": "8.6.0", + "eslint": "8.54.0", + "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", "eslint-plugin-import": "2.29.0", diff --git a/website/package.json b/website/package.json index b16f3c508b..f71cd7b8e4 100644 --- a/website/package.json +++ b/website/package.json @@ -30,7 +30,7 @@ "@types/promise-queue": "2.2.0", "buffer": "6.0.3", "classnames": "2.3.2", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-config-prettier": "9.0.0", "eslint-plugin-es": "4.1.0", "eslint-plugin-filenames": "1.3.2", diff --git a/yarn.lock b/yarn.lock index c44c1296e6..c8242f1684 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2782,10 +2782,10 @@ __metadata: languageName: node linkType: hard -"@eslint/js@npm:8.53.0": - version: 8.53.0 - resolution: "@eslint/js@npm:8.53.0" - checksum: e0d5cfb0000aaee237c8e6d6d6e366faa60b1ef7f928ce17778373aa44d3b886368f6d5e1f97f913f0f16801aad016db8b8df78418c9d18825c15590328028af +"@eslint/js@npm:8.54.0": + version: 8.54.0 + resolution: "@eslint/js@npm:8.54.0" + checksum: 6d88a6f711ef0133566b5340e3178a178fbb297585766460f195d0a9db85688f1e5cf8559fd5748aeb3131e2096c66595b323d8edab22df015acda68f1ebde92 languageName: node linkType: hard @@ -2840,7 +2840,7 @@ __metadata: "@typescript-eslint/eslint-plugin": 6.11.0 "@typescript-eslint/parser": 6.11.0 benny: ^3.7.1 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -2870,7 +2870,7 @@ __metadata: colors: 1.4.0 commander: 11.1.0 diff: ^5.1.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -2910,7 +2910,7 @@ __metadata: "@typescript-eslint/eslint-plugin": 6.11.0 "@typescript-eslint/parser": 6.11.0 commander: 11.1.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -2934,7 +2934,7 @@ __metadata: "@mcap/nodejs": "workspace:*" "@mcap/support": "workspace:*" "@types/node": 18.13.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -2955,7 +2955,7 @@ __metadata: "@mcap/core": "workspace:*" "@mcap/nodejs": "workspace:*" "@types/node": 18.13.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -2977,7 +2977,7 @@ __metadata: "@mcap/core": "workspace:*" "@mcap/nodejs": "workspace:*" "@types/node": 18.13.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -3011,7 +3011,7 @@ __metadata: "@typescript-eslint/eslint-plugin": 6.11.0 "@typescript-eslint/parser": 6.11.0 commander: 11.1.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -3616,9 +3616,9 @@ __metadata: "@types/jest": 29.5.8 "@typescript-eslint/eslint-plugin": 6.11.0 "@typescript-eslint/parser": 6.11.0 - eslint: 8.53.0 - eslint-config-prettier: 8.6.0 - eslint-import-resolver-typescript: 3.5.5 + eslint: 8.54.0 + eslint-config-prettier: 9.0.0 + eslint-import-resolver-typescript: 3.6.1 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 eslint-plugin-import: 2.29.0 @@ -3643,7 +3643,7 @@ __metadata: "@types/node": 18.13.0 "@typescript-eslint/eslint-plugin": 6.11.0 "@typescript-eslint/parser": 6.11.0 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 @@ -3672,9 +3672,9 @@ __metadata: "@types/node": 18.13.0 "@typescript-eslint/eslint-plugin": 6.11.0 "@typescript-eslint/parser": 6.11.0 - eslint: 8.53.0 - eslint-config-prettier: 8.6.0 - eslint-import-resolver-typescript: 3.5.5 + eslint: 8.54.0 + eslint-config-prettier: 9.0.0 + eslint-import-resolver-typescript: 3.6.1 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 eslint-plugin-import: 2.29.0 @@ -3710,8 +3710,8 @@ __metadata: "@types/node": 18.13.0 "@typescript-eslint/eslint-plugin": 6.11.0 "@typescript-eslint/parser": 6.11.0 - eslint: 8.53.0 - eslint-config-prettier: 8.6.0 + eslint: 8.54.0 + eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 eslint-plugin-import: 2.29.0 @@ -7778,17 +7778,6 @@ __metadata: languageName: node linkType: hard -"eslint-config-prettier@npm:8.6.0": - version: 8.6.0 - resolution: "eslint-config-prettier@npm:8.6.0" - peerDependencies: - eslint: ">=7.0.0" - bin: - eslint-config-prettier: bin/cli.js - checksum: ff0d0dfc839a556355422293428637e8d35693de58dabf8638bf0b6529131a109d0b2ade77521aa6e54573bb842d7d9d322e465dd73dd61c7590fa3834c3fa81 - languageName: node - linkType: hard - "eslint-config-prettier@npm:9.0.0": version: 9.0.0 resolution: "eslint-config-prettier@npm:9.0.0" @@ -7811,22 +7800,21 @@ __metadata: languageName: node linkType: hard -"eslint-import-resolver-typescript@npm:3.5.5": - version: 3.5.5 - resolution: "eslint-import-resolver-typescript@npm:3.5.5" +"eslint-import-resolver-typescript@npm:3.6.1": + version: 3.6.1 + resolution: "eslint-import-resolver-typescript@npm:3.6.1" dependencies: debug: ^4.3.4 enhanced-resolve: ^5.12.0 eslint-module-utils: ^2.7.4 + fast-glob: ^3.3.1 get-tsconfig: ^4.5.0 - globby: ^13.1.3 is-core-module: ^2.11.0 is-glob: ^4.0.3 - synckit: ^0.8.5 peerDependencies: eslint: "*" eslint-plugin-import: "*" - checksum: 27e6276fdff5d377c9036362ff736ac29852106e883ff589ea9092dc57d4bc2a67a82d75134221124f05045f9a7e2114a159b2c827d1f9f64d091f7afeab0f58 + checksum: 454fa0646533050fb57f13d27daf8c71f51b0bb9156d6a461290ccb8576d892209fcc6702a89553f3f5ea8e5b407395ca2e5de169a952c953685f1f7c46b4496 languageName: node linkType: hard @@ -8010,14 +7998,14 @@ __metadata: languageName: node linkType: hard -"eslint@npm:8.53.0": - version: 8.53.0 - resolution: "eslint@npm:8.53.0" +"eslint@npm:8.54.0": + version: 8.54.0 + resolution: "eslint@npm:8.54.0" dependencies: "@eslint-community/eslint-utils": ^4.2.0 "@eslint-community/regexpp": ^4.6.1 "@eslint/eslintrc": ^2.1.3 - "@eslint/js": 8.53.0 + "@eslint/js": 8.54.0 "@humanwhocodes/config-array": ^0.11.13 "@humanwhocodes/module-importer": ^1.0.1 "@nodelib/fs.walk": ^1.2.8 @@ -8054,7 +8042,7 @@ __metadata: text-table: ^0.2.0 bin: eslint: bin/eslint.js - checksum: 2da808655c7aa4b33f8970ba30d96b453c3071cc4d6cd60d367163430677e32ff186b65270816b662d29139283138bff81f28dddeb2e73265495245a316ed02c + checksum: 7e876e9da2a18a017271cf3733d05a3dfbbe469272d75753408c6ea5b1646c71c6bb18cb91e10ca930144c32c1ce3701e222f1ae6784a3975a69f8f8aa68e49f languageName: node linkType: hard @@ -8300,7 +8288,7 @@ __metadata: languageName: node linkType: hard -"fast-glob@npm:^3.2.11, fast-glob@npm:^3.2.9, fast-glob@npm:^3.3.0, fast-glob@npm:^3.3.2": +"fast-glob@npm:^3.2.11, fast-glob@npm:^3.2.9, fast-glob@npm:^3.3.0, fast-glob@npm:^3.3.1, fast-glob@npm:^3.3.2": version: 3.3.2 resolution: "fast-glob@npm:3.3.2" dependencies: @@ -8961,7 +8949,7 @@ __metadata: languageName: node linkType: hard -"globby@npm:^13.1.1, globby@npm:^13.1.3": +"globby@npm:^13.1.1": version: 13.2.2 resolution: "globby@npm:13.2.2" dependencies: @@ -15877,7 +15865,7 @@ __metadata: "@types/promise-queue": 2.2.0 buffer: 6.0.3 classnames: 2.3.2 - eslint: 8.53.0 + eslint: 8.54.0 eslint-config-prettier: 9.0.0 eslint-plugin-es: 4.1.0 eslint-plugin-filenames: 1.3.2 From 7f4f110dae8664a8728e7a62c2dbccff798dc17d Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 17 Nov 2023 18:43:00 -0800 Subject: [PATCH 24/27] update readmes --- typescript/README.md | 2 +- typescript/support/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/typescript/README.md b/typescript/README.md index e2b8e6469d..ebcbd87b25 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -5,6 +5,6 @@ The following NPM packages are provided for use with JavaScript and TypeScript: - **@mcap/core** – low-level readers and writers -- **@mcap/support** – support for well-known compression formats and encodings +- **@mcap/support** – support for well-known compression formats - **@mcap/nodejs** – support for Node.js environment - **@mcap/browser** – support for browser environment diff --git a/typescript/support/README.md b/typescript/support/README.md index 51dccec9ca..d22885fd8a 100644 --- a/typescript/support/README.md +++ b/typescript/support/README.md @@ -2,7 +2,7 @@ [MCAP](https://mcap.dev/) is a modular container format and logging library for pub/sub messages with arbitrary message serialization. It is primarily intended for use in robotics applications, and works well under various workloads, resource constraints, and durability requirements. -The `@mcap/support` package provides utilities for working with MCAP files that use [well-known compression formats and encodings](https://mcap.dev/specification/appendix.html), from Node.js and browsers. +The `@mcap/support` package provides utilities for working with MCAP files that use [well-known compression formats](https://mcap.dev/specification/appendix.html), from Node.js and browsers. ## Usage examples From 2d602c165499a89606e4231ff9a943d5a6077dd3 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 17 Nov 2023 18:46:41 -0800 Subject: [PATCH 25/27] cleanup --- .gitattributes | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index ec931e6b46..f889453344 100644 --- a/.gitattributes +++ b/.gitattributes @@ -12,4 +12,3 @@ python/mcap-ros1-support/mcap_ros1/vendor/** linguist-vendored=true tests/conformance/data/** linguist-generated=true typescript/examples/flatbuffer/output/** linguist-generated=true go/ros/testdata/markers.bz2.bag filter=lfs diff=lfs merge=lfs -text -typescript/support/src/fixtures/byte-vector.ts linguist-generated=true From 718bfd5890006e70bcd0c2f9e7a33bcb9fb8ec77 Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 17 Nov 2023 18:50:35 -0800 Subject: [PATCH 26/27] prepare for publishing --- .github/workflows/ci.yml | 35 ++-- package.json | 1 + tests/conformance/package.json | 2 +- typescript/benchmarks/package.json | 2 +- typescript/browser/package.json | 4 +- typescript/core/package.json | 2 +- typescript/examples/bag2mcap/package.json | 2 +- typescript/examples/basicwriter/package.json | 2 +- .../examples/flatbufferswriter/package.json | 2 +- .../text-annotation-demo/package.json | 2 +- typescript/examples/validate/package.json | 2 +- typescript/nodejs/package.json | 4 +- typescript/support/package.json | 19 +-- website/package.json | 2 +- yarn.lock | 152 +++++------------- 15 files changed, 74 insertions(+), 159 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44003f4c9b..a57cce882d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -235,24 +235,23 @@ jobs: env: YARN_NPM_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} - # To be enabled once API has settled - # - name: Publish @mcap/support to NPM - # if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/support/v') }} - # run: yarn workspace @mcap/support publish --access public - # env: - # NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} - - # - name: Publish @mcap/nodejs to NPM - # if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/nodejs/v') }} - # run: yarn workspace @mcap/nodejs publish --access public - # env: - # NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} - - # - name: Publish @mcap/browser to NPM - # if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/browser/v') }} - # run: yarn workspace @mcap/browser publish --access public - # env: - # NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + - name: Publish @mcap/support to NPM + if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/support/v') }} + run: yarn workspace @mcap/support publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + + - name: Publish @mcap/nodejs to NPM + if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/nodejs/v') }} + run: yarn workspace @mcap/nodejs publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + + - name: Publish @mcap/browser to NPM + if: ${{ startsWith(github.ref, 'refs/tags/releases/typescript/browser/v') }} + run: yarn workspace @mcap/browser publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} typescript-examples: runs-on: ubuntu-latest diff --git a/package.json b/package.json index 73f0753a4a..3a45234c1f 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ }, "packageManager": "yarn@3.5.0", "devDependencies": { + "@types/node": "20.9.1", "cspell": "8.0.0", "jest": "29.7.0", "prettier": "3.1.0", diff --git a/tests/conformance/package.json b/tests/conformance/package.json index 19a4f98943..a7bb815dde 100644 --- a/tests/conformance/package.json +++ b/tests/conformance/package.json @@ -11,7 +11,7 @@ }, "devDependencies": { "@foxglove/crc": "^0.0.3", - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/tsconfig": "1.1.0", "@mcap/core": "workspace:*", "@mcap/support": "workspace:*", diff --git a/typescript/benchmarks/package.json b/typescript/benchmarks/package.json index be989ae627..925a49dce3 100644 --- a/typescript/benchmarks/package.json +++ b/typescript/benchmarks/package.json @@ -21,7 +21,7 @@ "bench:debug": "NODE_OPTIONS='--inspect-brk' ts-node --files --project tsconfig.cjs.json index.ts" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/tsconfig": "1.1.0", "@mcap/core": "*", "@types/node": "18.13.0", diff --git a/typescript/browser/package.json b/typescript/browser/package.json index b21ec918ac..786bba60c4 100644 --- a/typescript/browser/package.json +++ b/typescript/browser/package.json @@ -1,6 +1,6 @@ { "name": "@mcap/browser", - "version": "0.1.0", + "version": "1.0.0", "description": "Support library for using MCAP in the browser", "license": "MIT", "repository": { @@ -33,7 +33,7 @@ "test": "jest" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/tsconfig": "1.1.0", "@mcap/core": "workspace:*", "@types/jest": "29.5.8", diff --git a/typescript/core/package.json b/typescript/core/package.json index 11885bb567..bbaea2cc50 100644 --- a/typescript/core/package.json +++ b/typescript/core/package.json @@ -31,7 +31,7 @@ "test": "jest" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/tsconfig": "1.1.0", "@types/jest": "29.5.8", "@types/lodash": "4.14.191", diff --git a/typescript/examples/bag2mcap/package.json b/typescript/examples/bag2mcap/package.json index b89565afea..658365ef80 100644 --- a/typescript/examples/bag2mcap/package.json +++ b/typescript/examples/bag2mcap/package.json @@ -20,7 +20,7 @@ "bag2mcap": "yarn typescript:build && ts-node --files --project tsconfig.cjs.json scripts/bag2mcap.ts" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/rosbag": "0.2.3", "@foxglove/rosmsg": "3.1.0", "@foxglove/rosmsg-serialization": "1.5.3", diff --git a/typescript/examples/basicwriter/package.json b/typescript/examples/basicwriter/package.json index 9d53d8492d..ab294dbca7 100644 --- a/typescript/examples/basicwriter/package.json +++ b/typescript/examples/basicwriter/package.json @@ -20,7 +20,7 @@ "main": "yarn typescript:build && ts-node --files --project tsconfig.cjs.json scripts/main.ts" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@mcap/core": "workspace:*", "@mcap/nodejs": "workspace:*", "@mcap/support": "workspace:*", diff --git a/typescript/examples/flatbufferswriter/package.json b/typescript/examples/flatbufferswriter/package.json index 6292a27321..1d0221bc82 100644 --- a/typescript/examples/flatbufferswriter/package.json +++ b/typescript/examples/flatbufferswriter/package.json @@ -20,7 +20,7 @@ "main": "yarn typescript:build && ts-node --files --project tsconfig.cjs.json scripts/main.ts" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/schemas": "^1.0.0", "@mcap/core": "workspace:*", "@mcap/nodejs": "workspace:*", diff --git a/typescript/examples/text-annotation-demo/package.json b/typescript/examples/text-annotation-demo/package.json index 5c15bf0c4b..a56a28853c 100644 --- a/typescript/examples/text-annotation-demo/package.json +++ b/typescript/examples/text-annotation-demo/package.json @@ -20,7 +20,7 @@ "main": "ts-node --files --project tsconfig.cjs.json scripts/main.ts" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@mcap/core": "workspace:*", "@mcap/nodejs": "workspace:*", "@types/node": "18.13.0", diff --git a/typescript/examples/validate/package.json b/typescript/examples/validate/package.json index 2dfb970a9c..cdea642010 100644 --- a/typescript/examples/validate/package.json +++ b/typescript/examples/validate/package.json @@ -20,7 +20,7 @@ "validate": "yarn typescript:build && ts-node --files --project tsconfig.cjs.json scripts/validate.ts" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/rosbag": "0.2.3", "@foxglove/rosmsg": "3.1.0", "@foxglove/rosmsg-serialization": "1.5.3", diff --git a/typescript/nodejs/package.json b/typescript/nodejs/package.json index a3e463aabb..f1f27f763a 100644 --- a/typescript/nodejs/package.json +++ b/typescript/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "@mcap/nodejs", - "version": "0.1.0", + "version": "1.0.0", "description": "Support library for using MCAP in Node.js", "license": "MIT", "repository": { @@ -33,7 +33,7 @@ "test": "jest" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/tsconfig": "1.1.0", "@mcap/core": "workspace:*", "@types/jest": "29.5.8", diff --git a/typescript/support/package.json b/typescript/support/package.json index 7da5929633..cdb91e4f6e 100644 --- a/typescript/support/package.json +++ b/typescript/support/package.json @@ -1,7 +1,7 @@ { "name": "@mcap/support", - "version": "0.0.1", - "description": "Common schema and message parsing logic for use with MCAP and related protocols", + "version": "1.0.1", + "description": "Common decompression for use with MCAP files", "license": "MIT", "repository": { "type": "git", @@ -32,7 +32,7 @@ "test": "jest" }, "devDependencies": { - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/tsconfig": "1.1.0", "@mcap/core": "workspace:*", "@types/jest": "29.5.8", @@ -52,20 +52,9 @@ "typescript": "5.2.2" }, "dependencies": { - "@foxglove/message-definition": "^0.3.0", - "@foxglove/omgidl-parser": "^1.0.0", - "@foxglove/omgidl-serialization": "^1.0.0", - "@foxglove/ros2idl-parser": "^0.3.0", - "@foxglove/rosmsg": "^4.2.2", - "@foxglove/rosmsg-serialization": "^2.0.2", - "@foxglove/rosmsg2-serialization": "^2.0.2", - "@foxglove/schemas": "^1.5.1", "@foxglove/wasm-bz2": "^0.1.1", "@foxglove/wasm-lz4": "^1.0.2", "@foxglove/wasm-zstd": "^1.0.1", - "@protobufjs/base64": "^1.1.2", - "flatbuffers": "^23.5.26", - "flatbuffers_reflection": "^0.0.7", - "protobufjs": "7.2.5" + "protobufjs": "^7.2.5" } } diff --git a/website/package.json b/website/package.json index f71cd7b8e4..bdfb4fa619 100644 --- a/website/package.json +++ b/website/package.json @@ -20,7 +20,7 @@ "@docusaurus/core": "2.4.1", "@docusaurus/module-type-aliases": "2.4.1", "@docusaurus/preset-classic": "2.4.1", - "@foxglove/eslint-plugin": "1.0.0", + "@foxglove/eslint-plugin": "1.0.1", "@foxglove/rostime": "1.1.2", "@foxglove/schemas": "1.6.2", "@foxglove/tsconfig": "2.0.0", diff --git a/yarn.lock b/yarn.lock index c8242f1684..dba31dc017 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2789,13 +2789,6 @@ __metadata: languageName: node linkType: hard -"@foxglove/cdr@npm:3.2.0, @foxglove/cdr@npm:^3.0.0": - version: 3.2.0 - resolution: "@foxglove/cdr@npm:3.2.0" - checksum: 1c8497debd32714bbe26724df1dbde783fddd6e8ef2e126df87f10221691419c1d8f1fdfb6dd9fd066408638aaa325d41aa6ccc9d1291c3d3774f2b844c9e546 - languageName: node - linkType: hard - "@foxglove/cdr@npm:^2.0.0": version: 2.0.0 resolution: "@foxglove/cdr@npm:2.0.0" @@ -2810,22 +2803,22 @@ __metadata: languageName: node linkType: hard -"@foxglove/eslint-plugin@npm:1.0.0": - version: 1.0.0 - resolution: "@foxglove/eslint-plugin@npm:1.0.0" +"@foxglove/eslint-plugin@npm:1.0.1": + version: 1.0.1 + resolution: "@foxglove/eslint-plugin@npm:1.0.1" dependencies: "@typescript-eslint/utils": ^6 tsutils: ^3 typescript: ^4 || ^5 peerDependencies: eslint: ^7 || ^8 - eslint-config-prettier: ^8 + eslint-config-prettier: ^8 || ^9 eslint-plugin-es: ^4 eslint-plugin-filenames: ^1 eslint-plugin-import: ^2 eslint-plugin-prettier: ^5 prettier: ^3 - checksum: 455819c6d2a0de3901fee957688433d631dd7ecd5ad691c838b853cd63a1e2e94362b0b065e83410d979deccabea5959b46d5de994006d80d287c3afc3e4aed0 + checksum: 53a802c11a373545932671151c1ab432fd3c9af9eef124a462ceff1bbc0c04605dfba72e7e078c38f56ff873baa8e135a24440ad1b69ef7d2fd78239795d9589 languageName: node linkType: hard @@ -2833,7 +2826,7 @@ __metadata: version: 0.0.0-use.local resolution: "@foxglove/mcap-benchmarks@workspace:typescript/benchmarks" dependencies: - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/tsconfig": 1.1.0 "@mcap/core": "*" "@types/node": 18.13.0 @@ -2858,7 +2851,7 @@ __metadata: resolution: "@foxglove/mcap-conformance@workspace:tests/conformance" dependencies: "@foxglove/crc": ^0.0.3 - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/tsconfig": 1.1.0 "@mcap/core": "workspace:*" "@mcap/support": "workspace:*" @@ -2893,7 +2886,7 @@ __metadata: version: 0.0.0-use.local resolution: "@foxglove/mcap-example-bag2mcap@workspace:typescript/examples/bag2mcap" dependencies: - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/rosbag": 0.2.3 "@foxglove/rosmsg": 3.1.0 "@foxglove/rosmsg-serialization": 1.5.3 @@ -2929,7 +2922,7 @@ __metadata: version: 0.0.0-use.local resolution: "@foxglove/mcap-example-basicwriter@workspace:typescript/examples/basicwriter" dependencies: - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@mcap/core": "workspace:*" "@mcap/nodejs": "workspace:*" "@mcap/support": "workspace:*" @@ -2950,7 +2943,7 @@ __metadata: version: 0.0.0-use.local resolution: "@foxglove/mcap-example-flatbufferswriter@workspace:typescript/examples/flatbufferswriter" dependencies: - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/schemas": ^1.0.0 "@mcap/core": "workspace:*" "@mcap/nodejs": "workspace:*" @@ -2972,7 +2965,7 @@ __metadata: version: 0.0.0-use.local resolution: "@foxglove/mcap-example-text-annotation-demo@workspace:typescript/examples/text-annotation-demo" dependencies: - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/schemas": 1.3.0 "@mcap/core": "workspace:*" "@mcap/nodejs": "workspace:*" @@ -2993,7 +2986,7 @@ __metadata: version: 0.0.0-use.local resolution: "@foxglove/mcap-example-validate@workspace:typescript/examples/validate" dependencies: - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/rosbag": 0.2.3 "@foxglove/rosmsg": 3.1.0 "@foxglove/rosmsg-serialization": 1.5.3 @@ -3036,52 +3029,6 @@ __metadata: languageName: node linkType: hard -"@foxglove/message-definition@npm:^0.3.0": - version: 0.3.1 - resolution: "@foxglove/message-definition@npm:0.3.1" - checksum: 71699bc9dafc90d07ce312bcaebbedc1f60a173c8ba66421d741cf713ea6bbf6b462b551fcfe6da86488814629639e1dc424e9664c7ea2474227a20ad88ac776 - languageName: node - linkType: hard - -"@foxglove/omgidl-parser@npm:1.0.1": - version: 1.0.1 - resolution: "@foxglove/omgidl-parser@npm:1.0.1" - dependencies: - tslib: ^2 - checksum: 4227cc13d93b1b90a6e7c60c83e2585153df8686b9db93dfb4accec4522461f20ac7152e8c46380043df2f8236835d86f0fd66eb7ea5d8e86df248f713f0358a - languageName: node - linkType: hard - -"@foxglove/omgidl-parser@npm:^1.0.0": - version: 1.0.2 - resolution: "@foxglove/omgidl-parser@npm:1.0.2" - dependencies: - tslib: ^2 - checksum: 4a2d217c83be17de8a1f4a84dddf4a74459162930b79f5ddfcb334a8af245067254f289bf6e3067c55405b863c38abc010b24efefefeda7504e2c6d2c265939b - languageName: node - linkType: hard - -"@foxglove/omgidl-serialization@npm:^1.0.0": - version: 1.0.2 - resolution: "@foxglove/omgidl-serialization@npm:1.0.2" - dependencies: - "@foxglove/cdr": 3.2.0 - "@foxglove/message-definition": ^0.2.0 - checksum: d03c9cd3ea68a41ad5b0696d3519bf435f5bbb7dfccf2750d593d2b51d3ae796070b52c57f41941c5957d53ccb0c43d0f5800b345aa6a7d7cf1fc7bfdd0446c3 - languageName: node - linkType: hard - -"@foxglove/ros2idl-parser@npm:^0.3.0": - version: 0.3.1 - resolution: "@foxglove/ros2idl-parser@npm:0.3.1" - dependencies: - "@foxglove/message-definition": ^0.2.0 - "@foxglove/omgidl-parser": 1.0.1 - md5-typescript: ^1.0.5 - checksum: cd80bc14d90a1c1ae99967ae177a34f27151b02760e128823cf06dd1054bd510dfba7eee285f69af7f482398334ae330dcf883cac45ce59f15ea3d7a451a0394 - languageName: node - linkType: hard - "@foxglove/rosbag@npm:0.2.3": version: 0.2.3 resolution: "@foxglove/rosbag@npm:0.2.3" @@ -3113,15 +3060,6 @@ __metadata: languageName: node linkType: hard -"@foxglove/rosmsg-serialization@npm:^2.0.2": - version: 2.0.2 - resolution: "@foxglove/rosmsg-serialization@npm:2.0.2" - dependencies: - "@foxglove/message-definition": ^0.2.0 - checksum: 066172af1ab659379ce50735f4627ecb45f2d75313fe3bba4e19f1e7c9f3f17f265492558c80669aa60ce1aa76539e5be3433b1ce59c0f78743d773e21a0c947 - languageName: node - linkType: hard - "@foxglove/rosmsg2-serialization@npm:1.1.1": version: 1.1.1 resolution: "@foxglove/rosmsg2-serialization@npm:1.1.1" @@ -3133,17 +3071,6 @@ __metadata: languageName: node linkType: hard -"@foxglove/rosmsg2-serialization@npm:^2.0.2": - version: 2.0.3 - resolution: "@foxglove/rosmsg2-serialization@npm:2.0.3" - dependencies: - "@foxglove/cdr": ^3.0.0 - "@foxglove/message-definition": ^0.2.0 - "@foxglove/rostime": ^1.1.2 - checksum: 24fb27ca90db82a868b752c0b904b766f8ff26254b1f42609c318001f0fb94fd32b056745fa12fa575c72a4d5be60413b9055d88afd94a9a5867aeb7f2420675 - languageName: node - linkType: hard - "@foxglove/rosmsg@npm:3.1.0, @foxglove/rosmsg@npm:^3.1.0": version: 3.1.0 resolution: "@foxglove/rosmsg@npm:3.1.0" @@ -3153,7 +3080,7 @@ __metadata: languageName: node linkType: hard -"@foxglove/rosmsg@npm:^4.0.0, @foxglove/rosmsg@npm:^4.2.2": +"@foxglove/rosmsg@npm:^4.0.0": version: 4.2.2 resolution: "@foxglove/rosmsg@npm:4.2.2" dependencies: @@ -3182,7 +3109,7 @@ __metadata: languageName: node linkType: hard -"@foxglove/schemas@npm:1.6.2, @foxglove/schemas@npm:^1.0.0, @foxglove/schemas@npm:^1.5.1": +"@foxglove/schemas@npm:1.6.2, @foxglove/schemas@npm:^1.0.0": version: 1.6.2 resolution: "@foxglove/schemas@npm:1.6.2" dependencies: @@ -3610,7 +3537,7 @@ __metadata: version: 0.0.0-use.local resolution: "@mcap/browser@workspace:typescript/browser" dependencies: - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/tsconfig": 1.1.0 "@mcap/core": "workspace:*" "@types/jest": 29.5.8 @@ -3636,7 +3563,7 @@ __metadata: resolution: "@mcap/core@workspace:typescript/core" dependencies: "@foxglove/crc": ^0.0.3 - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/tsconfig": 1.1.0 "@types/jest": 29.5.8 "@types/lodash": 4.14.191 @@ -3665,7 +3592,7 @@ __metadata: version: 0.0.0-use.local resolution: "@mcap/nodejs@workspace:typescript/nodejs" dependencies: - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/tsconfig": 1.1.0 "@mcap/core": "workspace:*" "@types/jest": 29.5.8 @@ -3691,21 +3618,12 @@ __metadata: version: 0.0.0-use.local resolution: "@mcap/support@workspace:typescript/support" dependencies: - "@foxglove/eslint-plugin": 1.0.0 - "@foxglove/message-definition": ^0.3.0 - "@foxglove/omgidl-parser": ^1.0.0 - "@foxglove/omgidl-serialization": ^1.0.0 - "@foxglove/ros2idl-parser": ^0.3.0 - "@foxglove/rosmsg": ^4.2.2 - "@foxglove/rosmsg-serialization": ^2.0.2 - "@foxglove/rosmsg2-serialization": ^2.0.2 - "@foxglove/schemas": ^1.5.1 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/tsconfig": 1.1.0 "@foxglove/wasm-bz2": ^0.1.1 "@foxglove/wasm-lz4": ^1.0.2 "@foxglove/wasm-zstd": ^1.0.1 "@mcap/core": "workspace:*" - "@protobufjs/base64": ^1.1.2 "@types/jest": 29.5.8 "@types/node": 18.13.0 "@typescript-eslint/eslint-plugin": 6.11.0 @@ -3717,11 +3635,9 @@ __metadata: eslint-plugin-import: 2.29.0 eslint-plugin-jest: 27.6.0 eslint-plugin-prettier: 5.0.1 - flatbuffers: ^23.5.26 - flatbuffers_reflection: ^0.0.7 jest: 29.7.0 prettier: 3.1.0 - protobufjs: 7.2.5 + protobufjs: ^7.2.5 ts-jest: 29.1.1 typescript: 5.2.2 languageName: unknown @@ -4463,6 +4379,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:20.9.1": + version: 20.9.1 + resolution: "@types/node@npm:20.9.1" + dependencies: + undici-types: ~5.26.4 + checksum: bb893c6790733dac32818c1ca170fa466622dec39a0ade4639463e1358cb811771e242accbd065e7a1bfe59adc989c0ee59be65e462d3a0ab49043426f0b7637 + languageName: node + linkType: hard + "@types/node@npm:^17.0.5": version: 17.0.45 resolution: "@types/node@npm:17.0.45" @@ -8513,20 +8438,13 @@ __metadata: languageName: node linkType: hard -"flatbuffers@npm:^23.1.21, flatbuffers@npm:^23.5.26": +"flatbuffers@npm:^23.1.21": version: 23.5.26 resolution: "flatbuffers@npm:23.5.26" checksum: 2528e549118d02fca9775cc9c17cf445741bb58b0a9575eb07bba823394a79dccb3bf97c48561720aa61d3dc9f42759129df0e05eb94c12f809cfb6126eaedd1 languageName: node linkType: hard -"flatbuffers_reflection@npm:^0.0.7": - version: 0.0.7 - resolution: "flatbuffers_reflection@npm:0.0.7" - checksum: 09e075ddb655f76b910472392a501c47bb50840c9f921c5d1d567a9fb472d77166b584b62354bc8f4423fa2606040c87b8c6c09942068e5c08bc5a370ddbf891 - languageName: node - linkType: hard - "flatted@npm:^3.2.9": version: 3.2.9 resolution: "flatted@npm:3.2.9" @@ -12979,7 +12897,7 @@ __metadata: languageName: node linkType: hard -"protobufjs@npm:7.2.5": +"protobufjs@npm:7.2.5, protobufjs@npm:^7.2.5": version: 7.2.5 resolution: "protobufjs@npm:7.2.5" dependencies: @@ -13785,6 +13703,7 @@ __metadata: version: 0.0.0-use.local resolution: "root-workspace-0b6124@workspace:." dependencies: + "@types/node": 20.9.1 cspell: 8.0.0 jest: 29.7.0 prettier: 3.1.0 @@ -15198,6 +15117,13 @@ __metadata: languageName: node linkType: hard +"undici-types@npm:~5.26.4": + version: 5.26.5 + resolution: "undici-types@npm:5.26.5" + checksum: 3192ef6f3fd5df652f2dc1cd782b49d6ff14dc98e5dced492aa8a8c65425227da5da6aafe22523c67f035a272c599bb89cfe803c1db6311e44bed3042fc25487 + languageName: node + linkType: hard + "unherit@npm:^1.0.4": version: 1.1.3 resolution: "unherit@npm:1.1.3" @@ -15855,7 +15781,7 @@ __metadata: "@docusaurus/core": 2.4.1 "@docusaurus/module-type-aliases": 2.4.1 "@docusaurus/preset-classic": 2.4.1 - "@foxglove/eslint-plugin": 1.0.0 + "@foxglove/eslint-plugin": 1.0.1 "@foxglove/rostime": 1.1.2 "@foxglove/schemas": 1.6.2 "@foxglove/tsconfig": 2.0.0 From b3467d16cabd979726796d8f5d0ea7d5b8e3326c Mon Sep 17 00:00:00 2001 From: Jacob Bandes-Storch Date: Fri, 17 Nov 2023 18:54:19 -0800 Subject: [PATCH 27/27] yarn dedupe --- package.json | 2 +- yarn.lock | 18 +----------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index 3a45234c1f..2a936cda28 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ }, "packageManager": "yarn@3.5.0", "devDependencies": { - "@types/node": "20.9.1", + "@types/node": "18.13.0", "cspell": "8.0.0", "jest": "29.7.0", "prettier": "3.1.0", diff --git a/yarn.lock b/yarn.lock index dba31dc017..cd61613919 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4379,15 +4379,6 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:20.9.1": - version: 20.9.1 - resolution: "@types/node@npm:20.9.1" - dependencies: - undici-types: ~5.26.4 - checksum: bb893c6790733dac32818c1ca170fa466622dec39a0ade4639463e1358cb811771e242accbd065e7a1bfe59adc989c0ee59be65e462d3a0ab49043426f0b7637 - languageName: node - linkType: hard - "@types/node@npm:^17.0.5": version: 17.0.45 resolution: "@types/node@npm:17.0.45" @@ -13703,7 +13694,7 @@ __metadata: version: 0.0.0-use.local resolution: "root-workspace-0b6124@workspace:." dependencies: - "@types/node": 20.9.1 + "@types/node": 18.13.0 cspell: 8.0.0 jest: 29.7.0 prettier: 3.1.0 @@ -15117,13 +15108,6 @@ __metadata: languageName: node linkType: hard -"undici-types@npm:~5.26.4": - version: 5.26.5 - resolution: "undici-types@npm:5.26.5" - checksum: 3192ef6f3fd5df652f2dc1cd782b49d6ff14dc98e5dced492aa8a8c65425227da5da6aafe22523c67f035a272c599bb89cfe803c1db6311e44bed3042fc25487 - languageName: node - linkType: hard - "unherit@npm:^1.0.4": version: 1.1.3 resolution: "unherit@npm:1.1.3"