From d55d08315f844c19523ac9bf2bb1d6f940d4e160 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Wed, 6 Apr 2022 12:38:11 -0400 Subject: [PATCH] feat: XIP-5 support different payload content types (#68) BREAKING CHANGE: The protocol format of messages is changing, old clients won't be able to decode new messages correctly and new clients won't be able to decode old messages correctly. The API changes are backward compatible. --- README.md | 31 +- package-lock.json | 11 + package.json | 1 + src/Client.ts | 132 ++++- src/Message.ts | 29 +- src/MessageContent.ts | 204 ++++++++ src/Stream.ts | 2 +- src/conversations/Conversation.ts | 11 +- src/index.ts | 23 +- src/proto/messaging.proto | 82 ++- src/proto/messaging.ts | 706 ++++++++++++++++++++------ src/types/streams-polyfill/index.d.ts | 13 + test/Client.test.ts | 105 +++- test/ContentTypeTestKey.ts | 27 + test/Message.test.ts | 15 +- test/MessageContent.test.ts | 95 ++++ test/helpers.ts | 33 +- 17 files changed, 1294 insertions(+), 226 deletions(-) create mode 100644 src/MessageContent.ts create mode 100644 src/types/streams-polyfill/index.d.ts create mode 100644 test/ContentTypeTestKey.ts create mode 100644 test/MessageContent.test.ts diff --git a/README.md b/README.md index a2c9960e6..1819bd64d 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ const newConversation = await xmtp.conversations.newConversation( #### Sending messages -To be able to send a message, the recipient must have already started their Client at least once and consequently advertised their key bundle on the network. Messages are addressed using wallet addresses. The message payload is a string but neither the SDK nor the network put any constraints on its contents or interpretation. +To be able to send a message, the recipient must have already started their Client at least once and consequently advertised their key bundle on the network. Messages are addressed using wallet addresses. The message payload can be a plain string, but other types of content can be supported through the use of SendOptions (see [Different types of content](#different-types-of-content) for more details) ```ts const conversation = await xmtp.conversations.newConversation( @@ -197,6 +197,35 @@ for await (const message of conversation.streamMessages()) { } ``` +#### Different types of content + +All the send functions support `SendOptions` as an optional parameter. Option `contentType` allows specifying different types of content than the default simple string, which is identified with content type identifier `ContentTypeText`. Support for other types of content can be added by registering additional `ContentCodecs` with the `Client`. Every codec is associated with a content type identifier, `ContentTypeId`, which is used to signal to the Client which codec should be used to process the content that is being sent or received. See XIP-5 for more details on Codecs and content types, new Codecs and content types are defined through XRCs. + +If there is a concern that the recipient may not be able to handle particular content type, the sender can use `contentFallback` option to provide a string that describes the content being sent. If the recipient fails to decode the original content, the fallback will replace it and can be used to inform the recipient what the original content was. + +```ts +// Assuming we've loaded a fictional NumberCodec that can be used to encode numbers, +// and is identified with ContentTypeNumber, we can use it as follows. + +xmtp.registerCodec:(new NumberCodec()) +conversation.send(3.14, { + contentType: ContentTypeNumber, + contentFallback: 'sending you a pie' +}) +``` + +#### Compression + +Message content can be optionally compressed using the `compression` option. The value of the option is the name of the compression algorithm to use. Currently supported are `gzip` and `deflate`. Compression is applied to the bytes produced by the content codec. + +Content will be decompressed transparently on the receiving end. Note that `Client` enforces maximum content size. The default limit can be overridden through the `ClientOptions`. Consequently a message that would expand beyond that limit on the receiving end will fail to decode. + +```ts +conversation.send('#'.repeat(1000), { + compression: 'deflate', +}) +``` + #### Under the hood Using `xmtp.conversations` hides the details of this, but for the curious this is how sending a message on XMTP works. The first message and first response between two parties is sent to three separate [Waku](https://rfc.vac.dev/spec/10/) content topics: diff --git a/package-lock.json b/package-lock.json index f01c7868b..1306cb232 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "@noble/secp256k1": "^1.5.2", + "@stardazed/streams-polyfill": "^2.4.0", "cross-fetch": "^3.1.5", "ethers": "^5.5.3", "js-waku": "^0.18", @@ -2589,6 +2590,11 @@ "@stablelib/wipe": "^1.0.1" } }, + "node_modules/@stardazed/streams-polyfill": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@stardazed/streams-polyfill/-/streams-polyfill-2.4.0.tgz", + "integrity": "sha512-W6Yg9cA8YT1b9qCQsz/2+kmKt7i/Za2Nj4QOLqdiANzpTiGy5mOyCQNyh0CVpbvXkjCBo2QxrwPvbDlP9u9k+Q==" + }, "node_modules/@tootallnate/once": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", @@ -16815,6 +16821,11 @@ "@stablelib/wipe": "^1.0.1" } }, + "@stardazed/streams-polyfill": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@stardazed/streams-polyfill/-/streams-polyfill-2.4.0.tgz", + "integrity": "sha512-W6Yg9cA8YT1b9qCQsz/2+kmKt7i/Za2Nj4QOLqdiANzpTiGy5mOyCQNyh0CVpbvXkjCBo2QxrwPvbDlP9u9k+Q==" + }, "@tootallnate/once": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", diff --git a/package.json b/package.json index 21525f96e..dd7cc437a 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ }, "dependencies": { "@noble/secp256k1": "^1.5.2", + "@stardazed/streams-polyfill": "^2.4.0", "cross-fetch": "^3.1.5", "ethers": "^5.5.3", "js-waku": "^0.18", diff --git a/src/Client.ts b/src/Client.ts index 6572ac515..9f87b421c 100644 --- a/src/Client.ts +++ b/src/Client.ts @@ -14,6 +14,21 @@ import Stream, { messageStream } from './Stream' import { Signer } from 'ethers' import { EncryptedStore, LocalStorageStore, PrivateTopicStore } from './store' import { Conversations } from './conversations' +import { + ContentTypeId, + EncodedContent, + ContentCodec, + ContentTypeText, + TextCodec, + decompress, + compress, + ContentTypeFallback, +} from './MessageContent' +import { Compression } from './proto/messaging' +import * as proto from './proto/messaging' + +/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ +/* eslint-disable @typescript-eslint/no-explicit-any */ const NODES_LIST_URL = 'https://nodes.xmtp.com/' @@ -23,6 +38,9 @@ type NodesList = { testnet: Nodes } +// Default maximum allowed content size +const MaxContentSize = 100 * 1024 * 1024 // 100M + // Parameters for the listMessages functions export type ListMessagesOptions = { pageSize?: number @@ -35,6 +53,14 @@ export enum KeyStoreType { localStorage, } +// Parameters for the send functions +export { Compression } +export type SendOptions = { + contentType: ContentTypeId + contentFallback?: string + compression?: Compression +} + /** * Network startup options */ @@ -50,6 +76,15 @@ type NetworkOptions = { waitForPeersTimeoutMs: number } +type ContentOptions = { + // Allow configuring codecs for additional content types + codecs: ContentCodec[] + + // Set the maximum content size in bytes that is allowed by the Client. + // Currently only checked when decompressing compressed content. + maxContentSize: number +} + type KeyStoreOptions = { /** Specify the keyStore which should be used for loading or saving privateKeyBundles */ keyStoreType: KeyStoreType @@ -59,7 +94,7 @@ type KeyStoreOptions = { * Aggregate type for client options. Optional properties are used when the default value is calculated on invocation, and are computed * as needed by each function. All other defaults are specified in defaultOptions. */ -export type ClientOptions = NetworkOptions & KeyStoreOptions +export type ClientOptions = NetworkOptions & KeyStoreOptions & ContentOptions /** * Provide a default client configuration. These settings can be used on their own, or as a starting point for custom configurations @@ -67,12 +102,16 @@ export type ClientOptions = NetworkOptions & KeyStoreOptions * @param opts additional options to override the default settings */ export function defaultOptions(opts?: Partial): ClientOptions { - const _defaultOptions = { + const _defaultOptions: ClientOptions = { keyStoreType: KeyStoreType.networkTopicStoreV1, env: 'testnet', waitForPeersTimeoutMs: 10000, + codecs: [new TextCodec()], + maxContentSize: MaxContentSize, + } + if (opts?.codecs) { + opts.codecs = _defaultOptions.codecs.concat(opts.codecs) } - return { ..._defaultOptions, ...opts } as ClientOptions } @@ -87,6 +126,8 @@ export default class Client { private contacts: Set // address which we have connected to private knownPublicKeyBundles: Map // addresses and key bundles that we have witnessed private _conversations: Conversations + private _codecs: Map> + private _maxContentSize: number constructor(waku: Waku, keys: PrivateKeyBundle) { this.waku = waku @@ -95,6 +136,8 @@ export default class Client { this.keys = keys this.address = keys.identityKey.publicKey.walletSignatureAddress() this._conversations = new Conversations(this) + this._codecs = new Map() + this._maxContentSize = MaxContentSize } /** @@ -119,6 +162,10 @@ export default class Client { const keyStore = createKeyStoreFromConfig(options, wallet, waku) const keys = await loadOrCreateKeys(wallet, keyStore) const client = new Client(waku, keys) + options.codecs.forEach((codec) => { + client.registerCodec(codec) + }) + client._maxContentSize = options.maxContentSize await client.publishUserContact() return client } @@ -192,7 +239,11 @@ export default class Client { /** * Send a message to the wallet identified by @peerAddress */ - async sendMessage(peerAddress: string, msgString: string): Promise { + async sendMessage( + peerAddress: string, + content: any, + options?: SendOptions + ): Promise { let topics: string[] const recipient = await this.getUserContact(peerAddress) @@ -213,7 +264,7 @@ export default class Client { topics = [buildDirectMessageTopic(this.address, peerAddress)] } const timestamp = new Date() - const msg = await Message.encode(this.keys, recipient, msgString, timestamp) + const msg = await this.encodeMessage(recipient, timestamp, content, options) await Promise.all( topics.map(async (topic) => { const wakuMsg = await WakuMessage.fromBytes(msg.toBytes(), topic, { @@ -231,6 +282,75 @@ export default class Client { } } + registerCodec(codec: ContentCodec): void { + const id = codec.contentType + const key = `${id.authorityId}/${id.typeId}` + this._codecs.set(key, codec) + } + + codecFor(contentType: ContentTypeId): ContentCodec | undefined { + const key = `${contentType.authorityId}/${contentType.typeId}` + const codec = this._codecs.get(key) + if (!codec) { + return undefined + } + if (contentType.versionMajor > codec.contentType.versionMajor) { + return undefined + } + return codec + } + + async encodeMessage( + recipient: PublicKeyBundle, + timestamp: Date, + content: any, + options?: SendOptions + ): Promise { + const contentType = options?.contentType || ContentTypeText + const codec = this.codecFor(contentType) + if (!codec) { + throw new Error('unknown content type ' + contentType) + } + const encoded = codec.encode(content, this) + if (options?.contentFallback) { + encoded.fallback = options.contentFallback + } + if (options?.compression) { + encoded.compression = options.compression + } + await compress(encoded) + const payload = proto.EncodedContent.encode(encoded).finish() + return Message.encode(this.keys, recipient, payload, timestamp) + } + + async decodeMessage(payload: Uint8Array): Promise { + const message = await Message.decode(this.keys, payload) + if (message.error) { + return message + } + if (!message.decrypted) { + throw new Error('decrypted bytes missing') + } + const encoded = proto.EncodedContent.decode(message.decrypted) + await decompress(encoded, this._maxContentSize) + if (!encoded.type) { + throw new Error('missing content type') + } + const contentType = new ContentTypeId(encoded.type) + const codec = this.codecFor(contentType) + if (codec) { + message.content = codec.decode(encoded as EncodedContent, this) + message.contentType = contentType + } else { + message.error = new Error('unknown content type ' + contentType) + if (encoded.fallback) { + message.content = encoded.fallback + message.contentType = ContentTypeFallback + } + } + return message + } + streamIntroductionMessages(): Stream { return this.streamMessages(buildUserIntroTopic(this.address)) } @@ -293,7 +413,7 @@ export default class Client { wakuMsgs .filter((wakuMsg) => wakuMsg?.payload) .map(async (wakuMsg) => - Message.decode(this.keys, wakuMsg.payload as Uint8Array) + this.decodeMessage(wakuMsg.payload as Uint8Array) ) ) } diff --git a/src/Message.ts b/src/Message.ts index c778a02fa..c1961bd0e 100644 --- a/src/Message.ts +++ b/src/Message.ts @@ -10,6 +10,7 @@ import { import { NoMatchingPreKeyError } from './crypto/errors' import { bytesToHex } from './crypto/utils' import { sha256 } from './crypto/encryption' +import { ContentTypeId } from './MessageContent' const extractV1Message = (msg: proto.Message): proto.V1Message => { if (!msg.v1) { @@ -22,10 +23,14 @@ const extractV1Message = (msg: proto.Message): proto.V1Message => { // Message header carries the sender and recipient keys used to protect message. // Message timestamp is set by the sender. export default class Message implements proto.V1Message { - header: proto.MessageHeader | undefined // eslint-disable-line camelcase + header: proto.MessageHeader // eslint-disable-line camelcase headerBytes: Uint8Array // encoded header bytes - ciphertext: Ciphertext | undefined - decrypted: string | undefined + ciphertext: Ciphertext + decrypted?: Uint8Array + // content allows attaching decoded content to the Message + // the message receiving APIs need to return a Message to provide access to the header fields like sender/recipient + contentType?: ContentTypeId + content?: any // eslint-disable-line @typescript-eslint/no-explicit-any error?: Error /** * Identifier that is deterministically derived from the bytes of the message @@ -46,9 +51,10 @@ export default class Message implements proto.V1Message { this.bytes = bytes this.headerBytes = msg.headerBytes this.header = header - if (msg.ciphertext) { - this.ciphertext = new Ciphertext(msg.ciphertext) + if (!msg.ciphertext) { + throw new Error('missing message ciphertext') } + this.ciphertext = new Ciphertext(msg.ciphertext) } toBytes(): Uint8Array { @@ -71,10 +77,6 @@ export default class Message implements proto.V1Message { return Message.create(msg, header, bytes) } - get text(): string | undefined { - return this.decrypted - } - get sent(): Date | undefined { return this.header ? new Date(this.header?.timestamp) : undefined } @@ -103,11 +105,9 @@ export default class Message implements proto.V1Message { static async encode( sender: PrivateKeyBundle, recipient: PublicKeyBundle, - message: string, + message: Uint8Array, timestamp: Date ): Promise { - const msgBytes = new TextEncoder().encode(message) - const secret = await sender.sharedSecret( recipient, sender.getCurrentPreKey().publicKey, @@ -120,7 +120,7 @@ export default class Message implements proto.V1Message { timestamp: timestamp.getTime(), } const headerBytes = proto.MessageHeader.encode(header).finish() - const ciphertext = await encrypt(msgBytes, secret, headerBytes) + const ciphertext = await encrypt(message, secret, headerBytes) const protoMsg = { v1: { headerBytes: headerBytes, ciphertext } } const bytes = proto.Message.encode(protoMsg).finish() const msg = await Message.create(protoMsg, header, bytes) @@ -188,8 +188,7 @@ export default class Message implements proto.V1Message { msg.error = e return msg } - bytes = await decrypt(ciphertext, secret, v1Message.headerBytes) - msg.decrypted = new TextDecoder().decode(bytes) + msg.decrypted = await decrypt(ciphertext, secret, v1Message.headerBytes) return msg } } diff --git a/src/MessageContent.ts b/src/MessageContent.ts new file mode 100644 index 000000000..d4abf2263 --- /dev/null +++ b/src/MessageContent.ts @@ -0,0 +1,204 @@ +import * as proto from './proto/messaging' + +if ( + typeof window !== 'object' || + typeof navigator !== 'object' || + navigator.userAgent.includes('jsdom') +) { + require('@stardazed/streams-polyfill') +} + +// Represents proto.ContentTypeId +export class ContentTypeId { + authorityId: string + typeId: string + versionMajor: number + versionMinor: number + + constructor(obj: proto.ContentTypeId) { + this.authorityId = obj.authorityId + this.typeId = obj.typeId + this.versionMajor = obj.versionMajor + this.versionMinor = obj.versionMinor + } + + toString(): string { + return `${this.authorityId}/${this.typeId}:${this.versionMajor}.${this.versionMinor}` + } + + sameAs(id: ContentTypeId): boolean { + return this.authorityId === id.authorityId && this.typeId === id.typeId + } +} + +// Represents proto.EncodedContent +export interface EncodedContent { + type: ContentTypeId + parameters: Record + fallback?: string + compression?: number + content: Uint8Array +} + +// Define an interface for the encoding machinery for a specific content type +// associated with a given ContentTypeId +// A codec can be registered with a Client to be automatically invoked when +// handling content of the corresponding content type. +export interface CodecRegistry { + // eslint-disable-next-line no-use-before-define, @typescript-eslint/no-explicit-any + codecFor(contentType: ContentTypeId): ContentCodec | undefined +} + +export interface ContentCodec { + contentType: ContentTypeId + encode(content: T, registry: CodecRegistry): EncodedContent + decode(content: EncodedContent, registry: CodecRegistry): T +} + +// xmtp.org/text +// +// This content type is used for a plain text content represented by a simple string +export const ContentTypeText = new ContentTypeId({ + authorityId: 'xmtp.org', + typeId: 'text', + versionMajor: 1, + versionMinor: 0, +}) + +export enum Encoding { + utf8 = 'UTF-8', +} + +export class TextCodec implements ContentCodec { + get contentType(): ContentTypeId { + return ContentTypeText + } + + encode(content: string): EncodedContent { + return { + type: ContentTypeText, + parameters: { encoding: Encoding.utf8 }, + content: new TextEncoder().encode(content), + } + } + + decode(content: EncodedContent): string { + const encoding = content.parameters.encoding + if (encoding && encoding !== Encoding.utf8) { + throw new Error(`unrecognized encoding ${encoding}`) + } + return new TextDecoder().decode(content.content) + } +} + +// xmtp.org/fallback +// +// This content type is used to indicate to the recipient +// that the content in the message is the fallback description (if present) +// in case the original content type is not supported. +// This content type is never used to send a message. +export const ContentTypeFallback = new ContentTypeId({ + authorityId: 'xmtp.org', + typeId: 'fallback', + versionMajor: 1, + versionMinor: 0, +}) + +// +// Compression +// + +export async function decompress( + encoded: proto.EncodedContent, + maxSize: number +): Promise { + if (encoded.compression === undefined) { + return + } + const sink = { bytes: new Uint8Array(encoded.content.length) } + await readStreamFromBytes(encoded.content) + .pipeThrough( + new DecompressionStream(compressionIdFromCode(encoded.compression)) + ) + .pipeTo(writeStreamToBytes(sink, maxSize)) + encoded.content = sink.bytes +} + +export async function compress(encoded: proto.EncodedContent): Promise { + if (encoded.compression === undefined) { + return + } + const sink = { bytes: new Uint8Array(encoded.content.length / 10) } + await readStreamFromBytes(encoded.content) + .pipeThrough( + new CompressionStream(compressionIdFromCode(encoded.compression)) + ) + .pipeTo(writeStreamToBytes(sink, encoded.content.length + 1000)) + encoded.content = sink.bytes +} + +function compressionIdFromCode(code: proto.Compression): string { + if (code === proto.Compression.gzip) { + return 'gzip' + } + if (code === proto.Compression.deflate) { + return 'deflate' + } + throw new Error('unrecognized compression algorithm') +} + +export function readStreamFromBytes( + bytes: Uint8Array, + chunkSize = 1024 +): ReadableStream { + let position = 0 + return new ReadableStream({ + pull(controller) { + if (position >= bytes.length) { + return controller.close() + } + let end = position + chunkSize + end = end <= bytes.length ? end : bytes.length + controller.enqueue(bytes.subarray(position, end)) + position = end + }, + }) +} + +export function writeStreamToBytes( + sink: { + bytes: Uint8Array + }, + maxSize: number +): WritableStream { + let position = 0 + return new WritableStream({ + write(chunk: Uint8Array) { + const end = position + chunk.length + if (end > maxSize) { + throw new Error('maximum output size exceeded') + } + while (sink.bytes.length < end) { + sink.bytes = growBytes(sink.bytes, maxSize) + } + sink.bytes.set(chunk, position) + position = end + }, + + close() { + if (position < sink.bytes.length) { + sink.bytes = sink.bytes.subarray(0, position) + } + }, + }) +} + +function growBytes(bytes: Uint8Array, maxSize: number): Uint8Array { + let newSize = bytes.length * 2 + if (newSize > maxSize) { + newSize = maxSize + } + const bigger = new Uint8Array(newSize) + bigger.set(bytes) + return bigger +} diff --git a/src/Stream.ts b/src/Stream.ts index 3428fad84..51f1aa5b4 100644 --- a/src/Stream.ts +++ b/src/Stream.ts @@ -44,7 +44,7 @@ export default class Stream { if (!wakuMsg.payload) { return } - const msg = await Message.decode(this.client.keys, wakuMsg.payload) + const msg = await this.client.decodeMessage(wakuMsg.payload) // If there is a filter on the stream, and the filter returns false, ignore the message if (filter && !filter(msg)) { return diff --git a/src/conversations/Conversation.ts b/src/conversations/Conversation.ts index d6541c995..156041b03 100644 --- a/src/conversations/Conversation.ts +++ b/src/conversations/Conversation.ts @@ -1,7 +1,9 @@ import Stream from '../Stream' -import Client, { ListMessagesOptions } from '../Client' +import Client, { ListMessagesOptions, SendOptions } from '../Client' import Message from '../Message' +/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ + /** * Conversation class allows you to view, stream, and send messages to/from a peer address */ @@ -31,7 +33,10 @@ export default class Conversation { /** * Send a message into the conversation */ - async send(message: string): Promise { - await this.client.sendMessage(this.peerAddress, message) + async send( + message: any, // eslint-disable-line @typescript-eslint/no-explicit-any + options?: SendOptions + ): Promise { + await this.client.sendMessage(this.peerAddress, message, options) } } diff --git a/src/index.ts b/src/index.ts index 2501c0781..461536757 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,8 +6,21 @@ import { PrivateKeyBundle, } from '../src/crypto' import Stream from './Stream' -import Client, { ClientOptions, ListMessagesOptions } from './Client' +import Client, { + ClientOptions, + ListMessagesOptions, + SendOptions, + Compression, +} from './Client' import { Conversation, Conversations } from './conversations' +import { + ContentTypeId, + ContentCodec, + EncodedContent, + TextCodec, + ContentTypeText, + ContentTypeFallback, +} from './MessageContent' export { Client, @@ -21,4 +34,12 @@ export { PublicKey, PublicKeyBundle, Stream, + ContentTypeId, + ContentCodec, + EncodedContent, + TextCodec, + ContentTypeText, + ContentTypeFallback, + SendOptions, + Compression, } diff --git a/src/proto/messaging.proto b/src/proto/messaging.proto index 69cba5e18..194d210b2 100644 --- a/src/proto/messaging.proto +++ b/src/proto/messaging.proto @@ -1,5 +1,7 @@ syntax = "proto3"; +// Signature represents a generalized public key signature, +// defined as a union to support cryptographic algorithm agility. message Signature { message ECDSACompact { bytes bytes = 1; // compact representation [ R || S ], 64 bytes @@ -10,6 +12,8 @@ message Signature { } } +// PublicKey represents a generalized public key, +// defined as a union to support cryptographic algorithm agility. message PublicKey { message Secp256k1Uncompresed { bytes bytes = 1; // uncompressed point with prefix (0x04) [ P || X || Y ], 65 bytes @@ -21,21 +25,51 @@ message PublicKey { } } -message PrivateKey { - message Secp256k1 { - bytes bytes = 1; // D big-endian, 32 bytes - } - uint64 timestamp = 1; - oneof union { - Secp256k1 secp256k1 = 2; - } - PublicKey publicKey = 3; +// PublicKeyBundle packages the cryptographic keys associated with a wallet, +// both senders and recipients are identified by their key bundles. +message PublicKeyBundle { + PublicKey identityKey = 1; + PublicKey preKey = 2; +} + +// ContentTypeId is used to identify the type of content stored in a Message. +message ContentTypeId { + string authorityId = 1; // authority governing this content type + string typeId = 2; // type identifier + uint32 versionMajor = 3; // major version of the type + uint32 versionMinor = 4; // minor version of the type } +// Recognized compression algorithms +enum Compression { + deflate = 0; + gzip = 1; +} + +// EncodedContent is the type embedded in Ciphertext.payload bytes, +// it bundles the encoded content with metadata identifying the type of content +// and parameters required for correct decoding and presentation of the content. +message EncodedContent { + // content type identifier used to match the payload with the correct decoding machinery + ContentTypeId type = 1; + // optional encoding parameters required to correctly decode the content + map parameters = 2; + // optional fallback description of the content that can be used in case + // the client cannot decode or render the content + optional string fallback = 3; + // optional compression; the value indicates algorithm used to compress the encoded content bytes + optional Compression compression = 5; + // encoded content itself + bytes content = 4; +} + +// Ciphertext represents the payload of the message encoded and encrypted for transport. +// It is definited as a union to support cryptographic algorithm agility. message Ciphertext { message AES256GCM_HKDFSHA256 { bytes hkdfSalt = 1; bytes gcmNonce = 2; + // payload MUST contain encoding of a EncodedContent message bytes payload = 3; } oneof union { @@ -43,17 +77,15 @@ message Ciphertext { } } -message PublicKeyBundle { - PublicKey identityKey = 1; - PublicKey preKey = 2; -} - +// MessageHeader is encoded separately as the bytes are also used +// as associated data for authenticated encryption message MessageHeader { PublicKeyBundle sender = 1; PublicKeyBundle recipient = 2; uint64 timestamp = 3; } +// Message is the top level protocol element message V1Message { bytes headerBytes = 1; // encapsulates the encoded MessageHeader Ciphertext ciphertext = 2; @@ -66,13 +98,27 @@ message Message { } // Private Key Storage +// +// Following definitions are not used in the protocol, instead +// they provide a way for encoding private keys for storage. + +message PrivateKey { + message Secp256k1 { + bytes bytes = 1; // D big-endian, 32 bytes + } + uint64 timestamp = 1; + oneof union { + Secp256k1 secp256k1 = 2; + } + PublicKey publicKey = 3; +} message PrivateKeyBundle { - PrivateKey identityKey = 1; - repeated PrivateKey preKeys = 2; + PrivateKey identityKey = 1; + repeated PrivateKey preKeys = 2; } message EncryptedPrivateKeyBundle { - bytes walletPreKey = 1; - Ciphertext ciphertext = 2; + bytes walletPreKey = 1; + Ciphertext ciphertext = 2; } diff --git a/src/proto/messaging.ts b/src/proto/messaging.ts index 56da05ab7..d85cd2522 100644 --- a/src/proto/messaging.ts +++ b/src/proto/messaging.ts @@ -4,6 +4,43 @@ import _m0 from 'protobufjs/minimal' export const protobufPackage = '' +/** Recognized compression algorithms */ +export enum Compression { + deflate = 0, + gzip = 1, + UNRECOGNIZED = -1, +} + +export function compressionFromJSON(object: any): Compression { + switch (object) { + case 0: + case 'deflate': + return Compression.deflate + case 1: + case 'gzip': + return Compression.gzip + case -1: + case 'UNRECOGNIZED': + default: + return Compression.UNRECOGNIZED + } +} + +export function compressionToJSON(object: Compression): string { + switch (object) { + case Compression.deflate: + return 'deflate' + case Compression.gzip: + return 'gzip' + default: + return 'UNKNOWN' + } +} + +/** + * Signature represents a generalized public key signature, + * defined as a union to support cryptographic algorithm agility. + */ export interface Signature { ecdsaCompact: Signature_ECDSACompact | undefined } @@ -15,6 +52,10 @@ export interface Signature_ECDSACompact { recovery: number } +/** + * PublicKey represents a generalized public key, + * defined as a union to support cryptographic algorithm agility. + */ export interface PublicKey { timestamp: number signature?: Signature | undefined @@ -26,17 +67,57 @@ export interface PublicKey_Secp256k1Uncompresed { bytes: Uint8Array } -export interface PrivateKey { - timestamp: number - secp256k1: PrivateKey_Secp256k1 | undefined - publicKey: PublicKey | undefined +/** + * PublicKeyBundle packages the cryptographic keys associated with a wallet, + * both senders and recipients are identified by their key bundles. + */ +export interface PublicKeyBundle { + identityKey: PublicKey | undefined + preKey: PublicKey | undefined } -export interface PrivateKey_Secp256k1 { - /** D big-endian, 32 bytes */ - bytes: Uint8Array +/** ContentTypeId is used to identify the type of content stored in a Message. */ +export interface ContentTypeId { + /** authority governing this content type */ + authorityId: string + /** type identifier */ + typeId: string + /** major version of the type */ + versionMajor: number + /** minor version of the type */ + versionMinor: number +} + +/** + * EncodedContent is the type embedded in Ciphertext.payload bytes, + * it bundles the encoded content with metadata identifying the type of content + * and parameters required for correct decoding and presentation of the content. + */ +export interface EncodedContent { + /** content type identifier used to match the payload with the correct decoding machinery */ + type: ContentTypeId | undefined + /** optional encoding parameters required to correctly decode the content */ + parameters: { [key: string]: string } + /** + * optional fallback description of the content that can be used in case + * the client cannot decode or render the content + */ + fallback?: string | undefined + /** optional compression; the value indicates algorithm used to compress the encoded content bytes */ + compression?: Compression | undefined + /** encoded content itself */ + content: Uint8Array +} + +export interface EncodedContent_ParametersEntry { + key: string + value: string } +/** + * Ciphertext represents the payload of the message encoded and encrypted for transport. + * It is definited as a union to support cryptographic algorithm agility. + */ export interface Ciphertext { aes256GcmHkdfSha256: Ciphertext_aes256gcmHkdfsha256 | undefined } @@ -44,20 +125,21 @@ export interface Ciphertext { export interface Ciphertext_aes256gcmHkdfsha256 { hkdfSalt: Uint8Array gcmNonce: Uint8Array + /** payload MUST contain encoding of a EncodedContent message */ payload: Uint8Array } -export interface PublicKeyBundle { - identityKey: PublicKey | undefined - preKey: PublicKey | undefined -} - +/** + * MessageHeader is encoded separately as the bytes are also used + * as associated data for authenticated encryption + */ export interface MessageHeader { sender: PublicKeyBundle | undefined recipient: PublicKeyBundle | undefined timestamp: number } +/** Message is the top level protocol element */ export interface V1Message { /** encapsulates the encoded MessageHeader */ headerBytes: Uint8Array @@ -68,6 +150,17 @@ export interface Message { v1: V1Message | undefined } +export interface PrivateKey { + timestamp: number + secp256k1: PrivateKey_Secp256k1 | undefined + publicKey: PublicKey | undefined +} + +export interface PrivateKey_Secp256k1 { + /** D big-endian, 32 bytes */ + bytes: Uint8Array +} + export interface PrivateKeyBundle { identityKey: PrivateKey | undefined preKeys: PrivateKey[] @@ -382,48 +475,235 @@ export const PublicKey_Secp256k1Uncompresed = { }, } -function createBasePrivateKey(): PrivateKey { - return { timestamp: 0, secp256k1: undefined, publicKey: undefined } +function createBasePublicKeyBundle(): PublicKeyBundle { + return { identityKey: undefined, preKey: undefined } } -export const PrivateKey = { +export const PublicKeyBundle = { encode( - message: PrivateKey, + message: PublicKeyBundle, writer: _m0.Writer = _m0.Writer.create() ): _m0.Writer { - if (message.timestamp !== 0) { - writer.uint32(8).uint64(message.timestamp) + if (message.identityKey !== undefined) { + PublicKey.encode(message.identityKey, writer.uint32(10).fork()).ldelim() } - if (message.secp256k1 !== undefined) { - PrivateKey_Secp256k1.encode( - message.secp256k1, + if (message.preKey !== undefined) { + PublicKey.encode(message.preKey, writer.uint32(18).fork()).ldelim() + } + return writer + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): PublicKeyBundle { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input) + let end = length === undefined ? reader.len : reader.pos + length + const message = createBasePublicKeyBundle() + while (reader.pos < end) { + const tag = reader.uint32() + switch (tag >>> 3) { + case 1: + message.identityKey = PublicKey.decode(reader, reader.uint32()) + break + case 2: + message.preKey = PublicKey.decode(reader, reader.uint32()) + break + default: + reader.skipType(tag & 7) + break + } + } + return message + }, + + fromJSON(object: any): PublicKeyBundle { + return { + identityKey: isSet(object.identityKey) + ? PublicKey.fromJSON(object.identityKey) + : undefined, + preKey: isSet(object.preKey) + ? PublicKey.fromJSON(object.preKey) + : undefined, + } + }, + + toJSON(message: PublicKeyBundle): unknown { + const obj: any = {} + message.identityKey !== undefined && + (obj.identityKey = message.identityKey + ? PublicKey.toJSON(message.identityKey) + : undefined) + message.preKey !== undefined && + (obj.preKey = message.preKey + ? PublicKey.toJSON(message.preKey) + : undefined) + return obj + }, + + fromPartial, I>>( + object: I + ): PublicKeyBundle { + const message = createBasePublicKeyBundle() + message.identityKey = + object.identityKey !== undefined && object.identityKey !== null + ? PublicKey.fromPartial(object.identityKey) + : undefined + message.preKey = + object.preKey !== undefined && object.preKey !== null + ? PublicKey.fromPartial(object.preKey) + : undefined + return message + }, +} + +function createBaseContentTypeId(): ContentTypeId { + return { authorityId: '', typeId: '', versionMajor: 0, versionMinor: 0 } +} + +export const ContentTypeId = { + encode( + message: ContentTypeId, + writer: _m0.Writer = _m0.Writer.create() + ): _m0.Writer { + if (message.authorityId !== '') { + writer.uint32(10).string(message.authorityId) + } + if (message.typeId !== '') { + writer.uint32(18).string(message.typeId) + } + if (message.versionMajor !== 0) { + writer.uint32(24).uint32(message.versionMajor) + } + if (message.versionMinor !== 0) { + writer.uint32(32).uint32(message.versionMinor) + } + return writer + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): ContentTypeId { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input) + let end = length === undefined ? reader.len : reader.pos + length + const message = createBaseContentTypeId() + while (reader.pos < end) { + const tag = reader.uint32() + switch (tag >>> 3) { + case 1: + message.authorityId = reader.string() + break + case 2: + message.typeId = reader.string() + break + case 3: + message.versionMajor = reader.uint32() + break + case 4: + message.versionMinor = reader.uint32() + break + default: + reader.skipType(tag & 7) + break + } + } + return message + }, + + fromJSON(object: any): ContentTypeId { + return { + authorityId: isSet(object.authorityId) ? String(object.authorityId) : '', + typeId: isSet(object.typeId) ? String(object.typeId) : '', + versionMajor: isSet(object.versionMajor) + ? Number(object.versionMajor) + : 0, + versionMinor: isSet(object.versionMinor) + ? Number(object.versionMinor) + : 0, + } + }, + + toJSON(message: ContentTypeId): unknown { + const obj: any = {} + message.authorityId !== undefined && (obj.authorityId = message.authorityId) + message.typeId !== undefined && (obj.typeId = message.typeId) + message.versionMajor !== undefined && + (obj.versionMajor = Math.round(message.versionMajor)) + message.versionMinor !== undefined && + (obj.versionMinor = Math.round(message.versionMinor)) + return obj + }, + + fromPartial, I>>( + object: I + ): ContentTypeId { + const message = createBaseContentTypeId() + message.authorityId = object.authorityId ?? '' + message.typeId = object.typeId ?? '' + message.versionMajor = object.versionMajor ?? 0 + message.versionMinor = object.versionMinor ?? 0 + return message + }, +} + +function createBaseEncodedContent(): EncodedContent { + return { + type: undefined, + parameters: {}, + fallback: undefined, + compression: undefined, + content: new Uint8Array(), + } +} + +export const EncodedContent = { + encode( + message: EncodedContent, + writer: _m0.Writer = _m0.Writer.create() + ): _m0.Writer { + if (message.type !== undefined) { + ContentTypeId.encode(message.type, writer.uint32(10).fork()).ldelim() + } + Object.entries(message.parameters).forEach(([key, value]) => { + EncodedContent_ParametersEntry.encode( + { key: key as any, value }, writer.uint32(18).fork() ).ldelim() + }) + if (message.fallback !== undefined) { + writer.uint32(26).string(message.fallback) } - if (message.publicKey !== undefined) { - PublicKey.encode(message.publicKey, writer.uint32(26).fork()).ldelim() + if (message.compression !== undefined) { + writer.uint32(40).int32(message.compression) + } + if (message.content.length !== 0) { + writer.uint32(34).bytes(message.content) } return writer }, - decode(input: _m0.Reader | Uint8Array, length?: number): PrivateKey { + decode(input: _m0.Reader | Uint8Array, length?: number): EncodedContent { const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input) let end = length === undefined ? reader.len : reader.pos + length - const message = createBasePrivateKey() + const message = createBaseEncodedContent() while (reader.pos < end) { const tag = reader.uint32() switch (tag >>> 3) { case 1: - message.timestamp = longToNumber(reader.uint64() as Long) + message.type = ContentTypeId.decode(reader, reader.uint32()) break case 2: - message.secp256k1 = PrivateKey_Secp256k1.decode( + const entry2 = EncodedContent_ParametersEntry.decode( reader, reader.uint32() ) + if (entry2.value !== undefined) { + message.parameters[entry2.key] = entry2.value + } break case 3: - message.publicKey = PublicKey.decode(reader, reader.uint32()) + message.fallback = reader.string() + break + case 5: + message.compression = reader.int32() as any + break + case 4: + message.content = reader.bytes() break default: reader.skipType(tag & 7) @@ -433,61 +713,90 @@ export const PrivateKey = { return message }, - fromJSON(object: any): PrivateKey { + fromJSON(object: any): EncodedContent { return { - timestamp: isSet(object.timestamp) ? Number(object.timestamp) : 0, - secp256k1: isSet(object.secp256k1) - ? PrivateKey_Secp256k1.fromJSON(object.secp256k1) + type: isSet(object.type) + ? ContentTypeId.fromJSON(object.type) : undefined, - publicKey: isSet(object.publicKey) - ? PublicKey.fromJSON(object.publicKey) + parameters: isObject(object.parameters) + ? Object.entries(object.parameters).reduce<{ [key: string]: string }>( + (acc, [key, value]) => { + acc[key] = String(value) + return acc + }, + {} + ) + : {}, + fallback: isSet(object.fallback) ? String(object.fallback) : undefined, + compression: isSet(object.compression) + ? compressionFromJSON(object.compression) : undefined, + content: isSet(object.content) + ? bytesFromBase64(object.content) + : new Uint8Array(), } }, - toJSON(message: PrivateKey): unknown { + toJSON(message: EncodedContent): unknown { const obj: any = {} - message.timestamp !== undefined && - (obj.timestamp = Math.round(message.timestamp)) - message.secp256k1 !== undefined && - (obj.secp256k1 = message.secp256k1 - ? PrivateKey_Secp256k1.toJSON(message.secp256k1) - : undefined) - message.publicKey !== undefined && - (obj.publicKey = message.publicKey - ? PublicKey.toJSON(message.publicKey) - : undefined) + message.type !== undefined && + (obj.type = message.type ? ContentTypeId.toJSON(message.type) : undefined) + obj.parameters = {} + if (message.parameters) { + Object.entries(message.parameters).forEach(([k, v]) => { + obj.parameters[k] = v + }) + } + message.fallback !== undefined && (obj.fallback = message.fallback) + message.compression !== undefined && + (obj.compression = + message.compression !== undefined + ? compressionToJSON(message.compression) + : undefined) + message.content !== undefined && + (obj.content = base64FromBytes( + message.content !== undefined ? message.content : new Uint8Array() + )) return obj }, - fromPartial, I>>( + fromPartial, I>>( object: I - ): PrivateKey { - const message = createBasePrivateKey() - message.timestamp = object.timestamp ?? 0 - message.secp256k1 = - object.secp256k1 !== undefined && object.secp256k1 !== null - ? PrivateKey_Secp256k1.fromPartial(object.secp256k1) - : undefined - message.publicKey = - object.publicKey !== undefined && object.publicKey !== null - ? PublicKey.fromPartial(object.publicKey) + ): EncodedContent { + const message = createBaseEncodedContent() + message.type = + object.type !== undefined && object.type !== null + ? ContentTypeId.fromPartial(object.type) : undefined + message.parameters = Object.entries(object.parameters ?? {}).reduce<{ + [key: string]: string + }>((acc, [key, value]) => { + if (value !== undefined) { + acc[key] = String(value) + } + return acc + }, {}) + message.fallback = object.fallback ?? undefined + message.compression = object.compression ?? undefined + message.content = object.content ?? new Uint8Array() return message }, } -function createBasePrivateKey_Secp256k1(): PrivateKey_Secp256k1 { - return { bytes: new Uint8Array() } +function createBaseEncodedContent_ParametersEntry(): EncodedContent_ParametersEntry { + return { key: '', value: '' } } -export const PrivateKey_Secp256k1 = { +export const EncodedContent_ParametersEntry = { encode( - message: PrivateKey_Secp256k1, + message: EncodedContent_ParametersEntry, writer: _m0.Writer = _m0.Writer.create() ): _m0.Writer { - if (message.bytes.length !== 0) { - writer.uint32(10).bytes(message.bytes) + if (message.key !== '') { + writer.uint32(10).string(message.key) + } + if (message.value !== '') { + writer.uint32(18).string(message.value) } return writer }, @@ -495,15 +804,18 @@ export const PrivateKey_Secp256k1 = { decode( input: _m0.Reader | Uint8Array, length?: number - ): PrivateKey_Secp256k1 { + ): EncodedContent_ParametersEntry { const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input) let end = length === undefined ? reader.len : reader.pos + length - const message = createBasePrivateKey_Secp256k1() + const message = createBaseEncodedContent_ParametersEntry() while (reader.pos < end) { const tag = reader.uint32() switch (tag >>> 3) { case 1: - message.bytes = reader.bytes() + message.key = reader.string() + break + case 2: + message.value = reader.string() break default: reader.skipType(tag & 7) @@ -513,28 +825,26 @@ export const PrivateKey_Secp256k1 = { return message }, - fromJSON(object: any): PrivateKey_Secp256k1 { + fromJSON(object: any): EncodedContent_ParametersEntry { return { - bytes: isSet(object.bytes) - ? bytesFromBase64(object.bytes) - : new Uint8Array(), + key: isSet(object.key) ? String(object.key) : '', + value: isSet(object.value) ? String(object.value) : '', } }, - toJSON(message: PrivateKey_Secp256k1): unknown { + toJSON(message: EncodedContent_ParametersEntry): unknown { const obj: any = {} - message.bytes !== undefined && - (obj.bytes = base64FromBytes( - message.bytes !== undefined ? message.bytes : new Uint8Array() - )) + message.key !== undefined && (obj.key = message.key) + message.value !== undefined && (obj.value = message.value) return obj }, - fromPartial, I>>( + fromPartial, I>>( object: I - ): PrivateKey_Secp256k1 { - const message = createBasePrivateKey_Secp256k1() - message.bytes = object.bytes ?? new Uint8Array() + ): EncodedContent_ParametersEntry { + const message = createBaseEncodedContent_ParametersEntry() + message.key = object.key ?? '' + message.value = object.value ?? '' return message }, } @@ -702,85 +1012,6 @@ export const Ciphertext_aes256gcmHkdfsha256 = { }, } -function createBasePublicKeyBundle(): PublicKeyBundle { - return { identityKey: undefined, preKey: undefined } -} - -export const PublicKeyBundle = { - encode( - message: PublicKeyBundle, - writer: _m0.Writer = _m0.Writer.create() - ): _m0.Writer { - if (message.identityKey !== undefined) { - PublicKey.encode(message.identityKey, writer.uint32(10).fork()).ldelim() - } - if (message.preKey !== undefined) { - PublicKey.encode(message.preKey, writer.uint32(18).fork()).ldelim() - } - return writer - }, - - decode(input: _m0.Reader | Uint8Array, length?: number): PublicKeyBundle { - const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input) - let end = length === undefined ? reader.len : reader.pos + length - const message = createBasePublicKeyBundle() - while (reader.pos < end) { - const tag = reader.uint32() - switch (tag >>> 3) { - case 1: - message.identityKey = PublicKey.decode(reader, reader.uint32()) - break - case 2: - message.preKey = PublicKey.decode(reader, reader.uint32()) - break - default: - reader.skipType(tag & 7) - break - } - } - return message - }, - - fromJSON(object: any): PublicKeyBundle { - return { - identityKey: isSet(object.identityKey) - ? PublicKey.fromJSON(object.identityKey) - : undefined, - preKey: isSet(object.preKey) - ? PublicKey.fromJSON(object.preKey) - : undefined, - } - }, - - toJSON(message: PublicKeyBundle): unknown { - const obj: any = {} - message.identityKey !== undefined && - (obj.identityKey = message.identityKey - ? PublicKey.toJSON(message.identityKey) - : undefined) - message.preKey !== undefined && - (obj.preKey = message.preKey - ? PublicKey.toJSON(message.preKey) - : undefined) - return obj - }, - - fromPartial, I>>( - object: I - ): PublicKeyBundle { - const message = createBasePublicKeyBundle() - message.identityKey = - object.identityKey !== undefined && object.identityKey !== null - ? PublicKey.fromPartial(object.identityKey) - : undefined - message.preKey = - object.preKey !== undefined && object.preKey !== null - ? PublicKey.fromPartial(object.preKey) - : undefined - return message - }, -} - function createBaseMessageHeader(): MessageHeader { return { sender: undefined, recipient: undefined, timestamp: 0 } } @@ -1007,6 +1238,163 @@ export const Message = { }, } +function createBasePrivateKey(): PrivateKey { + return { timestamp: 0, secp256k1: undefined, publicKey: undefined } +} + +export const PrivateKey = { + encode( + message: PrivateKey, + writer: _m0.Writer = _m0.Writer.create() + ): _m0.Writer { + if (message.timestamp !== 0) { + writer.uint32(8).uint64(message.timestamp) + } + if (message.secp256k1 !== undefined) { + PrivateKey_Secp256k1.encode( + message.secp256k1, + writer.uint32(18).fork() + ).ldelim() + } + if (message.publicKey !== undefined) { + PublicKey.encode(message.publicKey, writer.uint32(26).fork()).ldelim() + } + return writer + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): PrivateKey { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input) + let end = length === undefined ? reader.len : reader.pos + length + const message = createBasePrivateKey() + while (reader.pos < end) { + const tag = reader.uint32() + switch (tag >>> 3) { + case 1: + message.timestamp = longToNumber(reader.uint64() as Long) + break + case 2: + message.secp256k1 = PrivateKey_Secp256k1.decode( + reader, + reader.uint32() + ) + break + case 3: + message.publicKey = PublicKey.decode(reader, reader.uint32()) + break + default: + reader.skipType(tag & 7) + break + } + } + return message + }, + + fromJSON(object: any): PrivateKey { + return { + timestamp: isSet(object.timestamp) ? Number(object.timestamp) : 0, + secp256k1: isSet(object.secp256k1) + ? PrivateKey_Secp256k1.fromJSON(object.secp256k1) + : undefined, + publicKey: isSet(object.publicKey) + ? PublicKey.fromJSON(object.publicKey) + : undefined, + } + }, + + toJSON(message: PrivateKey): unknown { + const obj: any = {} + message.timestamp !== undefined && + (obj.timestamp = Math.round(message.timestamp)) + message.secp256k1 !== undefined && + (obj.secp256k1 = message.secp256k1 + ? PrivateKey_Secp256k1.toJSON(message.secp256k1) + : undefined) + message.publicKey !== undefined && + (obj.publicKey = message.publicKey + ? PublicKey.toJSON(message.publicKey) + : undefined) + return obj + }, + + fromPartial, I>>( + object: I + ): PrivateKey { + const message = createBasePrivateKey() + message.timestamp = object.timestamp ?? 0 + message.secp256k1 = + object.secp256k1 !== undefined && object.secp256k1 !== null + ? PrivateKey_Secp256k1.fromPartial(object.secp256k1) + : undefined + message.publicKey = + object.publicKey !== undefined && object.publicKey !== null + ? PublicKey.fromPartial(object.publicKey) + : undefined + return message + }, +} + +function createBasePrivateKey_Secp256k1(): PrivateKey_Secp256k1 { + return { bytes: new Uint8Array() } +} + +export const PrivateKey_Secp256k1 = { + encode( + message: PrivateKey_Secp256k1, + writer: _m0.Writer = _m0.Writer.create() + ): _m0.Writer { + if (message.bytes.length !== 0) { + writer.uint32(10).bytes(message.bytes) + } + return writer + }, + + decode( + input: _m0.Reader | Uint8Array, + length?: number + ): PrivateKey_Secp256k1 { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input) + let end = length === undefined ? reader.len : reader.pos + length + const message = createBasePrivateKey_Secp256k1() + while (reader.pos < end) { + const tag = reader.uint32() + switch (tag >>> 3) { + case 1: + message.bytes = reader.bytes() + break + default: + reader.skipType(tag & 7) + break + } + } + return message + }, + + fromJSON(object: any): PrivateKey_Secp256k1 { + return { + bytes: isSet(object.bytes) + ? bytesFromBase64(object.bytes) + : new Uint8Array(), + } + }, + + toJSON(message: PrivateKey_Secp256k1): unknown { + const obj: any = {} + message.bytes !== undefined && + (obj.bytes = base64FromBytes( + message.bytes !== undefined ? message.bytes : new Uint8Array() + )) + return obj + }, + + fromPartial, I>>( + object: I + ): PrivateKey_Secp256k1 { + const message = createBasePrivateKey_Secp256k1() + message.bytes = object.bytes ?? new Uint8Array() + return message + }, +} + function createBasePrivateKeyBundle(): PrivateKeyBundle { return { identityKey: undefined, preKeys: [] } } @@ -1241,6 +1629,10 @@ if (_m0.util.Long !== Long) { _m0.configure() } +function isObject(value: any): boolean { + return typeof value === 'object' && value !== null +} + function isSet(value: any): boolean { return value !== null && value !== undefined } diff --git a/src/types/streams-polyfill/index.d.ts b/src/types/streams-polyfill/index.d.ts new file mode 100644 index 000000000..35367f242 --- /dev/null +++ b/src/types/streams-polyfill/index.d.ts @@ -0,0 +1,13 @@ +declare class DecompressionStream { + constructor(format: string) + + readonly readable: ReadableStream + readonly writable: WritableStream +} + +declare class CompressionStream { + constructor(format: string) + + readonly readable: ReadableStream + readonly writable: WritableStream +} diff --git a/test/Client.test.ts b/test/Client.test.ts index 9bad2b056..2f2cbe97f 100644 --- a/test/Client.test.ts +++ b/test/Client.test.ts @@ -2,6 +2,15 @@ import assert from 'assert' import { pollFor, newWallet, dumpStream } from './helpers' import { promiseWithTimeout, sleep } from '../src/utils' import Client, { KeyStoreType } from '../src/Client' +import { TestKeyCodec, ContentTypeTestKey } from './ContentTypeTestKey' +import { + ContentTypeFallback, + PrivateKey, + Message, + ContentTypeText, + Compression, + ContentTypeId, +} from '../src' const newLocalDockerClient = (): Promise => Client.create(newWallet(), { @@ -62,40 +71,40 @@ describe('Client', () => { // alice sends intro await alice.sendMessage(bob.address, 'hi bob!') let msg = await aliceIntros.next() - assert.equal(msg.value.decrypted, 'hi bob!') + assert.equal(msg.value.content, 'hi bob!') // bob sends intro in response msg = await bobIntros.next() - assert.equal(msg.value.decrypted, 'hi bob!') + assert.equal(msg.value.content, 'hi bob!') await bob.sendMessage(alice.address, 'hi alice!') msg = await bobIntros.next() - assert.equal(msg.value.decrypted, 'hi alice!') + assert.equal(msg.value.content, 'hi alice!') // alice sends follow up msg = await aliceIntros.next() - assert.equal(msg.value.decrypted, 'hi alice!') + assert.equal(msg.value.content, 'hi alice!') await alice.sendMessage(bob.address, 'how are you?') msg = await aliceBob.next() - assert.equal(msg.value.decrypted, 'hi bob!') + assert.equal(msg.value.content, 'hi bob!') msg = await aliceBob.next() - assert.equal(msg.value.decrypted, 'hi alice!') + assert.equal(msg.value.content, 'hi alice!') msg = await aliceBob.next() - assert.equal(msg.value.decrypted, 'how are you?') + assert.equal(msg.value.content, 'how are you?') // bob responds to follow up msg = await bobAlice.next() - assert.equal(msg.value.decrypted, 'hi bob!') + assert.equal(msg.value.content, 'hi bob!') msg = await bobAlice.next() - assert.equal(msg.value.decrypted, 'hi alice!') + assert.equal(msg.value.content, 'hi alice!') msg = await bobAlice.next() - assert.equal(msg.value.decrypted, 'how are you?') + assert.equal(msg.value.content, 'how are you?') await bob.sendMessage(alice.address, 'fantastic!') msg = await bobAlice.next() - assert.equal(msg.value.decrypted, 'fantastic!') + assert.equal(msg.value.content, 'fantastic!') // alice receives follow up msg = await aliceBob.next() - assert.equal(msg.value.decrypted, 'fantastic!') + assert.equal(msg.value.content, 'fantastic!') // list messages sent previously const fixtures: [string, Client, string | null, string[]][] = [ @@ -129,7 +138,7 @@ describe('Client', () => { ) for (let i = 0; i < expected.length; i++) { assert.equal( - messages[i].decrypted, + messages[i].content, expected[i], `${name} message[${i}]` ) @@ -147,11 +156,11 @@ describe('Client', () => { const intros = await dumpStream(intro) assert.equal(intros.length, 1) - assert.equal(intros[0].decrypted, messages[0]) + assert.equal(intros[0].content, messages[0]) const convos = await dumpStream(convo) assert.equal(convos.length, messages.length) - convos.forEach((m, i) => assert.equal(m.decrypted, messages[i])) + convos.forEach((m, i) => assert.equal(m.content, messages[i])) }) it('for-await-of with stream', async () => { @@ -159,7 +168,7 @@ describe('Client', () => { let count = 5 await alice.sendMessage(bob.address, 'msg ' + count) for await (const msg of convo) { - assert.equal(msg.decrypted, 'msg ' + count) + assert.equal(msg.content, 'msg ' + count) count-- if (!count) { break @@ -184,6 +193,70 @@ describe('Client', () => { const can_mesg_b = await alice.canMessage(bob.address) assert.equal(can_mesg_b, true) }) + + it('can send compressed messages', async () => { + const convo = bob.streamConversationMessages(alice.address) + const content = 'A'.repeat(111) + await alice.sendMessage(bob.address, content, { + contentType: ContentTypeText, + compression: Compression.deflate, + }) + const result = await convo.next() + const msg = result.value as Message + assert.equal(msg.content, content) + await convo.return() + }) + + it('can send custom content type', async () => { + const stream = bob.streamConversationMessages(alice.address) + const key = PrivateKey.generate().publicKey + + // alice doesn't recognize the type + await expect( + alice.sendMessage(bob.address, key, { + contentType: ContentTypeTestKey, + }) + ).rejects.toThrow('unknown content type xmtp.test/public-key:1.0') + + // bob doesn't recognize the type + alice.registerCodec(new TestKeyCodec()) + await alice.sendMessage(bob.address, key, { + contentType: ContentTypeTestKey, + contentFallback: 'this is a public key', + }) + let result = await stream.next() + let msg = result.value as Message + assert.ok(msg.error) + assert.equal( + msg.error.message, + 'unknown content type xmtp.test/public-key:1.0' + ) + assert.ok(msg.contentType) + assert(msg.contentType.sameAs(ContentTypeFallback)) + assert.equal(msg.content, 'this is a public key') + + // both recognize the type + bob.registerCodec(new TestKeyCodec()) + await alice.sendMessage(bob.address, key, { + contentType: ContentTypeTestKey, + }) + result = await stream.next() + msg = result.value as Message + assert(msg.contentType) + assert(msg.contentType.sameAs(ContentTypeTestKey)) + assert(key.equals(msg.content)) + + // alice tries to send version that is not supported + const type2 = new ContentTypeId({ + ...ContentTypeTestKey, + versionMajor: 2, + }) + await expect( + alice.sendMessage(bob.address, key, { contentType: type2 }) + ).rejects.toThrow('unknown content type xmtp.test/public-key:2.0') + + stream.return() + }) }) }) }) diff --git a/test/ContentTypeTestKey.ts b/test/ContentTypeTestKey.ts new file mode 100644 index 000000000..bbda7a379 --- /dev/null +++ b/test/ContentTypeTestKey.ts @@ -0,0 +1,27 @@ +import * as proto from '../src/proto/messaging' +import { ContentTypeId, ContentCodec, PublicKey, EncodedContent } from '../src' + +export const ContentTypeTestKey = new ContentTypeId({ + authorityId: 'xmtp.test', + typeId: 'public-key', + versionMajor: 1, + versionMinor: 0, +}) + +export class TestKeyCodec implements ContentCodec { + get contentType(): ContentTypeId { + return ContentTypeTestKey + } + + encode(key: PublicKey): EncodedContent { + return { + type: ContentTypeTestKey, + parameters: {}, + content: proto.PublicKey.encode(key).finish(), + } + } + + decode(content: EncodedContent): PublicKey { + return new PublicKey(proto.PublicKey.decode(content.content)) + } +} diff --git a/test/Message.test.ts b/test/Message.test.ts index 715521f12..f4ba1a94c 100644 --- a/test/Message.test.ts +++ b/test/Message.test.ts @@ -20,19 +20,20 @@ describe('Message', function () { .identityKey.walletSignatureAddress() // Alice encodes message for Bob + const content = new TextEncoder().encode('Yo!') const msg1 = await Message.encode( alice, bob.getPublicKeyBundle(), - 'Yo!', + content, new Date() ) assert.equal(msg1.senderAddress, aliceWallet.address) assert.equal(msg1.recipientAddress, bobWalletAddress) - assert.equal(msg1.decrypted, 'Yo!') + assert.deepEqual(msg1.decrypted, content) // Bob decodes message from Alice const msg2 = await Message.decode(bob, msg1.toBytes()) - assert.equal(msg1.decrypted, msg2.decrypted) + assert.deepEqual(msg1.decrypted, msg2.decrypted) assert.equal(msg2.senderAddress, aliceWallet.address) assert.equal(msg2.recipientAddress, bobWalletAddress) }) @@ -44,12 +45,12 @@ describe('Message', function () { const msg = await Message.encode( alice, bob.getPublicKeyBundle(), - 'hi', + new TextEncoder().encode('hi'), new Date() ) assert.ok(!msg.error) const eveDecoded = await Message.decode(eve, msg.toBytes()) - assert.equal(eveDecoded.decrypted, undefined) + assert.equal(eveDecoded.contentType, undefined) assert.deepEqual(eveDecoded.error, new NoMatchingPreKeyError()) }) @@ -58,7 +59,7 @@ describe('Message', function () { const msg = await Message.encode( alice, alice.getPublicKeyBundle(), - 'hi', + new TextEncoder().encode('hi'), new Date() ) expect(() => { @@ -74,7 +75,7 @@ describe('Message', function () { const msg = await Message.encode( alice, alice.getPublicKeyBundle(), - 'hi', + new TextEncoder().encode('hi'), new Date() ) assert.equal(msg.id.length, 64) diff --git a/test/MessageContent.test.ts b/test/MessageContent.test.ts new file mode 100644 index 000000000..f5ce13b33 --- /dev/null +++ b/test/MessageContent.test.ts @@ -0,0 +1,95 @@ +import assert from 'assert' +import * as proto from '../src/proto/messaging' +import { + compress, + decompress, + readStreamFromBytes, + writeStreamToBytes, + ContentTypeText, + Encoding, +} from '../src/MessageContent' +import { CodecRegistry } from './helpers' + +describe('MessageContent', function () { + it('can stream bytes from source to sink', async function () { + let from = new Uint8Array(111).fill(42) + // make sink smaller so that it has to grow a lot + let to = { bytes: new Uint8Array(3) } + await readStreamFromBytes(from, 23).pipeTo(writeStreamToBytes(to, 1000)) + assert.deepEqual(from, to.bytes) + }) + + it('will not write beyond limit', async () => { + let from = new Uint8Array(111).fill(42) + let to = { bytes: new Uint8Array(10) } + await expect( + readStreamFromBytes(from, 23).pipeTo(writeStreamToBytes(to, 100)) + ).rejects.toThrow('maximum output size exceeded') + }) + + it('compresses and decompresses', async function () { + const uncompressed = new Uint8Array(55).fill(42) + const compressed = new Uint8Array([ + 120, 1, 211, 210, 34, 11, 0, 0, 252, 223, 9, 7, + ]) + let content = { + type: ContentTypeText, + parameters: {}, + content: uncompressed, + compression: proto.Compression.deflate, + } + await compress(content) + assert.deepEqual(content.content, compressed) + await decompress(content, 1000) + assert.deepEqual(content.content, uncompressed) + }) +}) + +describe('ContentTypeText', () => { + const codecs = new CodecRegistry() + const codec = codecs.codecFor(ContentTypeText) + assert(codec) + + it('can encode/decode text', () => { + const text = 'Hey' + const ec = codec.encode(text, codecs) + assert(ec.type.sameAs(ContentTypeText)) + assert.equal(ec.parameters.encoding, Encoding.utf8) + const text2 = codec.decode(ec, codecs) + assert.equal(text2, text) + }) + + it('defaults to utf-8', () => { + const text = 'Hey' + const ec = codec.encode(text, codecs) + assert(ec.type.sameAs(ContentTypeText)) + assert.equal(ec.parameters.encoding, Encoding.utf8) + delete ec.parameters.encoding + const text2 = codec.decode(ec, codecs) + assert.equal(text2, text) + }) + + it('throws on non-string', () => { + expect(codec.encode(7, codecs)).rejects + }) + + it('throws on invalid utf8', () => { + const ec = { + type: ContentTypeText, + parameters: {}, + content: new Uint8Array([0xe2, 0x28, 0x81]), + } + expect(() => codec.decode(ec, codecs)).rejects + }) + + it('throws on unknown encoding', () => { + const ec = { + type: ContentTypeText, + parameters: { encoding: 'UTF-16' }, + content: new Uint8Array(0), + } + expect(() => codec.decode(ec, codecs)).toThrow( + 'unrecognized encoding UTF-16' + ) + }) +}) diff --git a/test/helpers.ts b/test/helpers.ts index ad476c718..4b079aa40 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -1,5 +1,11 @@ import { Wallet } from 'ethers' -import { PrivateKey, Message } from '../src' +import { + PrivateKey, + Message, + ContentCodec, + ContentTypeId, + TextCodec, +} from '../src' import Stream from '../src/Stream' import { promiseWithTimeout } from '../src/utils' @@ -58,3 +64,28 @@ export function newWallet(): Wallet { } return new Wallet(key.secp256k1.bytes) } + +// A helper to replace a full Client in testing custom content types, +// extracting just the codec registry aspect of the client. +export class CodecRegistry { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private _codecs: Map> + + constructor() { + this._codecs = new Map() + this.registerCodec(new TextCodec()) + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + registerCodec(codec: ContentCodec): void { + const id = codec.contentType + const key = `${id.authorityId}/${id.typeId}` + this._codecs.set(key, codec) + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + codecFor(contentType: ContentTypeId): ContentCodec | undefined { + const key = `${contentType.authorityId}/${contentType.typeId}` + return this._codecs.get(key) + } +}