From 616590980ce604491626ab9fe6c07cd26293717d Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Fri, 12 Jan 2024 12:47:03 -0600 Subject: [PATCH 01/21] feat: add `shouldPush` to ContentCodec, TextCodec --- src/MessageContent.ts | 1 + src/codecs/Text.ts | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/MessageContent.ts b/src/MessageContent.ts index 5ac27029a..8ffb63e44 100644 --- a/src/MessageContent.ts +++ b/src/MessageContent.ts @@ -58,6 +58,7 @@ export interface ContentCodec { encode(content: T, registry: CodecRegistry): EncodedContent decode(content: EncodedContent, registry: CodecRegistry): T fallback(content: T): string | undefined + shouldPush: (content: T) => boolean } // xmtp.org/fallback diff --git a/src/codecs/Text.ts b/src/codecs/Text.ts index 598b5a7b6..126790586 100644 --- a/src/codecs/Text.ts +++ b/src/codecs/Text.ts @@ -40,4 +40,8 @@ export class TextCodec implements ContentCodec { fallback(content: string): string | undefined { return undefined } + + shouldPush() { + return true + } } From fe2e1d1c2790c96ed8882403c7a2d0030973a6ce Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Fri, 12 Jan 2024 13:53:40 -0600 Subject: [PATCH 02/21] fix: add `shouldPush` to `ContentTypeComposite` --- src/codecs/Composite.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/codecs/Composite.ts b/src/codecs/Composite.ts index 582467d61..88976a236 100644 --- a/src/codecs/Composite.ts +++ b/src/codecs/Composite.ts @@ -111,4 +111,8 @@ export class CompositeCodec implements ContentCodec { fallback(content: Composite): string | undefined { return undefined } + + shouldPush() { + return false + } } From d14f677e15aa0486dd9617a3f8938ed0adc9ff09 Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Fri, 12 Jan 2024 14:04:05 -0600 Subject: [PATCH 03/21] feat: add hmac encryption --- src/crypto/encryption.ts | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/crypto/encryption.ts b/src/crypto/encryption.ts index f8fbd6a7f..9d7cba164 100644 --- a/src/crypto/encryption.ts +++ b/src/crypto/encryption.ts @@ -83,3 +83,36 @@ async function hkdf(secret: Uint8Array, salt: Uint8Array): Promise { ['encrypt', 'decrypt'] ) } + +async function hkdfHmacKey( + secret: Uint8Array, + salt: Uint8Array +): Promise { + const key = await crypto.subtle.importKey('raw', secret, 'HKDF', false, [ + 'deriveKey', + ]) + return crypto.subtle.deriveKey( + { name: 'HKDF', hash: 'SHA-256', salt, info: hkdfNoInfo }, + key, + { name: 'HMAC', hash: 'SHA-256', length: 256 }, + false, + ['sign', 'verify'] + ) +} + +async function generateHmac( + key: CryptoKey, + message: Uint8Array +): Promise { + const signed = await crypto.subtle.sign('HMAC', key, message) + return new Uint8Array(signed) +} + +export async function generateHmacSignature( + secret: Uint8Array, + salt: Uint8Array, + message: Uint8Array +): Promise { + const key = await hkdfHmacKey(secret, salt) + return generateHmac(key, message) +} From f99d661b23e0e1c0fb45323a8054448b53e62504 Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Fri, 12 Jan 2024 14:00:44 -0600 Subject: [PATCH 04/21] feat: add `senderHmac` to `MessageV2` --- src/Message.ts | 12 ++++++++---- src/conversations/Conversation.ts | 22 +++++++++++----------- src/keystore/InMemoryKeystore.ts | 11 ++++++++--- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/Message.ts b/src/Message.ts index cedfd21e8..cd5b83e5b 100644 --- a/src/Message.ts +++ b/src/Message.ts @@ -188,26 +188,30 @@ export class MessageV1 extends MessageBase implements proto.MessageV1 { export class MessageV2 extends MessageBase implements proto.MessageV2 { senderAddress: string | undefined - private header: proto.MessageHeaderV2 // eslint-disable-line camelcase + private header: proto.MessageHeaderV2 + senderHmac: Uint8Array constructor( id: string, bytes: Uint8Array, obj: proto.Message, - header: proto.MessageHeaderV2 + header: proto.MessageHeaderV2, + senderHmac: Uint8Array ) { super(id, bytes, obj) this.header = header + this.senderHmac = senderHmac } static async create( obj: proto.Message, header: proto.MessageHeaderV2, - bytes: Uint8Array + bytes: Uint8Array, + senderHmac: Uint8Array ): Promise { const id = bytesToHex(await sha256(bytes)) - return new MessageV2(id, bytes, obj, header) + return new MessageV2(id, bytes, obj, header, senderHmac) } get sent(): Date { diff --git a/src/conversations/Conversation.ts b/src/conversations/Conversation.ts index 13ca2872e..478316db9 100644 --- a/src/conversations/Conversation.ts +++ b/src/conversations/Conversation.ts @@ -15,7 +15,7 @@ import type { import type Client from '../Client' import type { InvitationContext } from '../Invitation' import { DecodedMessage, MessageV1, MessageV2 } from '../Message' -import type { messageApi, keystore, ciphertext } from '@xmtp/proto' +import type { messageApi, keystore } from '@xmtp/proto' import { message, content as proto } from '@xmtp/proto' import { SignedPublicKey, @@ -658,14 +658,17 @@ export class ConversationV2 } const signedBytes = proto.SignedContent.encode(signed).finish() - const ciphertext = await this.encryptMessage(signedBytes, headerBytes) + const { encrypted: ciphertext, senderHmac } = await this.encryptMessage( + signedBytes, + headerBytes + ) const protoMsg = { v1: undefined, - v2: { headerBytes, ciphertext }, + v2: { headerBytes, ciphertext, senderHmac }, } const bytes = message.Message.encode(protoMsg).finish() - return MessageV2.create(protoMsg, header, bytes) + return MessageV2.create(protoMsg, header, bytes, senderHmac) } private async decryptBatch( @@ -709,10 +712,7 @@ export class ConversationV2 } } - private async encryptMessage( - payload: Uint8Array, - headerBytes: Uint8Array - ): Promise { + private async encryptMessage(payload: Uint8Array, headerBytes: Uint8Array) { const { responses } = await this.client.keystore.encryptV2({ requests: [ { @@ -725,8 +725,8 @@ export class ConversationV2 if (responses.length !== 1) { throw new Error('Invalid response length') } - const { encrypted } = getResultOrThrow(responses[0]) - return encrypted + const { encrypted, senderHmac } = getResultOrThrow(responses[0]) + return { encrypted, senderHmac } } private async buildDecodedMessage( @@ -829,7 +829,7 @@ export class ConversationV2 throw new Error('topic mismatch') } - return MessageV2.create(msg, header, env.message) + return MessageV2.create(msg, header, env.message, msg.v2.senderHmac) } async decodeMessage( diff --git a/src/keystore/InMemoryKeystore.ts b/src/keystore/InMemoryKeystore.ts index 4fa1c9c9f..374372ab6 100644 --- a/src/keystore/InMemoryKeystore.ts +++ b/src/keystore/InMemoryKeystore.ts @@ -35,6 +35,7 @@ import { generateUserPreferencesTopic, } from '../crypto/selfEncryption' import type { KeystoreInterface } from '..' +import { generateHmacSignature } from '../crypto/encryption' const { ErrorCode } = keystore @@ -295,10 +296,14 @@ export default class InMemoryKeystore implements KeystoreInterface { ) } + const keyMaterial = getKeyMaterial(topicData.invitation) + const ciphertext = await encryptV2(payload, keyMaterial, headerBytes) + return { - encrypted: await encryptV2( - payload, - getKeyMaterial(topicData.invitation), + encrypted: ciphertext, + senderHmac: await generateHmacSignature( + keyMaterial, + new TextEncoder().encode(contentTopic), headerBytes ), } From acee946b96b5624f575108d32c8c19f25b302558 Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Fri, 12 Jan 2024 14:02:04 -0600 Subject: [PATCH 05/21] fix: update `getResultOrThrow` return type --- src/utils/keystore.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/utils/keystore.ts b/src/utils/keystore.ts index 7a5353093..47994467f 100644 --- a/src/utils/keystore.ts +++ b/src/utils/keystore.ts @@ -4,6 +4,12 @@ import { KeystoreError } from '../keystore/errors' import type { MessageV1 } from '../Message' import type { WithoutUndefined } from './typedefs' +type EncryptionResponseResult< + T extends + | keystore.DecryptResponse_Response + | keystore.EncryptResponse_Response, +> = WithoutUndefined['result'] + // Validates the Keystore response. Throws on errors or missing fields. // Returns a type with all possibly undefined fields required to be defined export const getResultOrThrow = < @@ -12,10 +18,11 @@ export const getResultOrThrow = < | keystore.EncryptResponse_Response, >( response: T -): WithoutUndefined> => { +) => { if (response.error) { throw new KeystoreError(response.error.code, response.error.message) } + if (!response.result) { throw new KeystoreError( keystore.ErrorCode.ERROR_CODE_UNSPECIFIED, @@ -31,9 +38,7 @@ export const getResultOrThrow = < throw new Error('Missing decrypted result') } - return response.result as unknown as WithoutUndefined< - NonNullable - > + return response.result as EncryptionResponseResult } export const buildDecryptV1Request = ( From 4b6289a6eb6f351b4c4a0380ec7b695ff4eefd52 Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Wed, 17 Jan 2024 15:43:35 -0600 Subject: [PATCH 06/21] feat: add method to get conversation HMAC keys --- src/crypto/encryption.ts | 2 +- src/keystore/InMemoryKeystore.ts | 57 ++++++++++++++++++++++++++++---- src/keystore/interfaces.ts | 5 +++ src/keystore/rpcDefinitions.ts | 4 +++ 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/src/crypto/encryption.ts b/src/crypto/encryption.ts index 9d7cba164..d315d51fd 100644 --- a/src/crypto/encryption.ts +++ b/src/crypto/encryption.ts @@ -84,7 +84,7 @@ async function hkdf(secret: Uint8Array, salt: Uint8Array): Promise { ) } -async function hkdfHmacKey( +export async function hkdfHmacKey( secret: Uint8Array, salt: Uint8Array ): Promise { diff --git a/src/keystore/InMemoryKeystore.ts b/src/keystore/InMemoryKeystore.ts index 374372ab6..ff1358856 100644 --- a/src/keystore/InMemoryKeystore.ts +++ b/src/keystore/InMemoryKeystore.ts @@ -35,7 +35,7 @@ import { generateUserPreferencesTopic, } from '../crypto/selfEncryption' import type { KeystoreInterface } from '..' -import { generateHmacSignature } from '../crypto/encryption' +import { generateHmacSignature, hkdfHmacKey } from '../crypto/encryption' const { ErrorCode } = keystore @@ -298,14 +298,19 @@ export default class InMemoryKeystore implements KeystoreInterface { const keyMaterial = getKeyMaterial(topicData.invitation) const ciphertext = await encryptV2(payload, keyMaterial, headerBytes) + const thirtyDayPeriodsSinceEpoch = Math.floor( + Date.now() / 1000 / 60 / 60 / 24 / 30 + ) + const salt = `${thirtyDayPeriodsSinceEpoch}-${this.accountAddress}` + const hmac = await generateHmacSignature( + keyMaterial, + new TextEncoder().encode(salt), + headerBytes + ) return { encrypted: ciphertext, - senderHmac: await generateHmacSignature( - keyMaterial, - new TextEncoder().encode(contentTopic), - headerBytes - ), + senderHmac: hmac, } }, ErrorCode.ERROR_CODE_INVALID_INPUT @@ -585,4 +590,44 @@ export default class InMemoryKeystore implements KeystoreInterface { lookupTopic(topic: string) { return this.v2Store.lookup(topic) } + + async getV2ConversationHmacKeys(): Promise { + const thirtyDayPeriodsSinceEpoch = Math.floor( + Date.now() / 1000 / 60 / 60 / 24 / 30 + ) + + const hmacKeys: keystore.GetConversationHmacKeysResponse['hmacKeys'] = {} + + this.v2Store.topics.forEach(async (topicData) => { + if (topicData.invitation?.topic) { + const keyMaterial = getKeyMaterial(topicData.invitation) + const values = await Promise.all( + [ + thirtyDayPeriodsSinceEpoch - 1, + thirtyDayPeriodsSinceEpoch, + thirtyDayPeriodsSinceEpoch + 1, + ].map(async (value) => { + const salt = `${value}-${this.accountAddress}` + const hmacKey = await hkdfHmacKey( + keyMaterial, + new TextEncoder().encode(salt) + ) + return { + thirtyDayPeriodsSinceEpoch: value, + // convert CryptoKey to Uint8Array to match the proto + hmacKey: new Uint8Array( + await crypto.subtle.exportKey('raw', hmacKey) + ), + } + }) + ) + + hmacKeys[topicData.invitation.topic] = { + values, + } + } + }) + + return { hmacKeys } + } } diff --git a/src/keystore/interfaces.ts b/src/keystore/interfaces.ts index d3052c4c0..1f7cf7837 100644 --- a/src/keystore/interfaces.ts +++ b/src/keystore/interfaces.ts @@ -104,6 +104,11 @@ export interface Keystore { * Get the private preferences topic identifier */ getPrivatePreferencesTopicIdentifier(): Promise + /** + * Returns the conversation HMAC keys for the current, previous, and next + * 30 day periods since the epoch + */ + getV2ConversationHmacKeys(): Promise } export type TopicData = WithoutUndefined diff --git a/src/keystore/rpcDefinitions.ts b/src/keystore/rpcDefinitions.ts index af063558f..798f54368 100644 --- a/src/keystore/rpcDefinitions.ts +++ b/src/keystore/rpcDefinitions.ts @@ -199,6 +199,10 @@ export const apiDefs = { req: null, res: keystore.GetPrivatePreferencesTopicIdentifierResponse, }, + getV2ConversationHmacKeys: { + req: null, + res: keystore.GetConversationHmacKeysResponse, + }, } export type KeystoreApiDefs = typeof apiDefs From b338919dcdd1621c215e01b0b3484a2fc4685b4b Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Wed, 17 Jan 2024 21:51:26 -0600 Subject: [PATCH 07/21] test: update `TestKeyCodec` --- test/ContentTypeTestKey.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/ContentTypeTestKey.ts b/test/ContentTypeTestKey.ts index a36083b55..9c30b2191 100644 --- a/test/ContentTypeTestKey.ts +++ b/test/ContentTypeTestKey.ts @@ -30,4 +30,8 @@ export class TestKeyCodec implements ContentCodec { fallback(content: PublicKey): string | undefined { return 'publickey bundle' } + + shouldPush() { + return false + } } From b73dde4c4c8c5efdf40348ff2e89584c671476ae Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Thu, 18 Jan 2024 10:00:59 -0600 Subject: [PATCH 08/21] refactor: adjust function naming --- src/crypto/encryption.ts | 6 +++--- src/keystore/InMemoryKeystore.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/crypto/encryption.ts b/src/crypto/encryption.ts index d315d51fd..bcd005a27 100644 --- a/src/crypto/encryption.ts +++ b/src/crypto/encryption.ts @@ -100,7 +100,7 @@ export async function hkdfHmacKey( ) } -async function generateHmac( +async function generateHmacWithKey( key: CryptoKey, message: Uint8Array ): Promise { @@ -108,11 +108,11 @@ async function generateHmac( return new Uint8Array(signed) } -export async function generateHmacSignature( +export async function generateHmac( secret: Uint8Array, salt: Uint8Array, message: Uint8Array ): Promise { const key = await hkdfHmacKey(secret, salt) - return generateHmac(key, message) + return generateHmacWithKey(key, message) } diff --git a/src/keystore/InMemoryKeystore.ts b/src/keystore/InMemoryKeystore.ts index ff1358856..121e0c2b7 100644 --- a/src/keystore/InMemoryKeystore.ts +++ b/src/keystore/InMemoryKeystore.ts @@ -35,7 +35,7 @@ import { generateUserPreferencesTopic, } from '../crypto/selfEncryption' import type { KeystoreInterface } from '..' -import { generateHmacSignature, hkdfHmacKey } from '../crypto/encryption' +import { generateHmac, hkdfHmacKey } from '../crypto/encryption' const { ErrorCode } = keystore @@ -302,7 +302,7 @@ export default class InMemoryKeystore implements KeystoreInterface { Date.now() / 1000 / 60 / 60 / 24 / 30 ) const salt = `${thirtyDayPeriodsSinceEpoch}-${this.accountAddress}` - const hmac = await generateHmacSignature( + const hmac = await generateHmac( keyMaterial, new TextEncoder().encode(salt), headerBytes From cd394320a8971b1eae3771df3dd454d7f7d93f73 Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Thu, 18 Jan 2024 17:16:30 -0600 Subject: [PATCH 09/21] fix: allow for HMAC key to be extractable --- src/crypto/encryption.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crypto/encryption.ts b/src/crypto/encryption.ts index bcd005a27..d16a93fe1 100644 --- a/src/crypto/encryption.ts +++ b/src/crypto/encryption.ts @@ -95,7 +95,7 @@ export async function hkdfHmacKey( { name: 'HKDF', hash: 'SHA-256', salt, info: hkdfNoInfo }, key, { name: 'HMAC', hash: 'SHA-256', length: 256 }, - false, + true, ['sign', 'verify'] ) } From 6056942f50b6e61bd4ed05e2352079eaa84edf28 Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Fri, 19 Jan 2024 16:27:34 -0600 Subject: [PATCH 10/21] refactor: add encryption helpers --- src/crypto/encryption.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/crypto/encryption.ts b/src/crypto/encryption.ts index d16a93fe1..79c94d247 100644 --- a/src/crypto/encryption.ts +++ b/src/crypto/encryption.ts @@ -116,3 +116,26 @@ export async function generateHmac( const key = await hkdfHmacKey(secret, salt) return generateHmacWithKey(key, message) } + +export async function validateHmac( + key: CryptoKey, + signature: Uint8Array, + message: Uint8Array +): Promise { + return await crypto.subtle.verify('HMAC', key, signature, message) +} + +export async function exportHmacKey(key: CryptoKey): Promise { + const exported = await crypto.subtle.exportKey('raw', key) + return new Uint8Array(exported) +} + +export async function importHmacKey(key: Uint8Array): Promise { + return crypto.subtle.importKey( + 'raw', + key, + { name: 'HMAC', hash: 'SHA-256', length: 256 }, + true, + ['sign', 'verify'] + ) +} From c261f258573d94dd2521a2a5807d904d31ee7527 Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Fri, 19 Jan 2024 16:28:17 -0600 Subject: [PATCH 11/21] fix: await key gen before returning --- src/keystore/InMemoryKeystore.ts | 56 ++++++++++++++++---------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/keystore/InMemoryKeystore.ts b/src/keystore/InMemoryKeystore.ts index 121e0c2b7..248f5ae81 100644 --- a/src/keystore/InMemoryKeystore.ts +++ b/src/keystore/InMemoryKeystore.ts @@ -35,7 +35,7 @@ import { generateUserPreferencesTopic, } from '../crypto/selfEncryption' import type { KeystoreInterface } from '..' -import { generateHmac, hkdfHmacKey } from '../crypto/encryption' +import { exportHmacKey, generateHmac, hkdfHmacKey } from '../crypto/encryption' const { ErrorCode } = keystore @@ -598,35 +598,35 @@ export default class InMemoryKeystore implements KeystoreInterface { const hmacKeys: keystore.GetConversationHmacKeysResponse['hmacKeys'] = {} - this.v2Store.topics.forEach(async (topicData) => { - if (topicData.invitation?.topic) { - const keyMaterial = getKeyMaterial(topicData.invitation) - const values = await Promise.all( - [ - thirtyDayPeriodsSinceEpoch - 1, - thirtyDayPeriodsSinceEpoch, - thirtyDayPeriodsSinceEpoch + 1, - ].map(async (value) => { - const salt = `${value}-${this.accountAddress}` - const hmacKey = await hkdfHmacKey( - keyMaterial, - new TextEncoder().encode(salt) - ) - return { - thirtyDayPeriodsSinceEpoch: value, - // convert CryptoKey to Uint8Array to match the proto - hmacKey: new Uint8Array( - await crypto.subtle.exportKey('raw', hmacKey) - ), - } - }) - ) + await Promise.all( + this.v2Store.topics.map(async (topicData) => { + if (topicData.invitation?.topic) { + const keyMaterial = getKeyMaterial(topicData.invitation) + const values = await Promise.all( + [ + thirtyDayPeriodsSinceEpoch - 1, + thirtyDayPeriodsSinceEpoch, + thirtyDayPeriodsSinceEpoch + 1, + ].map(async (value) => { + const salt = `${value}-${this.accountAddress}` + const hmacKey = await hkdfHmacKey( + keyMaterial, + new TextEncoder().encode(salt) + ) + return { + thirtyDayPeriodsSinceEpoch: value, + // convert CryptoKey to Uint8Array to match the proto + hmacKey: await exportHmacKey(hmacKey), + } + }) + ) - hmacKeys[topicData.invitation.topic] = { - values, + hmacKeys[topicData.invitation.topic] = { + values, + } } - } - }) + }) + ) return { hmacKeys } } From 04a28bd9d2259a6f2e0951a1478e6d180e9b2869 Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Fri, 19 Jan 2024 16:29:17 -0600 Subject: [PATCH 12/21] test: add HMAC encryption tests --- test/crypto/encryption.test.ts | 62 ++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 test/crypto/encryption.test.ts diff --git a/test/crypto/encryption.test.ts b/test/crypto/encryption.test.ts new file mode 100644 index 000000000..4e95f925f --- /dev/null +++ b/test/crypto/encryption.test.ts @@ -0,0 +1,62 @@ +import { + importHmacKey, + exportHmacKey, + hkdfHmacKey, + validateHmac, + generateHmac, +} from '../../src/crypto/encryption' +import crypto from '../../src/crypto/crypto' + +describe('HMAC encryption', () => { + it('generates and validates HMAC', async () => { + const secret = crypto.getRandomValues(new Uint8Array(32)) + const salt = crypto.getRandomValues(new Uint8Array(32)) + const message = crypto.getRandomValues(new Uint8Array(32)) + const hmac = await generateHmac(secret, salt, message) + const key = await hkdfHmacKey(secret, salt) + const valid = await validateHmac(key, hmac, message) + expect(valid).toBe(true) + }) + + it('generates and validates HMAC with imported key', async () => { + const secret = crypto.getRandomValues(new Uint8Array(32)) + const salt = crypto.getRandomValues(new Uint8Array(32)) + const message = crypto.getRandomValues(new Uint8Array(32)) + const hmac = await generateHmac(secret, salt, message) + const key = await hkdfHmacKey(secret, salt) + const exportedKey = await exportHmacKey(key) + const importedKey = await importHmacKey(exportedKey) + const valid = await validateHmac(importedKey, hmac, message) + expect(valid).toBe(true) + }) + + it('fails to validate HMAC with wrong message', async () => { + const secret = crypto.getRandomValues(new Uint8Array(32)) + const salt = crypto.getRandomValues(new Uint8Array(32)) + const message = crypto.getRandomValues(new Uint8Array(32)) + const hmac = await generateHmac(secret, salt, message) + const key = await hkdfHmacKey(secret, salt) + const valid = await validateHmac( + key, + hmac, + crypto.getRandomValues(new Uint8Array(32)) + ) + expect(valid).toBe(false) + }) + + it('fails to validate HMAC with wrong key', async () => { + const secret = crypto.getRandomValues(new Uint8Array(32)) + const salt = crypto.getRandomValues(new Uint8Array(32)) + const message = crypto.getRandomValues(new Uint8Array(32)) + const hmac = await generateHmac(secret, salt, message) + const valid = await validateHmac( + await hkdfHmacKey( + crypto.getRandomValues(new Uint8Array(32)), + crypto.getRandomValues(new Uint8Array(32)) + ), + hmac, + message + ) + expect(valid).toBe(false) + }) +}) From 57b5a38727a797b338688e65ceef6e4d8c7dc886 Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Fri, 19 Jan 2024 16:29:58 -0600 Subject: [PATCH 13/21] test: add InMemoryKeystore tests --- test/keystore/InMemoryKeystore.test.ts | 157 +++++++++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/test/keystore/InMemoryKeystore.test.ts b/test/keystore/InMemoryKeystore.test.ts index bbee6bddd..952b00d8a 100644 --- a/test/keystore/InMemoryKeystore.test.ts +++ b/test/keystore/InMemoryKeystore.test.ts @@ -20,6 +20,13 @@ import Long from 'long' import { CreateInviteResponse } from '@xmtp/proto/ts/dist/types/keystore_api/v1/keystore.pb' import { assert } from 'vitest' import { toBytes } from 'viem' +import { getKeyMaterial } from '../../src/keystore/utils' +import { + generateHmac, + hkdfHmacKey, + importHmacKey, + validateHmac, +} from '../../src/crypto/encryption' describe('InMemoryKeystore', () => { let aliceKeys: PrivateKeyBundleV1 @@ -396,6 +403,54 @@ describe('InMemoryKeystore', () => { expect(equalBytes(payload, decrypted.result!.decrypted)).toBeTruthy() }) + + it('generates a valid sender HMAC', async () => { + const recipient = SignedPublicKeyBundle.fromLegacyBundle( + bobKeys.getPublicKeyBundle() + ) + const createdNs = dateToNs(new Date()) + const response = await aliceKeystore.createInvite({ + recipient, + createdNs, + context: undefined, + }) + + const payload = new TextEncoder().encode('Hello, world!') + const headerBytes = new Uint8Array(10) + + const { + responses: [encrypted], + } = await aliceKeystore.encryptV2({ + requests: [ + { + contentTopic: response.conversation!.topic, + payload, + headerBytes, + }, + ], + }) + + if (encrypted.error) { + throw encrypted.error + } + + const thirtyDayPeriodsSinceEpoch = Math.floor( + Date.now() / 1000 / 60 / 60 / 24 / 30 + ) + const topicData = aliceKeystore.lookupTopic(response.conversation!.topic) + const keyMaterial = getKeyMaterial(topicData!.invitation) + const hmacKey = await hkdfHmacKey( + keyMaterial, + new TextEncoder().encode( + `${thirtyDayPeriodsSinceEpoch}-${aliceKeystore.walletAddress}` + ) + ) + + expect(encrypted.result?.senderHmac).toBeTruthy() + expect( + await validateHmac(hmacKey, encrypted.result!.senderHmac, headerBytes) + ).toBeTruthy() + }) }) describe('SignDigest', () => { @@ -807,4 +862,106 @@ describe('InMemoryKeystore', () => { ).toBeTruthy() }) }) + + describe('getV2ConversationHmacKeys', () => { + it('returns conversation HMAC keys', async () => { + const baseTime = new Date() + const timestamps = Array.from( + { length: 5 }, + (_, i) => new Date(baseTime.getTime() + i) + ) + + const invites = await Promise.all( + [...timestamps].map(async (createdAt) => { + let keys = await PrivateKeyBundleV1.generate(newWallet()) + + const recipient = SignedPublicKeyBundle.fromLegacyBundle( + keys.getPublicKeyBundle() + ) + + return aliceKeystore.createInvite({ + recipient, + createdNs: dateToNs(createdAt), + context: undefined, + }) + }) + ) + + const thirtyDayPeriodsSinceEpoch = Math.floor( + Date.now() / 1000 / 60 / 60 / 24 / 30 + ) + + const periods = [ + thirtyDayPeriodsSinceEpoch - 1, + thirtyDayPeriodsSinceEpoch, + thirtyDayPeriodsSinceEpoch + 1, + ] + + const { hmacKeys } = await aliceKeystore.getV2ConversationHmacKeys() + + const topics = Object.keys(hmacKeys) + invites.forEach((invite) => { + expect(topics.includes(invite.conversation!.topic)).toBeTruthy() + }) + + const topicHmacs: { + [topic: string]: Uint8Array + } = {} + const headerBytes = new Uint8Array(10) + + await Promise.all( + invites.map(async (invite) => { + const topic = invite.conversation!.topic + const payload = new TextEncoder().encode('Hello, world!') + + const { + responses: [encrypted], + } = await aliceKeystore.encryptV2({ + requests: [ + { + contentTopic: topic, + payload, + headerBytes, + }, + ], + }) + + if (encrypted.error) { + throw encrypted.error + } + + const topicData = aliceKeystore.lookupTopic(topic) + const keyMaterial = getKeyMaterial(topicData!.invitation) + const salt = `${thirtyDayPeriodsSinceEpoch}-${aliceKeystore.walletAddress}` + const hmac = await generateHmac( + keyMaterial, + new TextEncoder().encode(salt), + headerBytes + ) + + topicHmacs[topic] = hmac + }) + ) + + await Promise.all( + Object.keys(hmacKeys).map(async (topic) => { + const hmacData = hmacKeys[topic] + + await Promise.all( + hmacData.values.map( + async ({ hmacKey, thirtyDayPeriodsSinceEpoch }, idx) => { + expect(thirtyDayPeriodsSinceEpoch).toBe(periods[idx]) + const valid = await validateHmac( + await importHmacKey(hmacKey), + topicHmacs[topic], + headerBytes + ) + expect(valid).toBe(idx === 1 ? true : false) + } + ) + ) + }) + ) + }) + }) }) From 49dfca777fc706db59e0fbfae24960369f395710 Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Fri, 19 Jan 2024 17:20:03 -0600 Subject: [PATCH 14/21] refactor: rename HMAC functions --- src/crypto/encryption.ts | 15 ++++----------- src/keystore/InMemoryKeystore.ts | 8 ++++++-- test/crypto/encryption.test.ts | 20 ++++++++++---------- test/keystore/InMemoryKeystore.test.ts | 14 +++++++++----- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/crypto/encryption.ts b/src/crypto/encryption.ts index 79c94d247..1fc2de253 100644 --- a/src/crypto/encryption.ts +++ b/src/crypto/encryption.ts @@ -100,24 +100,17 @@ export async function hkdfHmacKey( ) } -async function generateHmacWithKey( - key: CryptoKey, - message: Uint8Array -): Promise { - const signed = await crypto.subtle.sign('HMAC', key, message) - return new Uint8Array(signed) -} - -export async function generateHmac( +export async function generateHmacSignature( secret: Uint8Array, salt: Uint8Array, message: Uint8Array ): Promise { const key = await hkdfHmacKey(secret, salt) - return generateHmacWithKey(key, message) + const signed = await crypto.subtle.sign('HMAC', key, message) + return new Uint8Array(signed) } -export async function validateHmac( +export async function verifyHmacSignature( key: CryptoKey, signature: Uint8Array, message: Uint8Array diff --git a/src/keystore/InMemoryKeystore.ts b/src/keystore/InMemoryKeystore.ts index 248f5ae81..594062d38 100644 --- a/src/keystore/InMemoryKeystore.ts +++ b/src/keystore/InMemoryKeystore.ts @@ -35,7 +35,11 @@ import { generateUserPreferencesTopic, } from '../crypto/selfEncryption' import type { KeystoreInterface } from '..' -import { exportHmacKey, generateHmac, hkdfHmacKey } from '../crypto/encryption' +import { + exportHmacKey, + generateHmacSignature, + hkdfHmacKey, +} from '../crypto/encryption' const { ErrorCode } = keystore @@ -302,7 +306,7 @@ export default class InMemoryKeystore implements KeystoreInterface { Date.now() / 1000 / 60 / 60 / 24 / 30 ) const salt = `${thirtyDayPeriodsSinceEpoch}-${this.accountAddress}` - const hmac = await generateHmac( + const hmac = await generateHmacSignature( keyMaterial, new TextEncoder().encode(salt), headerBytes diff --git a/test/crypto/encryption.test.ts b/test/crypto/encryption.test.ts index 4e95f925f..556b21ee0 100644 --- a/test/crypto/encryption.test.ts +++ b/test/crypto/encryption.test.ts @@ -2,8 +2,8 @@ import { importHmacKey, exportHmacKey, hkdfHmacKey, - validateHmac, - generateHmac, + verifyHmacSignature, + generateHmacSignature, } from '../../src/crypto/encryption' import crypto from '../../src/crypto/crypto' @@ -12,9 +12,9 @@ describe('HMAC encryption', () => { const secret = crypto.getRandomValues(new Uint8Array(32)) const salt = crypto.getRandomValues(new Uint8Array(32)) const message = crypto.getRandomValues(new Uint8Array(32)) - const hmac = await generateHmac(secret, salt, message) + const hmac = await generateHmacSignature(secret, salt, message) const key = await hkdfHmacKey(secret, salt) - const valid = await validateHmac(key, hmac, message) + const valid = await verifyHmacSignature(key, hmac, message) expect(valid).toBe(true) }) @@ -22,11 +22,11 @@ describe('HMAC encryption', () => { const secret = crypto.getRandomValues(new Uint8Array(32)) const salt = crypto.getRandomValues(new Uint8Array(32)) const message = crypto.getRandomValues(new Uint8Array(32)) - const hmac = await generateHmac(secret, salt, message) + const hmac = await generateHmacSignature(secret, salt, message) const key = await hkdfHmacKey(secret, salt) const exportedKey = await exportHmacKey(key) const importedKey = await importHmacKey(exportedKey) - const valid = await validateHmac(importedKey, hmac, message) + const valid = await verifyHmacSignature(importedKey, hmac, message) expect(valid).toBe(true) }) @@ -34,9 +34,9 @@ describe('HMAC encryption', () => { const secret = crypto.getRandomValues(new Uint8Array(32)) const salt = crypto.getRandomValues(new Uint8Array(32)) const message = crypto.getRandomValues(new Uint8Array(32)) - const hmac = await generateHmac(secret, salt, message) + const hmac = await generateHmacSignature(secret, salt, message) const key = await hkdfHmacKey(secret, salt) - const valid = await validateHmac( + const valid = await verifyHmacSignature( key, hmac, crypto.getRandomValues(new Uint8Array(32)) @@ -48,8 +48,8 @@ describe('HMAC encryption', () => { const secret = crypto.getRandomValues(new Uint8Array(32)) const salt = crypto.getRandomValues(new Uint8Array(32)) const message = crypto.getRandomValues(new Uint8Array(32)) - const hmac = await generateHmac(secret, salt, message) - const valid = await validateHmac( + const hmac = await generateHmacSignature(secret, salt, message) + const valid = await verifyHmacSignature( await hkdfHmacKey( crypto.getRandomValues(new Uint8Array(32)), crypto.getRandomValues(new Uint8Array(32)) diff --git a/test/keystore/InMemoryKeystore.test.ts b/test/keystore/InMemoryKeystore.test.ts index 952b00d8a..314547dd4 100644 --- a/test/keystore/InMemoryKeystore.test.ts +++ b/test/keystore/InMemoryKeystore.test.ts @@ -22,10 +22,10 @@ import { assert } from 'vitest' import { toBytes } from 'viem' import { getKeyMaterial } from '../../src/keystore/utils' import { - generateHmac, + generateHmacSignature, hkdfHmacKey, importHmacKey, - validateHmac, + verifyHmacSignature, } from '../../src/crypto/encryption' describe('InMemoryKeystore', () => { @@ -448,7 +448,11 @@ describe('InMemoryKeystore', () => { expect(encrypted.result?.senderHmac).toBeTruthy() expect( - await validateHmac(hmacKey, encrypted.result!.senderHmac, headerBytes) + await verifyHmacSignature( + hmacKey, + encrypted.result!.senderHmac, + headerBytes + ) ).toBeTruthy() }) }) @@ -933,7 +937,7 @@ describe('InMemoryKeystore', () => { const topicData = aliceKeystore.lookupTopic(topic) const keyMaterial = getKeyMaterial(topicData!.invitation) const salt = `${thirtyDayPeriodsSinceEpoch}-${aliceKeystore.walletAddress}` - const hmac = await generateHmac( + const hmac = await generateHmacSignature( keyMaterial, new TextEncoder().encode(salt), headerBytes @@ -951,7 +955,7 @@ describe('InMemoryKeystore', () => { hmacData.values.map( async ({ hmacKey, thirtyDayPeriodsSinceEpoch }, idx) => { expect(thirtyDayPeriodsSinceEpoch).toBe(periods[idx]) - const valid = await validateHmac( + const valid = await verifyHmacSignature( await importHmacKey(hmacKey), topicHmacs[topic], headerBytes From 47d63da75efef695ca4cdaa09e0e3d1d90e0ca5a Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Fri, 19 Jan 2024 17:22:54 -0600 Subject: [PATCH 15/21] build: export HMAC functions --- src/crypto/index.ts | 15 ++++++++++++++- src/index.ts | 5 +++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/crypto/index.ts b/src/crypto/index.ts index 4a5ce6f2b..ac23b0327 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -9,7 +9,15 @@ import { import { UnsignedPublicKey, SignedPublicKey, PublicKey } from './PublicKey' import Signature, { WalletSigner } from './Signature' import * as utils from './utils' -import { encrypt, decrypt } from './encryption' +import { + decrypt, + encrypt, + exportHmacKey, + generateHmacSignature, + hkdfHmacKey, + importHmacKey, + verifyHmacSignature, +} from './encryption' import Ciphertext from './Ciphertext' import SignedEciesCiphertext from './SignedEciesCiphertext' @@ -18,6 +26,11 @@ export { utils, encrypt, decrypt, + exportHmacKey, + generateHmacSignature, + hkdfHmacKey, + importHmacKey, + verifyHmacSignature, Ciphertext, UnsignedPublicKey, SignedPublicKey, diff --git a/src/index.ts b/src/index.ts index 563c7d7b7..b6d5448f6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,11 @@ export { Signature, encrypt, decrypt, + exportHmacKey, + generateHmacSignature, + hkdfHmacKey, + importHmacKey, + verifyHmacSignature, } from './crypto' export { default as Stream } from './Stream' export type { Signer } from './types/Signer' From a0e3ecd1898c10f42b17eeb63b43063c7cc91c82 Mon Sep 17 00:00:00 2001 From: Nicholas Molnar <65710+neekolas@users.noreply.github.com> Date: Fri, 19 Jan 2024 16:02:28 -0800 Subject: [PATCH 16/21] fix: replace salt with info --- src/crypto/encryption.ts | 9 +++++---- src/keystore/InMemoryKeystore.ts | 4 ++-- test/crypto/encryption.test.ts | 32 +++++++++++++++++++++----------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/crypto/encryption.ts b/src/crypto/encryption.ts index 1fc2de253..a4b1c6d08 100644 --- a/src/crypto/encryption.ts +++ b/src/crypto/encryption.ts @@ -3,6 +3,7 @@ import Ciphertext, { AESGCMNonceSize, KDFSaltSize } from './Ciphertext' import crypto from './crypto' const hkdfNoInfo = new Uint8Array().buffer +const hkdfNoSalt = new Uint8Array().buffer // This is a variation of https://github.com/paulmillr/noble-secp256k1/blob/main/index.ts#L1378-L1388 // that uses `digest('SHA-256', bytes)` instead of `digest('SHA-256', bytes.buffer)` @@ -86,13 +87,13 @@ async function hkdf(secret: Uint8Array, salt: Uint8Array): Promise { export async function hkdfHmacKey( secret: Uint8Array, - salt: Uint8Array + info: Uint8Array ): Promise { const key = await crypto.subtle.importKey('raw', secret, 'HKDF', false, [ 'deriveKey', ]) return crypto.subtle.deriveKey( - { name: 'HKDF', hash: 'SHA-256', salt, info: hkdfNoInfo }, + { name: 'HKDF', hash: 'SHA-256', salt: hkdfNoSalt, info }, key, { name: 'HMAC', hash: 'SHA-256', length: 256 }, true, @@ -102,10 +103,10 @@ export async function hkdfHmacKey( export async function generateHmacSignature( secret: Uint8Array, - salt: Uint8Array, + info: Uint8Array, message: Uint8Array ): Promise { - const key = await hkdfHmacKey(secret, salt) + const key = await hkdfHmacKey(secret, info) const signed = await crypto.subtle.sign('HMAC', key, message) return new Uint8Array(signed) } diff --git a/src/keystore/InMemoryKeystore.ts b/src/keystore/InMemoryKeystore.ts index 594062d38..6f0dc0170 100644 --- a/src/keystore/InMemoryKeystore.ts +++ b/src/keystore/InMemoryKeystore.ts @@ -305,10 +305,10 @@ export default class InMemoryKeystore implements KeystoreInterface { const thirtyDayPeriodsSinceEpoch = Math.floor( Date.now() / 1000 / 60 / 60 / 24 / 30 ) - const salt = `${thirtyDayPeriodsSinceEpoch}-${this.accountAddress}` + const info = `${thirtyDayPeriodsSinceEpoch}-${this.accountAddress}` const hmac = await generateHmacSignature( keyMaterial, - new TextEncoder().encode(salt), + new TextEncoder().encode(info), headerBytes ) diff --git a/test/crypto/encryption.test.ts b/test/crypto/encryption.test.ts index 556b21ee0..8c114deff 100644 --- a/test/crypto/encryption.test.ts +++ b/test/crypto/encryption.test.ts @@ -10,32 +10,42 @@ import crypto from '../../src/crypto/crypto' describe('HMAC encryption', () => { it('generates and validates HMAC', async () => { const secret = crypto.getRandomValues(new Uint8Array(32)) - const salt = crypto.getRandomValues(new Uint8Array(32)) + const info = crypto.getRandomValues(new Uint8Array(32)) const message = crypto.getRandomValues(new Uint8Array(32)) - const hmac = await generateHmacSignature(secret, salt, message) - const key = await hkdfHmacKey(secret, salt) + const hmac = await generateHmacSignature(secret, info, message) + const key = await hkdfHmacKey(secret, info) const valid = await verifyHmacSignature(key, hmac, message) expect(valid).toBe(true) }) it('generates and validates HMAC with imported key', async () => { const secret = crypto.getRandomValues(new Uint8Array(32)) - const salt = crypto.getRandomValues(new Uint8Array(32)) + const info = crypto.getRandomValues(new Uint8Array(32)) const message = crypto.getRandomValues(new Uint8Array(32)) - const hmac = await generateHmacSignature(secret, salt, message) - const key = await hkdfHmacKey(secret, salt) + const hmac = await generateHmacSignature(secret, info, message) + const key = await hkdfHmacKey(secret, info) const exportedKey = await exportHmacKey(key) const importedKey = await importHmacKey(exportedKey) const valid = await verifyHmacSignature(importedKey, hmac, message) expect(valid).toBe(true) }) + it('generates different HMAC keys with different infos', async () => { + const secret = crypto.getRandomValues(new Uint8Array(32)) + const info1 = crypto.getRandomValues(new Uint8Array(32)) + const info2 = crypto.getRandomValues(new Uint8Array(32)) + const key1 = await hkdfHmacKey(secret, info1) + const key2 = await hkdfHmacKey(secret, info2) + + expect(await exportHmacKey(key1)).not.toEqual(await exportHmacKey(key2)) + }) + it('fails to validate HMAC with wrong message', async () => { const secret = crypto.getRandomValues(new Uint8Array(32)) - const salt = crypto.getRandomValues(new Uint8Array(32)) + const info = crypto.getRandomValues(new Uint8Array(32)) const message = crypto.getRandomValues(new Uint8Array(32)) - const hmac = await generateHmacSignature(secret, salt, message) - const key = await hkdfHmacKey(secret, salt) + const hmac = await generateHmacSignature(secret, info, message) + const key = await hkdfHmacKey(secret, info) const valid = await verifyHmacSignature( key, hmac, @@ -46,9 +56,9 @@ describe('HMAC encryption', () => { it('fails to validate HMAC with wrong key', async () => { const secret = crypto.getRandomValues(new Uint8Array(32)) - const salt = crypto.getRandomValues(new Uint8Array(32)) + const info = crypto.getRandomValues(new Uint8Array(32)) const message = crypto.getRandomValues(new Uint8Array(32)) - const hmac = await generateHmacSignature(secret, salt, message) + const hmac = await generateHmacSignature(secret, info, message) const valid = await verifyHmacSignature( await hkdfHmacKey( crypto.getRandomValues(new Uint8Array(32)), From ef2f1671279181b077f69078794848a427609f83 Mon Sep 17 00:00:00 2001 From: Nicholas Molnar <65710+neekolas@users.noreply.github.com> Date: Fri, 19 Jan 2024 16:04:26 -0800 Subject: [PATCH 17/21] fix: replace one more salt with info --- src/keystore/InMemoryKeystore.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/keystore/InMemoryKeystore.ts b/src/keystore/InMemoryKeystore.ts index 6f0dc0170..bcd8c1c43 100644 --- a/src/keystore/InMemoryKeystore.ts +++ b/src/keystore/InMemoryKeystore.ts @@ -612,10 +612,10 @@ export default class InMemoryKeystore implements KeystoreInterface { thirtyDayPeriodsSinceEpoch, thirtyDayPeriodsSinceEpoch + 1, ].map(async (value) => { - const salt = `${value}-${this.accountAddress}` + const info = `${value}-${this.accountAddress}` const hmacKey = await hkdfHmacKey( keyMaterial, - new TextEncoder().encode(salt) + new TextEncoder().encode(info) ) return { thirtyDayPeriodsSinceEpoch: value, From 6c2079b9ef907c5ffaf15424b2668c67339d3bdf Mon Sep 17 00:00:00 2001 From: Nicholas Molnar <65710+neekolas@users.noreply.github.com> Date: Fri, 19 Jan 2024 16:05:36 -0800 Subject: [PATCH 18/21] fix: replace last salt with info --- test/keystore/InMemoryKeystore.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/keystore/InMemoryKeystore.test.ts b/test/keystore/InMemoryKeystore.test.ts index 314547dd4..41f2aacbe 100644 --- a/test/keystore/InMemoryKeystore.test.ts +++ b/test/keystore/InMemoryKeystore.test.ts @@ -936,10 +936,10 @@ describe('InMemoryKeystore', () => { const topicData = aliceKeystore.lookupTopic(topic) const keyMaterial = getKeyMaterial(topicData!.invitation) - const salt = `${thirtyDayPeriodsSinceEpoch}-${aliceKeystore.walletAddress}` + const info = `${thirtyDayPeriodsSinceEpoch}-${aliceKeystore.walletAddress}` const hmac = await generateHmacSignature( keyMaterial, - new TextEncoder().encode(salt), + new TextEncoder().encode(info), headerBytes ) From c50c81b1afcc0421255345098401a7e559ab5adc Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Sat, 20 Jan 2024 00:13:42 -0600 Subject: [PATCH 19/21] feat: add shouldPush to messages --- bench/decode.ts | 7 +-- bench/encode.ts | 6 +-- src/Client.ts | 10 ++++- src/Message.ts | 10 +++-- src/conversations/Conversation.ts | 40 +++++++++++++---- test/Client.test.ts | 59 +++++++++++++++++++++++-- test/Message.test.ts | 2 +- test/conversations/Conversation.test.ts | 31 ++++++++++--- 8 files changed, 136 insertions(+), 29 deletions(-) diff --git a/bench/decode.ts b/bench/decode.ts index e87b131bd..7e99e2769 100644 --- a/bench/decode.ts +++ b/bench/decode.ts @@ -22,9 +22,10 @@ const decodeV1 = () => { const bob = await newPrivateKeyBundle() const message = randomBytes(size) + const { payload } = await alice.encodeContent(message) const encodedMessage = await MessageV1.encode( alice.keystore, - await alice.encodeContent(message), + payload, alice.publicKeyBundle, bob.getPublicKeyBundle(), new Date() @@ -75,8 +76,8 @@ const decodeV2 = () => { new Date(), undefined ) - const payload = await alice.encodeContent(message) - const encodedMessage = await convo.createMessage(payload) + const { payload, shouldPush } = await alice.encodeContent(message) + const encodedMessage = await convo.createMessage(payload, shouldPush) const messageBytes = encodedMessage.toBytes() const envelope = { diff --git a/bench/encode.ts b/bench/encode.ts index 7c52b9959..fa712bcb3 100644 --- a/bench/encode.ts +++ b/bench/encode.ts @@ -22,7 +22,7 @@ const encodeV1 = () => { // The returned function is the actual benchmark. Everything above is setup return async () => { - const encodedMessage = await alice.encodeContent(message) + const { payload: encodedMessage } = await alice.encodeContent(message) await MessageV1.encode( alice.keystore, encodedMessage, @@ -57,11 +57,11 @@ const encodeV2 = () => { undefined ) const message = randomBytes(size) - const payload = await alice.encodeContent(message) + const { payload, shouldPush } = await alice.encodeContent(message) // The returned function is the actual benchmark. Everything above is setup return async () => { - await convo.createMessage(payload) + await convo.createMessage(payload, shouldPush) } }) ) diff --git a/src/Client.ts b/src/Client.ts index e3f787068..46524ee76 100644 --- a/src/Client.ts +++ b/src/Client.ts @@ -634,7 +634,10 @@ export default class Client { async encodeContent( content: ContentTypes, options?: SendOptions - ): Promise { + ): Promise<{ + payload: Uint8Array + shouldPush: boolean + }> { const contentType = options?.contentType || ContentTypeText const codec = this.codecFor(contentType) if (!codec) { @@ -654,7 +657,10 @@ export default class Client { encoded.compression = options.compression } await compress(encoded) - return proto.EncodedContent.encode(encoded).finish() + return { + payload: proto.EncodedContent.encode(encoded).finish(), + shouldPush: codec.shouldPush(content), + } } async decodeContent(contentBytes: Uint8Array): Promise<{ diff --git a/src/Message.ts b/src/Message.ts index cd5b83e5b..b33a08b31 100644 --- a/src/Message.ts +++ b/src/Message.ts @@ -190,28 +190,32 @@ export class MessageV2 extends MessageBase implements proto.MessageV2 { senderAddress: string | undefined private header: proto.MessageHeaderV2 senderHmac: Uint8Array + shouldPush: boolean constructor( id: string, bytes: Uint8Array, obj: proto.Message, header: proto.MessageHeaderV2, - senderHmac: Uint8Array + senderHmac: Uint8Array, + shouldPush: boolean ) { super(id, bytes, obj) this.header = header this.senderHmac = senderHmac + this.shouldPush = shouldPush } static async create( obj: proto.Message, header: proto.MessageHeaderV2, bytes: Uint8Array, - senderHmac: Uint8Array + senderHmac: Uint8Array, + shouldPush: boolean ): Promise { const id = bytesToHex(await sha256(bytes)) - return new MessageV2(id, bytes, obj, header, senderHmac) + return new MessageV2(id, bytes, obj, header, senderHmac, shouldPush) } get sent(): Date { diff --git a/src/conversations/Conversation.ts b/src/conversations/Conversation.ts index 478316db9..7602abd82 100644 --- a/src/conversations/Conversation.ts +++ b/src/conversations/Conversation.ts @@ -286,7 +286,7 @@ export class ConversationV1 } else { topics = [topic] } - const payload = await this.client.encodeContent(content, options) + const { payload } = await this.client.encodeContent(content, options) const msg = await this.createMessage(payload, recipient, options?.timestamp) const msgBytes = msg.toBytes() @@ -395,7 +395,7 @@ export class ConversationV1 topics = [topic] } const contentType = options?.contentType || ContentTypeText - const payload = await this.client.encodeContent(content, options) + const { payload } = await this.client.encodeContent(content, options) const msg = await this.createMessage(payload, recipient, options?.timestamp) await this.client.publishEnvelopes( @@ -604,8 +604,15 @@ export class ConversationV2 content: Exclude, options?: SendOptions ): Promise> { - const payload = await this.client.encodeContent(content, options) - const msg = await this.createMessage(payload, options?.timestamp) + const { payload, shouldPush } = await this.client.encodeContent( + content, + options + ) + const msg = await this.createMessage( + payload, + shouldPush, + options?.timestamp + ) const topic = options?.ephemeral ? this.ephemeralTopic : this.topic @@ -639,6 +646,7 @@ export class ConversationV2 async createMessage( // Payload is expected to have already gone through `client.encodeContent` payload: Uint8Array, + shouldPush: boolean, timestamp?: Date ): Promise { const header: message.MessageHeaderV2 = { @@ -662,13 +670,14 @@ export class ConversationV2 signedBytes, headerBytes ) + const protoMsg = { v1: undefined, - v2: { headerBytes, ciphertext, senderHmac }, + v2: { headerBytes, ciphertext, senderHmac, shouldPush }, } const bytes = message.Message.encode(protoMsg).finish() - return MessageV2.create(protoMsg, header, bytes, senderHmac) + return MessageV2.create(protoMsg, header, bytes, senderHmac, shouldPush) } private async decryptBatch( @@ -781,8 +790,15 @@ export class ConversationV2 content: any, // eslint-disable-line @typescript-eslint/no-explicit-any options?: SendOptions ): Promise { - const payload = await this.client.encodeContent(content, options) - const msg = await this.createMessage(payload, options?.timestamp) + const { payload, shouldPush } = await this.client.encodeContent( + content, + options + ) + const msg = await this.createMessage( + payload, + shouldPush, + options?.timestamp + ) const msgBytes = msg.toBytes() const topic = options?.ephemeral ? this.ephemeralTopic : this.topic @@ -829,7 +845,13 @@ export class ConversationV2 throw new Error('topic mismatch') } - return MessageV2.create(msg, header, env.message, msg.v2.senderHmac) + return MessageV2.create( + msg, + header, + env.message, + msg.v2.senderHmac, + msg.v2.shouldPush + ) } async decodeMessage( diff --git a/test/Client.test.ts b/test/Client.test.ts index 085fca8e0..0289593ee 100644 --- a/test/Client.test.ts +++ b/test/Client.test.ts @@ -5,7 +5,7 @@ import { waitForUserContact, newLocalHostClientWithCustomWallet, } from './helpers' -import { buildUserContactTopic } from '../src/utils' +import { EnvelopeWithMessage, buildUserContactTopic } from '../src/utils' import Client, { ClientOptions } from '../src/Client' import { ApiUrls, @@ -19,16 +19,18 @@ import { } from '../src' import NetworkKeyManager from '../src/keystore/providers/NetworkKeyManager' import TopicPersistence from '../src/keystore/persistence/TopicPersistence' -import { PrivateKeyBundleV1 } from '../src/crypto' +import { PrivateKey, PrivateKeyBundleV1 } from '../src/crypto' import { Wallet } from 'ethers' import { NetworkKeystoreProvider } from '../src/keystore/providers' import { PublishResponse } from '@xmtp/proto/ts/dist/types/message_api/v1/message_api.pb' import LocalStoragePonyfill from '../src/keystore/persistence/LocalStoragePonyfill' +import { message } from '@xmtp/proto' import { createWalletClient, http } from 'viem' import { privateKeyToAccount } from 'viem/accounts' import { mainnet } from 'viem/chains' import { generatePrivateKey } from 'viem/accounts' import { vi, assert } from 'vitest' +import { ContentTypeTestKey, TestKeyCodec } from './ContentTypeTestKey' type TestCase = { name: string @@ -196,11 +198,25 @@ describe('encodeContent', () => { 3, 0, 139, 43, 173, 229, ]) - const payload = await c.encodeContent(uncompressed, { + const { payload } = await c.encodeContent(uncompressed, { compression: Compression.COMPRESSION_DEFLATE, }) expect(Uint8Array.from(payload)).toEqual(compressed) }) + + it('returns shouldPush based on content codec', async () => { + const alice = await newLocalHostClient() + alice.registerCodec(new TestKeyCodec()) + + const { shouldPush: result1 } = await alice.encodeContent('gm') + expect(result1).toBe(true) + + const key = PrivateKey.generate().publicKey + const { shouldPush: result2 } = await alice.encodeContent(key, { + contentType: ContentTypeTestKey, + }) + expect(result2).toBe(false) + }) }) describe('canMessage', () => { @@ -296,6 +312,43 @@ describe('canMessageMultipleBatches', () => { }) }) +describe('listEnvelopes', () => { + it('has envelopes with senderHmac and shouldPush', async () => { + const alice = await newLocalHostClient() + const bob = await newLocalHostClient() + alice.registerCodec(new TestKeyCodec()) + const convo = await alice.conversations.newConversation(bob.address) + await convo.send('hi') + const key = PrivateKey.generate().publicKey + await convo.send(key, { + contentType: ContentTypeTestKey, + }) + + const envelopes = await alice.listEnvelopes( + convo.topic, + (env: EnvelopeWithMessage) => Promise.resolve(env) + ) + + const msg1 = message.Message.decode(envelopes[0].message) + if (!msg1.v2) { + throw new Error('unknown message version') + } + const header1 = message.MessageHeaderV2.decode(msg1.v2.headerBytes) + expect(header1.topic).toEqual(convo.topic) + expect(msg1.v2.senderHmac).toBeDefined() + expect(msg1.v2.shouldPush).toBe(true) + + const msg2 = message.Message.decode(envelopes[1].message) + if (!msg2.v2) { + throw new Error('unknown message version') + } + const header2 = message.MessageHeaderV2.decode(msg2.v2.headerBytes) + expect(header2.topic).toEqual(convo.topic) + expect(msg2.v2.senderHmac).toBeDefined() + expect(msg2.v2.shouldPush).toBe(false) + }) +}) + describe('publishEnvelopes', () => { it('can send a valid envelope', async () => { const c = await newLocalHostClient() diff --git a/test/Message.test.ts b/test/Message.test.ts index b8158eac3..1f9617624 100644 --- a/test/Message.test.ts +++ b/test/Message.test.ts @@ -153,7 +153,7 @@ describe('Message', function () { env: 'local', privateKeyOverride: alice.encode(), }) - const payload = await aliceClient.encodeContent(text) + const { payload } = await aliceClient.encodeContent(text) const timestamp = new Date() const sender = alice.getPublicKeyBundle() const recipient = bob.getPublicKeyBundle() diff --git a/test/conversations/Conversation.test.ts b/test/conversations/Conversation.test.ts index 5407df302..1b70d5dcc 100644 --- a/test/conversations/Conversation.test.ts +++ b/test/conversations/Conversation.test.ts @@ -1,4 +1,4 @@ -import { DecodedMessage, MessageV1 } from './../../src/Message' +import { DecodedMessage, MessageV1, MessageV2 } from './../../src/Message' import { buildDirectMessageTopic } from './../../src/utils' import { Client, Compression, ContentTypeId, ContentTypeText } from '../../src' import { SortDirection } from '../../src/ApiClient' @@ -543,6 +543,17 @@ describe('conversation', () => { await bs.return() await as.return() + + const messages = await alice.listEnvelopes( + ac.topic, + ac.processEnvelope.bind(ac) + ) + + expect(messages).toHaveLength(2) + expect(messages[0].shouldPush).toBe(true) + expect(messages[0].senderHmac).toBeDefined() + expect(messages[1].shouldPush).toBe(true) + expect(messages[1].senderHmac).toBeDefined() }) // it('rejects spoofed contact bundles', async () => { @@ -733,6 +744,9 @@ describe('conversation', () => { metadata: {}, } ) + if (!(aliceConvo instanceof ConversationV2)) { + assert.fail() + } await sleep(100) const bobConvo = await bob.conversations.newConversation(alice.address, { conversationId: 'xmtp.org/key', @@ -744,7 +758,6 @@ describe('conversation', () => { // alice doesn't recognize the type expect( - // @ts-expect-error aliceConvo.send(key, { contentType: ContentTypeTestKey, }) @@ -752,7 +765,6 @@ describe('conversation', () => { // bob doesn't recognize the type alice.registerCodec(new TestKeyCodec()) - // @ts-expect-error await aliceConvo.send(key, { contentType: ContentTypeTestKey, }) @@ -774,7 +786,6 @@ describe('conversation', () => { // both recognize the type bob.registerCodec(new TestKeyCodec()) - // @ts-expect-error await aliceConvo.send(key, { contentType: ContentTypeTestKey, }) @@ -789,13 +800,23 @@ describe('conversation', () => { ...ContentTypeTestKey, versionMajor: 2, }) - // @ts-expect-error expect(aliceConvo.send(key, { contentType: type2 })).rejects.toThrow( 'unknown content type xmtp.test/public-key:2.0' ) await bobStream.return() await aliceStream.return() + + const messages = await alice.listEnvelopes( + aliceConvo.topic, + aliceConvo.processEnvelope.bind(aliceConvo) + ) + + expect(messages).toHaveLength(2) + expect(messages[0].shouldPush).toBe(false) + expect(messages[0].senderHmac).toBeDefined() + expect(messages[1].shouldPush).toBe(false) + expect(messages[1].senderHmac).toBeDefined() }) }) }) From bbd7010bed9a5cadaac981c999cf139f0f8b3a12 Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Wed, 24 Jan 2024 15:25:20 -0600 Subject: [PATCH 20/21] fix: allow specific topics when getting HMAC keys --- src/keystore/InMemoryKeystore.ts | 19 ++++- src/keystore/rpcDefinitions.ts | 6 +- test/keystore/InMemoryKeystore.test.ts | 107 ++++++++++++++++++++++++- 3 files changed, 127 insertions(+), 5 deletions(-) diff --git a/src/keystore/InMemoryKeystore.ts b/src/keystore/InMemoryKeystore.ts index bcd8c1c43..a2ee30776 100644 --- a/src/keystore/InMemoryKeystore.ts +++ b/src/keystore/InMemoryKeystore.ts @@ -34,12 +34,12 @@ import { userPreferencesEncrypt, generateUserPreferencesTopic, } from '../crypto/selfEncryption' -import type { KeystoreInterface } from '..' import { exportHmacKey, generateHmacSignature, hkdfHmacKey, } from '../crypto/encryption' +import type { KeystoreInterface } from './rpcDefinitions' const { ErrorCode } = keystore @@ -595,15 +595,28 @@ export default class InMemoryKeystore implements KeystoreInterface { return this.v2Store.lookup(topic) } - async getV2ConversationHmacKeys(): Promise { + async getV2ConversationHmacKeys( + req?: keystore.GetConversationHmacKeysRequest + ): Promise { const thirtyDayPeriodsSinceEpoch = Math.floor( Date.now() / 1000 / 60 / 60 / 24 / 30 ) const hmacKeys: keystore.GetConversationHmacKeysResponse['hmacKeys'] = {} + let topics = this.v2Store.topics + + // if specific topics are requested, only include those topics + if (req?.topics) { + topics = topics.filter( + (topicData) => + topicData.invitation !== undefined && + req.topics.includes(topicData.invitation.topic) + ) + } + await Promise.all( - this.v2Store.topics.map(async (topicData) => { + topics.map(async (topicData) => { if (topicData.invitation?.topic) { const keyMaterial = getKeyMaterial(topicData.invitation) const values = await Promise.all( diff --git a/src/keystore/rpcDefinitions.ts b/src/keystore/rpcDefinitions.ts index 798f54368..db73e08cf 100644 --- a/src/keystore/rpcDefinitions.ts +++ b/src/keystore/rpcDefinitions.ts @@ -199,8 +199,12 @@ export const apiDefs = { req: null, res: keystore.GetPrivatePreferencesTopicIdentifierResponse, }, + /** + * Returns the conversation HMAC keys for the current, previous, and next + * 30 day periods since the epoch + */ getV2ConversationHmacKeys: { - req: null, + req: keystore.GetConversationHmacKeysRequest, res: keystore.GetConversationHmacKeysResponse, }, } diff --git a/test/keystore/InMemoryKeystore.test.ts b/test/keystore/InMemoryKeystore.test.ts index 41f2aacbe..c1a03736a 100644 --- a/test/keystore/InMemoryKeystore.test.ts +++ b/test/keystore/InMemoryKeystore.test.ts @@ -868,7 +868,7 @@ describe('InMemoryKeystore', () => { }) describe('getV2ConversationHmacKeys', () => { - it('returns conversation HMAC keys', async () => { + it('returns all conversation HMAC keys', async () => { const baseTime = new Date() const timestamps = Array.from( { length: 5 }, @@ -967,5 +967,110 @@ describe('InMemoryKeystore', () => { }) ) }) + + it('returns specific conversation HMAC keys', async () => { + const baseTime = new Date() + const timestamps = Array.from( + { length: 10 }, + (_, i) => new Date(baseTime.getTime() + i) + ) + + const invites = await Promise.all( + [...timestamps].map(async (createdAt) => { + let keys = await PrivateKeyBundleV1.generate(newWallet()) + + const recipient = SignedPublicKeyBundle.fromLegacyBundle( + keys.getPublicKeyBundle() + ) + + return aliceKeystore.createInvite({ + recipient, + createdNs: dateToNs(createdAt), + context: undefined, + }) + }) + ) + + const thirtyDayPeriodsSinceEpoch = Math.floor( + Date.now() / 1000 / 60 / 60 / 24 / 30 + ) + + const periods = [ + thirtyDayPeriodsSinceEpoch - 1, + thirtyDayPeriodsSinceEpoch, + thirtyDayPeriodsSinceEpoch + 1, + ] + + const randomInvites = invites.slice(3, 8) + + const { hmacKeys } = await aliceKeystore.getV2ConversationHmacKeys({ + topics: randomInvites.map((invite) => invite.conversation!.topic), + }) + + const topics = Object.keys(hmacKeys) + expect(topics.length).toBe(randomInvites.length) + randomInvites.forEach((invite) => { + expect(topics.includes(invite.conversation!.topic)).toBeTruthy() + }) + + const topicHmacs: { + [topic: string]: Uint8Array + } = {} + const headerBytes = new Uint8Array(10) + + await Promise.all( + randomInvites.map(async (invite) => { + const topic = invite.conversation!.topic + const payload = new TextEncoder().encode('Hello, world!') + + const { + responses: [encrypted], + } = await aliceKeystore.encryptV2({ + requests: [ + { + contentTopic: topic, + payload, + headerBytes, + }, + ], + }) + + if (encrypted.error) { + throw encrypted.error + } + + const topicData = aliceKeystore.lookupTopic(topic) + const keyMaterial = getKeyMaterial(topicData!.invitation) + const info = `${thirtyDayPeriodsSinceEpoch}-${aliceKeystore.walletAddress}` + const hmac = await generateHmacSignature( + keyMaterial, + new TextEncoder().encode(info), + headerBytes + ) + + topicHmacs[topic] = hmac + }) + ) + + await Promise.all( + Object.keys(hmacKeys).map(async (topic) => { + const hmacData = hmacKeys[topic] + + await Promise.all( + hmacData.values.map( + async ({ hmacKey, thirtyDayPeriodsSinceEpoch }, idx) => { + expect(thirtyDayPeriodsSinceEpoch).toBe(periods[idx]) + const valid = await verifyHmacSignature( + await importHmacKey(hmacKey), + topicHmacs[topic], + headerBytes + ) + expect(valid).toBe(idx === 1 ? true : false) + } + ) + ) + }) + ) + }) }) }) From ca498c0e70a1dbd7794976d1715e7bb6ef7c236c Mon Sep 17 00:00:00 2001 From: Ry Racherbaumer Date: Tue, 12 Mar 2024 13:21:27 -0500 Subject: [PATCH 21/21] fix: adjust types --- src/Message.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Message.ts b/src/Message.ts index b33a08b31..9edf0d677 100644 --- a/src/Message.ts +++ b/src/Message.ts @@ -189,16 +189,16 @@ export class MessageV1 extends MessageBase implements proto.MessageV1 { export class MessageV2 extends MessageBase implements proto.MessageV2 { senderAddress: string | undefined private header: proto.MessageHeaderV2 - senderHmac: Uint8Array - shouldPush: boolean + senderHmac?: Uint8Array + shouldPush?: boolean constructor( id: string, bytes: Uint8Array, obj: proto.Message, header: proto.MessageHeaderV2, - senderHmac: Uint8Array, - shouldPush: boolean + senderHmac?: Uint8Array, + shouldPush?: boolean ) { super(id, bytes, obj) this.header = header @@ -210,8 +210,8 @@ export class MessageV2 extends MessageBase implements proto.MessageV2 { obj: proto.Message, header: proto.MessageHeaderV2, bytes: Uint8Array, - senderHmac: Uint8Array, - shouldPush: boolean + senderHmac?: Uint8Array, + shouldPush?: boolean ): Promise { const id = bytesToHex(await sha256(bytes))