From 0dfbe3ac5688e0e56858843e4e08f4a3dceaa4d5 Mon Sep 17 00:00:00 2001 From: Jazz Turner-Baggs Date: Wed, 9 Mar 2022 15:11:00 -0800 Subject: [PATCH 1/7] test: added tests for networkStore --- src/store/NetworkStore.ts | 23 +++++++++++ test/store/NetworkStore.test.ts | 69 +++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 src/store/NetworkStore.ts create mode 100644 test/store/NetworkStore.test.ts diff --git a/src/store/NetworkStore.ts b/src/store/NetworkStore.ts new file mode 100644 index 000000000..6aad63388 --- /dev/null +++ b/src/store/NetworkStore.ts @@ -0,0 +1,23 @@ +// This creates an interface for storing data to the storage network. +import { Store } from './Store' + +const KEY_PREFIX = '/xmtp/' +const ENCODING = 'binary' + +export default class NetworkStore implements Store { + // Include a key prefix to namespace items in LocalStorage + // This will prevent us from squashing any values set by other libraries on the site + keyPrefix: string + + constructor(keyPrefix: string = KEY_PREFIX) { + this.keyPrefix = keyPrefix + } + + async get(key: string): Promise { + return Promise.resolve(Buffer.from(key)) + } + + async set(key: string, value: Buffer): Promise { + return Promise.resolve() + } +} diff --git a/test/store/NetworkStore.test.ts b/test/store/NetworkStore.test.ts new file mode 100644 index 000000000..f1c524c3a --- /dev/null +++ b/test/store/NetworkStore.test.ts @@ -0,0 +1,69 @@ +import { NetworkStore } from '../../src/store' + +describe('NetworkStore', () => { + jest.setTimeout(10000) + const tests = [ + { + name: 'local docker node', + newWaku: undefined, + }, + ] + if (process.env.CI || process.env.TESTNET) { + tests.push({ + name: 'testnet', + newWaku: undefined, + }) + } + tests.forEach((testCase) => { + describe(testCase.name, () => { + let store: NetworkStore + + beforeEach(async () => { + store = new NetworkStore() + }) + + it('roundtrip', async () => { + const key = 'key' + + const value = new TextEncoder().encode('hello') + const empty = await store.get(key) + expect(empty).toBeNull() + + await store.set(key, Buffer.from(value)) + const full = await store.get(key) + + expect(full).toBeDefined() + expect(full).toEqual(Buffer.from(value)) + }) + + it('distinct topics', async () => { + const valueA = Buffer.from(new TextEncoder().encode('helloA')) + const valueB = Buffer.from(new TextEncoder().encode('helloB')) + const keyA = 'keyA' + const keyB = 'keyB' + + store.set(keyA, valueA) + store.set(keyB, valueB) + const responseA = await store.get(keyA) + const responseB = await store.get(keyB) + + expect(responseA).toEqual(valueA) + expect(responseB).toEqual(valueB) + expect(responseA).not.toEqual(responseB) + }) + + it('over write safety', async () => { + const key = 'key' + + const first_value = new TextEncoder().encode('a') + const second_value = new TextEncoder().encode('bb') + + await store.set(key, Buffer.from(first_value)) + await store.set(key, Buffer.from(second_value)) + const returned_value = await store.get(key) + + expect(returned_value).toEqual(Buffer.from(first_value)) + }) + }) + }) +}) From 957f344d2fcdddb3bd65c0b8a0b39607a285689b Mon Sep 17 00:00:00 2001 From: Jazz Turner-Baggs Date: Fri, 11 Mar 2022 14:30:41 -0800 Subject: [PATCH 2/7] feat: networkStore impl --- src/Client.ts | 27 +++++++++++++++++----- src/store/NetworkStore.ts | 30 ++++++++++++++++--------- src/store/index.ts | 1 + src/utils.ts | 4 ++++ test/store/NetworkStore.test.ts | 40 +++++++++++++++++++++++++++------ 5 files changed, 80 insertions(+), 22 deletions(-) diff --git a/src/Client.ts b/src/Client.ts index 320ea9949..ad534fdc1 100644 --- a/src/Client.ts +++ b/src/Client.ts @@ -7,12 +7,13 @@ import { buildDirectMessageTopic, buildUserContactTopic, buildUserIntroTopic, + buildUserPrivateStoreTopic, promiseWithTimeout, } from './utils' import { sleep } from '../test/helpers' import Stream, { messageStream } from './Stream' import { Signer } from 'ethers' -import { EncryptedStore, LocalStorageStore } from './store' +import { EncryptedStore, LocalStorageStore, NetworkStore } from './store' import { Conversations } from './conversations' const NODES_LIST_URL = 'https://nodes.xmtp.com/' @@ -81,7 +82,10 @@ export default class Client { */ static async create(wallet: Signer, opts?: CreateOptions): Promise { const waku = await createWaku(opts || {}) - const keys = await loadOrCreateKeys(wallet) + const keys = await loadOrCreateKeys( + wallet, + createPrivateKeyStore(wallet, waku) + ) const client = new Client(waku, keys) await client.publishUserContact() return client @@ -262,10 +266,23 @@ export default class Client { } } +// select appropriate store for privateKeyBundles +export function createPrivateKeyStore( + wallet: Signer, + waku: Waku | undefined = undefined +): EncryptedStore { + const backingStore = waku + ? new NetworkStore(waku, buildUserPrivateStoreTopic) + : new LocalStorageStore() + return new EncryptedStore(wallet, backingStore) +} + // attempt to load pre-existing key bundle from storage, // otherwise create new key-bundle, store it and return it -async function loadOrCreateKeys(wallet: Signer): Promise { - const store = new EncryptedStore(wallet, new LocalStorageStore()) +export async function loadOrCreateKeys( + wallet: Signer, + store: EncryptedStore +): Promise { let keys = await store.loadPrivateKeyBundle() if (keys) { return keys @@ -276,7 +293,7 @@ async function loadOrCreateKeys(wallet: Signer): Promise { } // initialize connection to the network -async function createWaku({ +export async function createWaku({ bootstrapAddrs, env = 'testnet', waitForPeersTimeoutMs, diff --git a/src/store/NetworkStore.ts b/src/store/NetworkStore.ts index 6aad63388..7265cd5b7 100644 --- a/src/store/NetworkStore.ts +++ b/src/store/NetworkStore.ts @@ -1,23 +1,33 @@ // This creates an interface for storing data to the storage network. import { Store } from './Store' - -const KEY_PREFIX = '/xmtp/' -const ENCODING = 'binary' +import { Waku, WakuMessage, PageDirection } from 'js-waku' export default class NetworkStore implements Store { - // Include a key prefix to namespace items in LocalStorage - // This will prevent us from squashing any values set by other libraries on the site - keyPrefix: string + private waku: Waku + keyGenerator: (str: string) => string - constructor(keyPrefix: string = KEY_PREFIX) { - this.keyPrefix = keyPrefix + constructor(waku: Waku, keyGenerator: (str: string) => string) { + this.waku = waku + this.keyGenerator = keyGenerator } + // Returns the first record in a topic if it is present. async get(key: string): Promise { - return Promise.resolve(Buffer.from(key)) + const contents = ( + await this.waku.store.queryHistory([this.keyGenerator(key)], { + pageSize: 1, + pageDirection: PageDirection.FORWARD, + }) + ) + .filter((msg: WakuMessage) => msg.payload) + .map((msg: WakuMessage) => msg.payload as Uint8Array) + return contents.length > 0 ? Buffer.from(contents[0]) : null } async set(key: string, value: Buffer): Promise { - return Promise.resolve() + const keys = Uint8Array.from(value) + await this.waku.relay.send( + await WakuMessage.fromBytes(keys, this.keyGenerator(key)) + ) } } diff --git a/src/store/index.ts b/src/store/index.ts index 3ee69a99f..a88897c71 100644 --- a/src/store/index.ts +++ b/src/store/index.ts @@ -1,3 +1,4 @@ export { default as LocalStorageStore } from './LocalStorageStore' export { default as EncryptedStore } from './EncryptedStore' +export { default as NetworkStore } from './NetworkStore' export { Store } from './Store' diff --git a/src/utils.ts b/src/utils.ts index 8d6b2ea36..cacb54421 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -18,6 +18,10 @@ export const buildUserIntroTopic = (walletAddr: string): string => { return buildContentTopic(`intro-${walletAddr}`) } +export const buildUserPrivateStoreTopic = (walletAddr: string): string => { + return buildContentTopic(`privatestore-${walletAddr}`) +} + export const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) diff --git a/test/store/NetworkStore.test.ts b/test/store/NetworkStore.test.ts index f1c524c3a..b1856bf88 100644 --- a/test/store/NetworkStore.test.ts +++ b/test/store/NetworkStore.test.ts @@ -1,29 +1,53 @@ +import { Wallet } from 'ethers' +import { Waku } from 'js-waku' + +import { newWallet, sleep } from '../helpers' +import { createWaku } from '../../src/Client' import { NetworkStore } from '../../src/store' +import { buildUserPrivateStoreTopic } from '../../src/utils' + +const newLocalDockerWaku = (): Promise => + createWaku({ + bootstrapAddrs: [ + '/ip4/127.0.0.1/tcp/9001/ws/p2p/16Uiu2HAmNCxLZCkXNbpVPBpSSnHj9iq4HZQj7fxRzw2kj1kKSHHA', + ], + }) + +const newTestnetWaku = (): Promise => createWaku({ env: 'testnet' }) describe('NetworkStore', () => { jest.setTimeout(10000) const tests = [ { name: 'local docker node', - newWaku: undefined, + newWaku: newLocalDockerWaku, }, ] if (process.env.CI || process.env.TESTNET) { tests.push({ name: 'testnet', - newWaku: undefined, + newWaku: newTestnetWaku, }) } tests.forEach((testCase) => { describe(testCase.name, () => { + let waku: Waku + let wallet: Wallet let store: NetworkStore + beforeAll(async () => { + waku = await testCase.newWaku() + }) + afterAll(async () => { + if (waku) await waku.stop() + }) beforeEach(async () => { - store = new NetworkStore() + wallet = newWallet() + store = new NetworkStore(waku, buildUserPrivateStoreTopic) }) it('roundtrip', async () => { - const key = 'key' + const key = wallet.address const value = new TextEncoder().encode('hello') const empty = await store.get(key) @@ -39,8 +63,8 @@ describe('NetworkStore', () => { it('distinct topics', async () => { const valueA = Buffer.from(new TextEncoder().encode('helloA')) const valueB = Buffer.from(new TextEncoder().encode('helloB')) - const keyA = 'keyA' - const keyB = 'keyB' + const keyA = wallet.address + 'A' + const keyB = wallet.address + 'B' store.set(keyA, valueA) store.set(keyB, valueB) @@ -53,13 +77,15 @@ describe('NetworkStore', () => { }) it('over write safety', async () => { - const key = 'key' + const key = wallet.address const first_value = new TextEncoder().encode('a') const second_value = new TextEncoder().encode('bb') await store.set(key, Buffer.from(first_value)) + await sleep(10) // Add wait to enforce a consistent order of messages await store.set(key, Buffer.from(second_value)) + await sleep(10) // Add wait to enforce a consistent order of messages const returned_value = await store.get(key) expect(returned_value).toEqual(Buffer.from(first_value)) From f4ab71a8a49845372041aef676d3d8509de48994 Mon Sep 17 00:00:00 2001 From: Jazz Turner-Baggs Date: Mon, 14 Mar 2022 09:53:27 -0700 Subject: [PATCH 3/7] feat: add option to use localStorage keyBundles --- src/Client.ts | 31 +++++++++++++++++++------------ src/store/NetworkStore.ts | 13 ++++++++----- test/store/NetworkStore.test.ts | 2 +- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/Client.ts b/src/Client.ts index ad534fdc1..a8c11c812 100644 --- a/src/Client.ts +++ b/src/Client.ts @@ -79,13 +79,18 @@ export default class Client { * * @param wallet the wallet as a Signer instance * @param opts specify how to to connect to the network + * @param useLocalKeyStore force client to use localStorage for persisting privateKeyBundles */ - static async create(wallet: Signer, opts?: CreateOptions): Promise { + static async create( + wallet: Signer, + opts?: CreateOptions, + useLocalKeyStore = false + ): Promise { const waku = await createWaku(opts || {}) - const keys = await loadOrCreateKeys( - wallet, - createPrivateKeyStore(wallet, waku) - ) + const keyStore = useLocalKeyStore + ? createNetworkPrivateKeyStore(wallet, waku) + : createLocalPrivateKeyStore(wallet) + const keys = await loadOrCreateKeys(wallet, keyStore) const client = new Client(waku, keys) await client.publishUserContact() return client @@ -266,15 +271,17 @@ export default class Client { } } -// select appropriate store for privateKeyBundles -export function createPrivateKeyStore( +// Create Encrypted store which uses the Network to store KeyBundles +export function createNetworkPrivateKeyStore( wallet: Signer, - waku: Waku | undefined = undefined + waku: Waku ): EncryptedStore { - const backingStore = waku - ? new NetworkStore(waku, buildUserPrivateStoreTopic) - : new LocalStorageStore() - return new EncryptedStore(wallet, backingStore) + return new EncryptedStore(wallet, new NetworkStore(waku)) +} + +// Create Encrypted store which uses LocalStorage to store KeyBundles +export function createLocalPrivateKeyStore(wallet: Signer): EncryptedStore { + return new EncryptedStore(wallet, new LocalStorageStore()) } // attempt to load pre-existing key bundle from storage, diff --git a/src/store/NetworkStore.ts b/src/store/NetworkStore.ts index 7265cd5b7..2ed48c7bd 100644 --- a/src/store/NetworkStore.ts +++ b/src/store/NetworkStore.ts @@ -1,20 +1,19 @@ // This creates an interface for storing data to the storage network. import { Store } from './Store' import { Waku, WakuMessage, PageDirection } from 'js-waku' +import { buildUserPrivateStoreTopic } from '../utils' export default class NetworkStore implements Store { private waku: Waku - keyGenerator: (str: string) => string - constructor(waku: Waku, keyGenerator: (str: string) => string) { + constructor(waku: Waku) { this.waku = waku - this.keyGenerator = keyGenerator } // Returns the first record in a topic if it is present. async get(key: string): Promise { const contents = ( - await this.waku.store.queryHistory([this.keyGenerator(key)], { + await this.waku.store.queryHistory([buildUserPrivateStoreTopic(key)], { pageSize: 1, pageDirection: PageDirection.FORWARD, }) @@ -27,7 +26,11 @@ export default class NetworkStore implements Store { async set(key: string, value: Buffer): Promise { const keys = Uint8Array.from(value) await this.waku.relay.send( - await WakuMessage.fromBytes(keys, this.keyGenerator(key)) + await WakuMessage.fromBytes(keys, this.buildTopic(key)) ) } + + private buildTopic(key: string): string { + return buildUserPrivateStoreTopic(key) + } } diff --git a/test/store/NetworkStore.test.ts b/test/store/NetworkStore.test.ts index b1856bf88..c0d184fbe 100644 --- a/test/store/NetworkStore.test.ts +++ b/test/store/NetworkStore.test.ts @@ -43,7 +43,7 @@ describe('NetworkStore', () => { beforeEach(async () => { wallet = newWallet() - store = new NetworkStore(waku, buildUserPrivateStoreTopic) + store = new NetworkStore(waku) }) it('roundtrip', async () => { From b10623a66098e29429324562c97ed2784a5dabd7 Mon Sep 17 00:00:00 2001 From: Jazz Turner-Baggs Date: Mon, 14 Mar 2022 13:10:14 -0700 Subject: [PATCH 4/7] refactor: add short-circut topic fetch --- src/store/NetworkStore.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/store/NetworkStore.ts b/src/store/NetworkStore.ts index 2ed48c7bd..0ad5f17ba 100644 --- a/src/store/NetworkStore.ts +++ b/src/store/NetworkStore.ts @@ -16,6 +16,9 @@ export default class NetworkStore implements Store { await this.waku.store.queryHistory([buildUserPrivateStoreTopic(key)], { pageSize: 1, pageDirection: PageDirection.FORWARD, + callback: function (msgs: WakuMessage[]): boolean | void { + return Boolean(msgs[0].payload) + }, }) ) .filter((msg: WakuMessage) => msg.payload) From 4b2081e467ef514fae0a9938ddd250b64329833f Mon Sep 17 00:00:00 2001 From: Jazz Turner-Baggs Date: Mon, 14 Mar 2022 13:18:54 -0700 Subject: [PATCH 5/7] refactor: renamed networkStore --- src/Client.ts | 5 ++--- .../{NetworkStore.ts => PrivateTopicStore.ts} | 0 src/store/index.ts | 2 +- test/store/NetworkStore.test.ts | 21 +++++++++---------- 4 files changed, 13 insertions(+), 15 deletions(-) rename src/store/{NetworkStore.ts => PrivateTopicStore.ts} (100%) diff --git a/src/Client.ts b/src/Client.ts index a8c11c812..e8f6f051f 100644 --- a/src/Client.ts +++ b/src/Client.ts @@ -7,13 +7,12 @@ import { buildDirectMessageTopic, buildUserContactTopic, buildUserIntroTopic, - buildUserPrivateStoreTopic, promiseWithTimeout, } from './utils' import { sleep } from '../test/helpers' import Stream, { messageStream } from './Stream' import { Signer } from 'ethers' -import { EncryptedStore, LocalStorageStore, NetworkStore } from './store' +import { EncryptedStore, LocalStorageStore, PrivateTopicStore } from './store' import { Conversations } from './conversations' const NODES_LIST_URL = 'https://nodes.xmtp.com/' @@ -276,7 +275,7 @@ export function createNetworkPrivateKeyStore( wallet: Signer, waku: Waku ): EncryptedStore { - return new EncryptedStore(wallet, new NetworkStore(waku)) + return new EncryptedStore(wallet, new PrivateTopicStore(waku)) } // Create Encrypted store which uses LocalStorage to store KeyBundles diff --git a/src/store/NetworkStore.ts b/src/store/PrivateTopicStore.ts similarity index 100% rename from src/store/NetworkStore.ts rename to src/store/PrivateTopicStore.ts diff --git a/src/store/index.ts b/src/store/index.ts index a88897c71..b4598b9ce 100644 --- a/src/store/index.ts +++ b/src/store/index.ts @@ -1,4 +1,4 @@ export { default as LocalStorageStore } from './LocalStorageStore' export { default as EncryptedStore } from './EncryptedStore' -export { default as NetworkStore } from './NetworkStore' +export { default as PrivateTopicStore } from './PrivateTopicStore' export { Store } from './Store' diff --git a/test/store/NetworkStore.test.ts b/test/store/NetworkStore.test.ts index c0d184fbe..d9c997f3d 100644 --- a/test/store/NetworkStore.test.ts +++ b/test/store/NetworkStore.test.ts @@ -3,8 +3,7 @@ import { Waku } from 'js-waku' import { newWallet, sleep } from '../helpers' import { createWaku } from '../../src/Client' -import { NetworkStore } from '../../src/store' -import { buildUserPrivateStoreTopic } from '../../src/utils' +import { PrivateTopicStore } from '../../src/store' const newLocalDockerWaku = (): Promise => createWaku({ @@ -15,7 +14,7 @@ const newLocalDockerWaku = (): Promise => const newTestnetWaku = (): Promise => createWaku({ env: 'testnet' }) -describe('NetworkStore', () => { +describe('PrivateTopicStore', () => { jest.setTimeout(10000) const tests = [ { @@ -33,7 +32,7 @@ describe('NetworkStore', () => { describe(testCase.name, () => { let waku: Waku let wallet: Wallet - let store: NetworkStore + let store: PrivateTopicStore beforeAll(async () => { waku = await testCase.newWaku() }) @@ -43,7 +42,7 @@ describe('NetworkStore', () => { beforeEach(async () => { wallet = newWallet() - store = new NetworkStore(waku) + store = new PrivateTopicStore(waku) }) it('roundtrip', async () => { @@ -79,16 +78,16 @@ describe('NetworkStore', () => { it('over write safety', async () => { const key = wallet.address - const first_value = new TextEncoder().encode('a') - const second_value = new TextEncoder().encode('bb') + const firstValue = new TextEncoder().encode('a') + const secondValue = new TextEncoder().encode('bb') - await store.set(key, Buffer.from(first_value)) + await store.set(key, Buffer.from(firstValue)) await sleep(10) // Add wait to enforce a consistent order of messages - await store.set(key, Buffer.from(second_value)) + await store.set(key, Buffer.from(secondValue)) await sleep(10) // Add wait to enforce a consistent order of messages - const returned_value = await store.get(key) + const returnedValue = await store.get(key) - expect(returned_value).toEqual(Buffer.from(first_value)) + expect(returnedValue).toEqual(Buffer.from(firstValue)) }) }) }) From a39cd2ca093195636554f7a1646056718a85e70c Mon Sep 17 00:00:00 2001 From: Jazz Turner-Baggs Date: Mon, 14 Mar 2022 16:51:11 -0700 Subject: [PATCH 6/7] fix: export changes + type fixes --- src/Client.ts | 6 +++--- src/store/PrivateTopicStore.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Client.ts b/src/Client.ts index e8f6f051f..87753c12a 100644 --- a/src/Client.ts +++ b/src/Client.ts @@ -271,7 +271,7 @@ export default class Client { } // Create Encrypted store which uses the Network to store KeyBundles -export function createNetworkPrivateKeyStore( +function createNetworkPrivateKeyStore( wallet: Signer, waku: Waku ): EncryptedStore { @@ -279,13 +279,13 @@ export function createNetworkPrivateKeyStore( } // Create Encrypted store which uses LocalStorage to store KeyBundles -export function createLocalPrivateKeyStore(wallet: Signer): EncryptedStore { +function createLocalPrivateKeyStore(wallet: Signer): EncryptedStore { return new EncryptedStore(wallet, new LocalStorageStore()) } // attempt to load pre-existing key bundle from storage, // otherwise create new key-bundle, store it and return it -export async function loadOrCreateKeys( +async function loadOrCreateKeys( wallet: Signer, store: EncryptedStore ): Promise { diff --git a/src/store/PrivateTopicStore.ts b/src/store/PrivateTopicStore.ts index 0ad5f17ba..5a23ee94a 100644 --- a/src/store/PrivateTopicStore.ts +++ b/src/store/PrivateTopicStore.ts @@ -16,7 +16,7 @@ export default class NetworkStore implements Store { await this.waku.store.queryHistory([buildUserPrivateStoreTopic(key)], { pageSize: 1, pageDirection: PageDirection.FORWARD, - callback: function (msgs: WakuMessage[]): boolean | void { + callback: function (msgs: WakuMessage[]): boolean { return Boolean(msgs[0].payload) }, }) From 3e8eb6eb9d5d0d821c8d99f9bd1894cfe079aac0 Mon Sep 17 00:00:00 2001 From: Jazz Turner-Baggs Date: Tue, 15 Mar 2022 08:45:54 -0700 Subject: [PATCH 7/7] refactor: removed key create api changes --- src/Client.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/Client.ts b/src/Client.ts index 87753c12a..4ffe76d75 100644 --- a/src/Client.ts +++ b/src/Client.ts @@ -78,17 +78,10 @@ export default class Client { * * @param wallet the wallet as a Signer instance * @param opts specify how to to connect to the network - * @param useLocalKeyStore force client to use localStorage for persisting privateKeyBundles */ - static async create( - wallet: Signer, - opts?: CreateOptions, - useLocalKeyStore = false - ): Promise { + static async create(wallet: Signer, opts?: CreateOptions): Promise { const waku = await createWaku(opts || {}) - const keyStore = useLocalKeyStore - ? createNetworkPrivateKeyStore(wallet, waku) - : createLocalPrivateKeyStore(wallet) + const keyStore = createNetworkPrivateKeyStore(wallet, waku) const keys = await loadOrCreateKeys(wallet, keyStore) const client = new Client(waku, keys) await client.publishUserContact()