From 3d1f8258465e1177d6d5fe17c2e1ea786baa331d Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Fri, 23 Aug 2024 16:53:52 -0400 Subject: [PATCH] [WIP] SyncEngine for specific protocols + delegate sync. (#836) * first pass at connect flow and grants api * PermissionsApi for Agent, `permissions` API for `Web5` (#833) This refactors a lot of what's in https://github.com/TBD54566975/web5-js/pull/824 with regards to creating/fetching grants. Satisfies: https://github.com/TBD54566975/web5-js/issues/827 Introduces a `PermissionsApi` interface and an `AgentPermissionsApi` concrete implementation. The interface implements the following methods `fetchGrants`, `fetchRequests`, `isGrantRevoked`, `createGrant`, `createRequest`, `createRevocation` as convenience methods for dealing with the built-in permission protocol records. The `AgentPermissionsApi` implements an additional static method `matchGrantFromArray` which was moved from a `PermissionsUtil` class, which is used to find the appropriate grant to use when authoring a message. A Private API used in a connected state to find and cache the correct grants to use for the request. A Permissions API which implements `request`, `grant`, `queryRequests`, and `queryGrants` that a user can utilize The `Web5` permissions api introduces 3 helper classes to represent permissions: Class to represent a permission request record. It implements convenience methods similar to the `Record` class where you can `store()`, `import()` or `send()` the underlying request record. Additionally a `grant()` method will create a `PermissionGrant` object. Class to represent a grant record. It implements convenience methods similar to the `Record` class where you can `store()`, `import()` or `send()` the underlying grant record. Additionally a `revoke()` method will create a `GrantRevocation` object, and `isRevoked()` will check if the underlying grant has been revoked. Class to represent a permission grant revocation record. It implements convenience methods similar to the `Record` class where you can `store()` or `send()` the underlying revocation record. --- .changeset/blue-roses-cough.md | 8 + .changeset/green-dolls-provide.md | 5 + audit-ci.json | 3 +- packages/agent/src/cached-permissions.ts | 66 + packages/agent/src/dwn-api.ts | 12 + packages/agent/src/index.ts | 1 + packages/agent/src/store-data.ts | 2 +- packages/agent/src/sync-api.ts | 8 +- packages/agent/src/sync-engine-level.ts | 234 ++- packages/agent/src/types/sync.ts | 7 +- .../agent/tests/cached-permissions.spec.ts | 240 ++++ packages/agent/tests/dwn-api.spec.ts | 48 +- packages/agent/tests/store-data.spec.ts | 2 +- .../agent/tests/sync-engine-level.spec.ts | 1270 ++++++++++++++++- packages/api/src/dwn-api.ts | 62 +- packages/api/src/web5.ts | 72 +- packages/api/tests/dwn-api.spec.ts | 11 +- packages/api/tests/web5.spec.ts | 1019 +++++-------- 18 files changed, 2267 insertions(+), 803 deletions(-) create mode 100644 .changeset/blue-roses-cough.md create mode 100644 .changeset/green-dolls-provide.md create mode 100644 packages/agent/src/cached-permissions.ts create mode 100644 packages/agent/tests/cached-permissions.spec.ts diff --git a/.changeset/blue-roses-cough.md b/.changeset/blue-roses-cough.md new file mode 100644 index 000000000..60f3b5668 --- /dev/null +++ b/.changeset/blue-roses-cough.md @@ -0,0 +1,8 @@ +--- +"@web5/agent": minor +"@web5/identity-agent": minor +"@web5/proxy-agent": minor +"@web5/user-agent": minor +--- + +Add ability to Sync a subset of protocols as a delegate diff --git a/.changeset/green-dolls-provide.md b/.changeset/green-dolls-provide.md new file mode 100644 index 000000000..5f5d26a9c --- /dev/null +++ b/.changeset/green-dolls-provide.md @@ -0,0 +1,5 @@ +--- +"@web5/api": minor +--- + +Finalize ability to WalletConnect with sync involved diff --git a/audit-ci.json b/audit-ci.json index 2b81d5ed7..0311fe9e4 100644 --- a/audit-ci.json +++ b/audit-ci.json @@ -5,6 +5,7 @@ "ip", "mysql2", "braces", - "GHSA-rv95-896h-c2vc" + "GHSA-rv95-896h-c2vc", + "GHSA-952p-6rrq-rcjv" ] } \ No newline at end of file diff --git a/packages/agent/src/cached-permissions.ts b/packages/agent/src/cached-permissions.ts new file mode 100644 index 000000000..81c11e29e --- /dev/null +++ b/packages/agent/src/cached-permissions.ts @@ -0,0 +1,66 @@ +import { TtlCache } from '@web5/common'; +import { AgentPermissionsApi } from './permissions-api.js'; +import { Web5Agent } from './types/agent.js'; +import { PermissionGrantEntry } from './types/permissions.js'; +import { DwnInterface } from './types/dwn.js'; + +export class CachedPermissions { + + /** the default value for whether a fetch is cached or not */ + private cachedDefault: boolean; + + /** Holds the instance of {@link AgentPermissionsApi} that helps when dealing with permissions protocol records */ + private permissionsApi: AgentPermissionsApi; + + /** cache for fetching a permission {@link PermissionGrant}, keyed by a specific MessageType and protocol */ + private cachedPermissions: TtlCache = new TtlCache({ ttl: 60 * 1000 }); + + constructor({ agent, cachedDefault }:{ agent: Web5Agent, cachedDefault?: boolean }) { + this.permissionsApi = new AgentPermissionsApi({ agent }); + this.cachedDefault = cachedDefault ?? false; + } + + public async getPermission({ connectedDid, delegateDid, delegate, messageType, protocol, cached = this.cachedDefault }: { + connectedDid: string; + delegateDid: string; + messageType: T; + protocol?: string; + cached?: boolean; + delegate?: boolean; + }): Promise { + // Currently we only support finding grants based on protocols + // A different approach may be necessary when we introduce `protocolPath` and `contextId` specific impersonation + const cacheKey = [ connectedDid, delegateDid, messageType, protocol ].join('~'); + const cachedGrant = cached ? this.cachedPermissions.get(cacheKey) : undefined; + if (cachedGrant) { + return cachedGrant; + } + + const permissionGrants = await this.permissionsApi.fetchGrants({ + author : delegateDid, + target : delegateDid, + grantor : connectedDid, + grantee : delegateDid, + }); + + // get the delegate grants that match the messageParams and are associated with the connectedDid as the grantor + const grant = await AgentPermissionsApi.matchGrantFromArray( + connectedDid, + delegateDid, + { messageType, protocol }, + permissionGrants, + delegate + ); + + if (!grant) { + throw new Error(`CachedPermissions: No permissions found for ${messageType}: ${protocol}`); + } + + this.cachedPermissions.set(cacheKey, grant); + return grant; + } + + public async clear(): Promise { + this.cachedPermissions.clear(); + } +} \ No newline at end of file diff --git a/packages/agent/src/dwn-api.ts b/packages/agent/src/dwn-api.ts index ca6414b67..4626ef2e3 100644 --- a/packages/agent/src/dwn-api.ts +++ b/packages/agent/src/dwn-api.ts @@ -5,6 +5,7 @@ import { DataStoreLevel, Dwn, DwnConfig, + DwnInterfaceName, DwnMethodName, EventLogLevel, GenericMessage, @@ -23,8 +24,11 @@ import type { DwnMessageInstance, DwnMessageParams, DwnMessageReply, + DwnMessagesPermissionScope, DwnMessageWithData, + DwnPermissionScope, DwnRecordsInterfaces, + DwnRecordsPermissionScope, DwnResponse, DwnSigner, MessageHandler, @@ -70,6 +74,14 @@ export function isRecordsType(messageType: DwnInterface): messageType is DwnReco messageType === DwnInterface.RecordsWrite; } +export function isRecordPermissionScope(scope: DwnPermissionScope): scope is DwnRecordsPermissionScope { + return scope.interface === DwnInterfaceName.Records; +} + +export function isMessagesPermissionScope(scope: DwnPermissionScope): scope is DwnMessagesPermissionScope { + return scope.interface === DwnInterfaceName.Messages; +} + export class AgentDwnApi { /** * Holds the instance of a `Web5PlatformAgent` that represents the current execution context for diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index b3f1b3f02..5ed8caa62 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -8,6 +8,7 @@ export type * from './types/sync.js'; export type * from './types/vc.js'; export * from './bearer-identity.js'; +export * from './cached-permissions.js'; export * from './crypto-api.js'; export * from './did-api.js'; export * from './dwn-api.js'; diff --git a/packages/agent/src/store-data.ts b/packages/agent/src/store-data.ts index 21060e558..6873eaa0e 100644 --- a/packages/agent/src/store-data.ts +++ b/packages/agent/src/store-data.ts @@ -165,7 +165,7 @@ export class DwnDataStore = Jwk> implem // If the write fails, throw an error. if (!(message && status.code === 202)) { - throw new Error(`${this.name}: Failed to write data to store for: ${id}`); + throw new Error(`${this.name}: Failed to write data to store for ${id}: ${status.detail}`); } // Add the ID of the newly created record to the index. diff --git a/packages/agent/src/sync-api.ts b/packages/agent/src/sync-api.ts index f68f45737..3c7ea1089 100644 --- a/packages/agent/src/sync-api.ts +++ b/packages/agent/src/sync-api.ts @@ -1,4 +1,4 @@ -import type { SyncEngine } from './types/sync.js'; +import type { SyncEngine, SyncIdentityOptions } from './types/sync.js'; import type { Web5PlatformAgent } from './types/agent.js'; export type SyncApiParams = { @@ -41,10 +41,14 @@ export class AgentSyncApi implements SyncEngine { this._syncEngine.agent = agent; } - public async registerIdentity(params: { did: string; }): Promise { + public async registerIdentity(params: { did: string; options: SyncIdentityOptions }): Promise { await this._syncEngine.registerIdentity(params); } + public sync(direction?: 'push' | 'pull'): Promise { + return this._syncEngine.sync(direction); + } + public startSync(params: { interval: string; }): Promise { return this._syncEngine.startSync(params); } diff --git a/packages/agent/src/sync-engine-level.ts b/packages/agent/src/sync-engine-level.ts index 833a60cce..259667cb8 100644 --- a/packages/agent/src/sync-engine-level.ts +++ b/packages/agent/src/sync-engine-level.ts @@ -17,11 +17,12 @@ import { DwnMethodName, } from '@tbd54566975/dwn-sdk-js'; -import type { SyncEngine } from './types/sync.js'; -import type { Web5PlatformAgent } from './types/agent.js'; +import type { SyncEngine, SyncIdentityOptions } from './types/sync.js'; +import type { Web5Agent, Web5PlatformAgent } from './types/agent.js'; import { DwnInterface } from './types/dwn.js'; import { getDwnServiceEndpointUrls, isRecordsWrite } from './utils.js'; +import { CachedPermissions } from './cached-permissions.js'; export type SyncEngineLevelParams = { agent?: Web5PlatformAgent; @@ -35,8 +36,20 @@ type SyncDirection = 'push' | 'pull'; type SyncState = { did: string; + delegateDid?: string; dwnUrl: string; cursor?: PaginationCursor, + protocol?: string; +} + +type SyncMessageParams = { + did: string; + messageCid: string; + watermark: string; + dwnUrl: string; + delegateDid?: string; + cursor?: PaginationCursor, + protocol?: string; } export class SyncEngineLevel implements SyncEngine { @@ -48,12 +61,18 @@ export class SyncEngineLevel implements SyncEngine { */ private _agent?: Web5PlatformAgent; + /** + * An instance of the `AgentPermissionsApi` that is used to interact with permissions grants used during sync + */ + private _cachedPermissionsApi: CachedPermissions; + private _db: AbstractLevel; private _syncIntervalId?: ReturnType; private _ulidFactory: ULIDFactory; constructor({ agent, dataPath, db }: SyncEngineLevelParams) { this._agent = agent; + this._cachedPermissionsApi = new CachedPermissions({ agent: agent as Web5Agent, cachedDefault: true }); this._db = (db) ? db : new Level(dataPath ?? 'DATA/AGENT/SYNC_STORE'); this._ulidFactory = monotonicFactory(); } @@ -74,9 +93,11 @@ export class SyncEngineLevel implements SyncEngine { set agent(agent: Web5PlatformAgent) { this._agent = agent; + this._cachedPermissionsApi = new CachedPermissions({ agent: agent as Web5Agent, cachedDefault: true }); } public async clear(): Promise { + await this._cachedPermissionsApi.clear(); await this._db.clear(); } @@ -84,7 +105,7 @@ export class SyncEngineLevel implements SyncEngine { await this._db.close(); } - public async pull(): Promise { + private async pull(): Promise { const syncPeerState = await this.getSyncPeerState({ syncDirection: 'pull' }); await this.enqueueOperations({ syncDirection: 'pull', syncPeerState }); @@ -96,8 +117,7 @@ export class SyncEngineLevel implements SyncEngine { for (let job of pullJobs) { const [key] = job; - const [did, dwnUrl, _, messageCid] = key.split('~'); - + const { did, dwnUrl, messageCid, delegateDid, protocol } = SyncEngineLevel.parseSyncMessageParamsKey(key); // If a particular DWN service endpoint is unreachable, skip subsequent pull operations. if (errored.has(dwnUrl)) { continue; @@ -109,13 +129,34 @@ export class SyncEngineLevel implements SyncEngine { continue; } + let permissionGrantId: string | undefined; + let granteeDid: string | undefined; + if (delegateDid) { + try { + const messagesReadGrant = await this._cachedPermissionsApi.getPermission({ + connectedDid : did, + messageType : DwnInterface.MessagesRead, + delegateDid, + protocol, + }); + + permissionGrantId = messagesReadGrant.grant.id; + granteeDid = delegateDid; + } catch(error:any) { + console.error('SyncEngineLevel: pull - Error fetching MessagesRead permission grant for delegate DID', error); + continue; + } + } + const messagesRead = await this.agent.processDwnRequest({ store : false, author : did, target : did, messageType : DwnInterface.MessagesRead, + granteeDid, messageParams : { - messageCid: messageCid + messageCid, + permissionGrantId } }); @@ -123,8 +164,7 @@ export class SyncEngineLevel implements SyncEngine { try { reply = await this.agent.rpc.sendDwnRequest({ - dwnUrl, - targetDid : did, + dwnUrl, targetDid : did, message : messagesRead.message, }) as MessagesReadReply; } catch(e) { @@ -157,7 +197,7 @@ export class SyncEngineLevel implements SyncEngine { await pullQueue.batch(deleteOperations as any); } - public async push(): Promise { + private async push(): Promise { const syncPeerState = await this.getSyncPeerState({ syncDirection: 'push' }); await this.enqueueOperations({ syncDirection: 'push', syncPeerState }); @@ -169,15 +209,14 @@ export class SyncEngineLevel implements SyncEngine { for (let job of pushJobs) { const [key] = job; - const [did, dwnUrl, _, messageCid] = key.split('~'); - + const { did, delegateDid, protocol, dwnUrl, messageCid } = SyncEngineLevel.parseSyncMessageParamsKey(key); // If a particular DWN service endpoint is unreachable, skip subsequent push operations. if (errored.has(dwnUrl)) { continue; } // Attempt to retrieve the message from the local DWN. - const dwnMessage = await this.getDwnMessage({ author: did, messageCid }); + const dwnMessage = await this.getDwnMessage({ author: did, messageCid, delegateDid, protocol }); // If the message does not exist on the local DWN, remove the sync operation from the // push queue, update the push watermark for this DID/DWN endpoint combination, add the @@ -198,9 +237,6 @@ export class SyncEngineLevel implements SyncEngine { }); // Update the watermark and add the messageCid to the Sync Message Store if either: - // - 202: message was successfully written to the remote DWN - // - 409: message was already present on the remote DWN - // - RecordsDelete and the status code is 404: the initial write message was not found or the message was already deleted if (SyncEngineLevel.syncMessageReplyIsSuccessful(reply)) { await this.addMessage(did, messageCid); deleteOperations.push({ type: 'del', key: key }); @@ -214,12 +250,25 @@ export class SyncEngineLevel implements SyncEngine { await pushQueue.batch(deleteOperations as any); } - public async registerIdentity({ did }: { did: string; }): Promise { + public async registerIdentity({ did, options }: { did: string; options: SyncIdentityOptions }): Promise { // Get a reference to the `registeredIdentities` sublevel. const registeredIdentities = this._db.sublevel('registeredIdentities'); // Add (or overwrite, if present) the Identity's DID as a registered identity. - await registeredIdentities.put(did, ''); + await registeredIdentities.put(did, JSON.stringify(options)); + } + + public async sync(direction?: 'push' | 'pull'): Promise { + if (this._syncIntervalId) { + throw new Error('SyncEngineLevel: Cannot call sync while a sync interval is active. Call `stopSync()` first.'); + } + + if (!direction || direction === 'push') { + await this.push(); + } + if (!direction || direction === 'pull') { + await this.pull(); + } } public startSync({ interval }: { @@ -236,8 +285,7 @@ export class SyncEngineLevel implements SyncEngine { } try { - await this.push(); - await this.pull(); + await this.sync(); } catch (error: any) { this.stopSync(); reject(error); @@ -258,8 +306,18 @@ export class SyncEngineLevel implements SyncEngine { } } + /** + * 202: message was successfully written to the remote DWN + * 204: an initial write message was written without any data, cannot yet be read until a subsequent message is written with data + * 409: message was already present on the remote DWN + * RecordsDelete and the status code is 404: the initial write message was not found or the message was already deleted + */ private static syncMessageReplyIsSuccessful(reply: UnionMessageReply): boolean { return reply.status.code === 202 || + // a 204 status code is returned when the message was accepted without any data. + // This is the case for an initial RecordsWrite messages for records that have been updated. + // For context: https://github.com/TBD54566975/dwn-sdk-js/issues/695 + reply.status.code === 204 || reply.status.code === 409 || ( // If the message is a RecordsDelete and the status code is 404, the initial write message was not found or the message was already deleted @@ -276,9 +334,11 @@ export class SyncEngineLevel implements SyncEngine { for (let syncState of syncPeerState) { // Get the event log from the remote DWN if pull sync, or local DWN if push sync. const eventLog = await this.getDwnEventLog({ - did : syncState.did, - dwnUrl : syncState.dwnUrl, - cursor : syncState.cursor, + did : syncState.did, + delegateDid : syncState.delegateDid, + dwnUrl : syncState.dwnUrl, + cursor : syncState.cursor, + protocol : syncState.protocol, syncDirection }); @@ -286,15 +346,11 @@ export class SyncEngineLevel implements SyncEngine { for (let messageCid of eventLog) { const watermark = this._ulidFactory(); - // Use "did~dwnUrl~watermark~messageCid" as the key in the sync queue. - // Note: It is critical that `watermark` precedes `messageCid` to ensure that when the sync - // jobs are pulled off the queue, they are lexographically sorted oldest to newest. - const operationKey = [ - syncState.did, - syncState.dwnUrl, + const operationKey = SyncEngineLevel.generateSyncMessageParamsKey({ + ...syncState, watermark, messageCid - ].join('~'); + }); syncOperations.push({ type: 'put', key: operationKey, value: '' }); } @@ -308,68 +364,130 @@ export class SyncEngineLevel implements SyncEngine { } } - private async getDwnEventLog({ did, dwnUrl, syncDirection, cursor }: { + private static generateSyncMessageParamsKey({ did, delegateDid, dwnUrl, protocol, watermark, messageCid }:SyncMessageParams): string { + // Use "did~dwnUrl~watermark~messageCid" as the key in the sync queue. + // Note: It is critical that `watermark` precedes `messageCid` to ensure that when the sync + // jobs are pulled off the queue, they are lexographically sorted oldest to newest. + // + // `protocol` and `delegateDid` may be undefined, which is fine, its part of the key will be stored as an empty string. + // Later, when parsing the key, we will handle this case and return an actual undefined. + // This is information useful for subset and delegated sync. + return [did, delegateDid, dwnUrl, protocol, watermark, messageCid ].join('~'); + } + + private static parseSyncMessageParamsKey(key: string): SyncMessageParams { + // The order is import here, see `generateKey` for more information. + const [did, delegateDidString, dwnUrl, protocolString, watermark, messageCid] = key.split('~'); + + // `protocol` or `delegateDid` may be parsed as an empty string, so we need to handle that case and returned an actual undefined. + const protocol = protocolString === '' ? undefined : protocolString; + const delegateDid = delegateDidString === '' ? undefined : delegateDidString; + return { did, delegateDid, dwnUrl, watermark, messageCid, protocol }; + } + + private async getDwnEventLog({ did, delegateDid, dwnUrl, syncDirection, cursor, protocol }: { did: string, + delegateDid?: string, dwnUrl: string, syncDirection: SyncDirection, cursor?: PaginationCursor + protocol?: string }) { let messagesReply = {} as MessagesQueryReply; + let permissionGrantId: string | undefined; + if (delegateDid) { + // fetch the grants for the delegate DID + try { + const messagesQueryGrant = await this._cachedPermissionsApi.getPermission({ + connectedDid : did, + messageType : DwnInterface.MessagesQuery, + delegateDid, + protocol, + }); + + permissionGrantId = messagesQueryGrant.grant.id; + } catch(error:any) { + console.error('SyncEngineLevel: Error fetching MessagesQuery permission grant for delegate DID', error); + return []; + } + } if (syncDirection === 'pull') { + // filter for a specific protocol if one is provided + const filters = protocol ? [{ protocol }] : []; // When sync is a pull, get the event log from the remote DWN. - const messagesReadMessage = await this.agent.dwn.processRequest({ + const messagesQueryMessage = await this.agent.dwn.processRequest({ store : false, target : did, author : did, messageType : DwnInterface.MessagesQuery, - messageParams : { filters: [], cursor } + granteeDid : delegateDid, + messageParams : { filters, cursor, permissionGrantId } }); try { messagesReply = await this.agent.rpc.sendDwnRequest({ dwnUrl : dwnUrl, targetDid : did, - message : messagesReadMessage.message + message : messagesQueryMessage.message, }) as MessagesQueryReply; } catch { // If a particular DWN service endpoint is unreachable, silently ignore. } } else if (syncDirection === 'push') { + const filters = protocol ? [{ protocol }] : []; // When sync is a push, get the event log from the local DWN. - const messagesReadDwnResponse = await this.agent.dwn.processRequest({ + const messagesQueryDwnResponse = await this.agent.dwn.processRequest({ author : did, target : did, messageType : DwnInterface.MessagesQuery, - messageParams : { filters: [], cursor } + granteeDid : delegateDid, + messageParams : { filters, cursor, permissionGrantId } }); - messagesReply = messagesReadDwnResponse.reply as MessagesQueryReply; + messagesReply = messagesQueryDwnResponse.reply as MessagesQueryReply; } const eventLog = messagesReply.entries ?? []; if (messagesReply.cursor) { - this.setCursor(did, dwnUrl, syncDirection, messagesReply.cursor); + this.setCursor(did, dwnUrl, syncDirection, messagesReply.cursor, protocol); } return eventLog; } - private async getDwnMessage({ author, messageCid }: { + private async getDwnMessage({ author, delegateDid, protocol, messageCid }: { author: string; + delegateDid?: string; + protocol?: string; messageCid: string; }): Promise<{ message: GenericMessage, data?: Blob } | undefined> { + let permissionGrantId: string | undefined; + if (delegateDid) { + try { + const messagesReadGrant = await this._cachedPermissionsApi.getPermission({ + connectedDid : author, + messageType : DwnInterface.MessagesRead, + delegateDid, + protocol, + }); + + permissionGrantId = messagesReadGrant.grant.id; + } catch(error:any) { + console.error('SyncEngineLevel: push - Error fetching MessagesRead permission grant for delegate DID', error); + return; + } + } + let { reply } = await this.agent.dwn.processRequest({ author : author, target : author, messageType : DwnInterface.MessagesRead, - messageParams : { - messageCid: messageCid - } + granteeDid : delegateDid, + messageParams : { messageCid, permissionGrantId } }); - // Absence of a messageEntry or message within messageEntry can happen because updating a // Record creates another RecordsWrite with the same recordId. Only the first and // most recent RecordsWrite messages are kept for a given recordId. Any RecordsWrite messages @@ -392,15 +510,15 @@ export class SyncEngineLevel implements SyncEngine { } private async getSyncPeerState({ syncDirection }: { - syncDirection: SyncDirection + syncDirection: SyncDirection; }): Promise { - // Get a list of the DIDs of all registered identities. - const registeredIdentities = await this._db.sublevel('registeredIdentities').keys().all(); // Array to accumulate the list of sync peers for each DID. const syncPeerState: SyncState[] = []; - for (let did of registeredIdentities) { + // iterate over all registered identities + for await (const [ did, options ] of this._db.sublevel('registeredIdentities').iterator()) { + const { protocols, delegateDid } = JSON.parse(options) as SyncIdentityOptions; // First, confirm the DID can be resolved and extract the DWN service endpoint URLs. const dwnEndpointUrls = await getDwnServiceEndpointUrls(did, this.agent.did); if (dwnEndpointUrls.length === 0) { @@ -412,16 +530,27 @@ export class SyncEngineLevel implements SyncEngine { // Get the cursor (or undefined) for each (DID, DWN service endpoint, sync direction) // combination and add it to the sync peer state array. for (let dwnUrl of dwnEndpointUrls) { - const cursor = await this.getCursor(did, dwnUrl, syncDirection); - syncPeerState.push({ did, dwnUrl, cursor}); + if (protocols.length === 0) { + const cursor = await this.getCursor(did, dwnUrl, syncDirection); + syncPeerState.push({ did, delegateDid, dwnUrl, cursor }); + } else { + for (const protocol of protocols) { + const cursor = await this.getCursor(did, dwnUrl, syncDirection, protocol); + syncPeerState.push({ did, delegateDid, dwnUrl, cursor, protocol }); + } + } } } return syncPeerState; } - private async getCursor(did: string, dwnUrl: string, direction: SyncDirection): Promise { - const cursorKey = `${did}~${dwnUrl}~${direction}`; + private async getCursor(did: string, dwnUrl: string, direction: SyncDirection, protocol?: string): Promise { + + // if a protocol is provided, we append it to the key + const cursorKey = protocol ? `${did}~${dwnUrl}~${direction}-${protocol}` : + `${did}~${dwnUrl}~${direction}`; + const cursorsStore = this.getCursorStore(); try { const cursorValue = await cursorsStore.get(cursorKey); @@ -436,8 +565,9 @@ export class SyncEngineLevel implements SyncEngine { } } - private async setCursor(did: string, dwnUrl: string, direction: SyncDirection, cursor: PaginationCursor) { - const cursorKey = `${did}~${dwnUrl}~${direction}`; + private async setCursor(did: string, dwnUrl: string, direction: SyncDirection, cursor: PaginationCursor, protocol?: string) { + const cursorKey = protocol ? `${did}~${dwnUrl}~${direction}-${protocol}` : + `${did}~${dwnUrl}~${direction}`; const cursorsStore = this.getCursorStore(); await cursorsStore.put(cursorKey, JSON.stringify(cursor)); } diff --git a/packages/agent/src/types/sync.ts b/packages/agent/src/types/sync.ts index a1903be0f..ee4dd3182 100644 --- a/packages/agent/src/types/sync.ts +++ b/packages/agent/src/types/sync.ts @@ -1,8 +1,13 @@ import type { Web5PlatformAgent } from './agent.js'; +export type SyncIdentityOptions = { + delegateDid?: string; + protocols: string[] +} export interface SyncEngine { agent: Web5PlatformAgent; - registerIdentity(params: { did: string }): Promise; + registerIdentity(params: { did: string, options: SyncIdentityOptions }): Promise; + sync(direction?: 'push' | 'pull'): Promise; startSync(params: { interval: string }): Promise; stopSync(): void; } \ No newline at end of file diff --git a/packages/agent/tests/cached-permissions.spec.ts b/packages/agent/tests/cached-permissions.spec.ts new file mode 100644 index 000000000..d65540acc --- /dev/null +++ b/packages/agent/tests/cached-permissions.spec.ts @@ -0,0 +1,240 @@ +import sinon from 'sinon'; +import { expect } from 'chai'; +import { AgentPermissionsApi } from '../src/permissions-api.js'; +import { PlatformAgentTestHarness } from '../src/test-harness.js'; +import { TestAgent } from './utils/test-agent.js'; +import { BearerDid } from '@web5/dids'; + +import { testDwnUrl } from './utils/test-config.js'; +import { DwnInterfaceName, DwnMethodName, Time } from '@tbd54566975/dwn-sdk-js'; +import { CachedPermissions, DwnInterface } from '../src/index.js'; +import { Convert } from '@web5/common'; + +let testDwnUrls: string[] = [testDwnUrl]; + +describe('CachedPermissions', () => { + let permissions: AgentPermissionsApi; + let testHarness: PlatformAgentTestHarness; + let aliceDid: BearerDid; + let bobDid: BearerDid; + + before(async () => { + testHarness = await PlatformAgentTestHarness.setup({ + agentClass : TestAgent, + agentStores : 'dwn' + }); + }); + + after(async () => { + sinon.restore(); + await testHarness.clearStorage(); + await testHarness.closeStorage(); + }); + + beforeEach(async () => { + sinon.restore(); + await testHarness.clearStorage(); + await testHarness.createAgentDid(); + + // Create an "alice" Identity to author the DWN messages. + const alice = await testHarness.createIdentity({ name: 'Alice', testDwnUrls }); + await testHarness.agent.identity.manage({ portableIdentity: await alice.export() }); + aliceDid = alice.did; + + const bob = await testHarness.createIdentity({ name: 'Bob', testDwnUrls }); + await testHarness.agent.identity.manage({ portableIdentity: await bob.export() }); + bobDid = bob.did; + + permissions = new AgentPermissionsApi({ agent: testHarness.agent }); + }); + + describe('cachedDefault', () => { + it('caches permissions by default if defaultCache is set to true', async () => { + // create a permission grant to fetch + const messagesQueryGrant = await permissions.createGrant({ + store : true, + author : aliceDid.uri, + grantedTo : bobDid.uri, + dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), + scope : { + interface : DwnInterfaceName.Messages, + method : DwnMethodName.Query, + } + }); + + // store the grant as owner from bob so that it can be fetched + const { encodedData, ...messagesQueryGrantMessage } = messagesQueryGrant.message; + const grantReply = await testHarness.agent.processDwnRequest({ + target : bobDid.uri, + author : bobDid.uri, + signAsOwner : true, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesQueryGrantMessage, + dataStream : new Blob([ Convert.base64Url(encodedData).toUint8Array() ]) + }); + expect(grantReply.reply.status.code).to.equal(202); + + const permissionGrantsApiSpy = sinon.spy(AgentPermissionsApi.prototype, 'fetchGrants'); + + // with defaultCache set to true + const cachedPermissions = new CachedPermissions({ agent: testHarness.agent, cachedDefault: true }); + + // fetch the grant + let fetchedMessagesQueryGrant = await cachedPermissions.getPermission({ + connectedDid : aliceDid.uri, + delegateDid : bobDid.uri, + messageType : DwnInterface.MessagesQuery, + }); + expect(fetchedMessagesQueryGrant).to.not.be.undefined; + expect(fetchedMessagesQueryGrant.message.recordId).to.equal(messagesQueryGrant.message.recordId); + + // confirm that the permission was fetched from the API + expect(permissionGrantsApiSpy.calledOnce).to.be.true; + permissionGrantsApiSpy.resetHistory(); + + // fetch the grant again to confirm that it was cached + fetchedMessagesQueryGrant = await cachedPermissions.getPermission({ + connectedDid : aliceDid.uri, + delegateDid : bobDid.uri, + messageType : DwnInterface.MessagesQuery, + }); + expect(fetchedMessagesQueryGrant).to.not.be.undefined; + expect(fetchedMessagesQueryGrant.message.recordId).to.equal(messagesQueryGrant.message.recordId); + + // confirm that the permission was not fetched again from the API + expect(permissionGrantsApiSpy.called).to.be.false; + + // confirm that the permissions is fetched from teh api if cache is set to false on a single call + fetchedMessagesQueryGrant = await cachedPermissions.getPermission({ + connectedDid : aliceDid.uri, + delegateDid : bobDid.uri, + messageType : DwnInterface.MessagesQuery, + cached : false, + }); + expect(fetchedMessagesQueryGrant).to.not.be.undefined; + expect(fetchedMessagesQueryGrant.message.recordId).to.equal(messagesQueryGrant.message.recordId); + + // confirm that the permission was fetched from the API + expect(permissionGrantsApiSpy.calledOnce).to.be.true; + }); + + it('does not cache permission by default defaultCache is set to false', async () => { + // create a permission grant to fetch + const messagesQueryGrant = await permissions.createGrant({ + store : true, + author : aliceDid.uri, + grantedTo : bobDid.uri, + dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), + scope : { + interface : DwnInterfaceName.Messages, + method : DwnMethodName.Query, + } + }); + + // store the grant as owner from bob so that it can be fetched + const { encodedData, ...messagesQueryGrantMessage } = messagesQueryGrant.message; + const grantReply = await testHarness.agent.processDwnRequest({ + target : bobDid.uri, + author : bobDid.uri, + signAsOwner : true, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesQueryGrantMessage, + dataStream : new Blob([ Convert.base64Url(encodedData).toUint8Array() ]) + }); + expect(grantReply.reply.status.code).to.equal(202); + + const permissionGrantsApiSpy = sinon.spy(AgentPermissionsApi.prototype, 'fetchGrants'); + + // with defaultCache set to false by default + const cachedPermissions = new CachedPermissions({ agent: testHarness.agent }); + + // fetch the grant + let fetchedMessagesQueryGrant = await cachedPermissions.getPermission({ + connectedDid : aliceDid.uri, + delegateDid : bobDid.uri, + messageType : DwnInterface.MessagesQuery, + }); + expect(fetchedMessagesQueryGrant).to.not.be.undefined; + expect(fetchedMessagesQueryGrant.message.recordId).to.equal(messagesQueryGrant.message.recordId); + + // confirm that the permission was fetched from the API + expect(permissionGrantsApiSpy.calledOnce).to.be.true; + permissionGrantsApiSpy.resetHistory(); + + // fetch the grant again to confirm that it was cached + fetchedMessagesQueryGrant = await cachedPermissions.getPermission({ + connectedDid : aliceDid.uri, + delegateDid : bobDid.uri, + messageType : DwnInterface.MessagesQuery, + }); + expect(fetchedMessagesQueryGrant).to.not.be.undefined; + expect(fetchedMessagesQueryGrant.message.recordId).to.equal(messagesQueryGrant.message.recordId); + + // confirm that the permission was fetched a second time from the API + expect(permissionGrantsApiSpy.called).to.be.true; + permissionGrantsApiSpy.resetHistory(); + + // confirm that the permissions is not fetched from the api if cache is set to true on a single call + fetchedMessagesQueryGrant = await cachedPermissions.getPermission({ + connectedDid : aliceDid.uri, + delegateDid : bobDid.uri, + messageType : DwnInterface.MessagesQuery, + cached : true, + }); + expect(fetchedMessagesQueryGrant).to.not.be.undefined; + expect(fetchedMessagesQueryGrant.message.recordId).to.equal(messagesQueryGrant.message.recordId); + + // confirm that the permission was fetched from the API + expect(permissionGrantsApiSpy.calledOnce).to.be.false; + }); + }); + + describe('getPermission', () => { + it('throws an error if no permissions are found', async () => { + const cachedPermissions = new CachedPermissions({ agent: testHarness.agent }); + + try { + await cachedPermissions.getPermission({ + connectedDid : aliceDid.uri, + delegateDid : bobDid.uri, + messageType : DwnInterface.MessagesQuery, + }); + expect.fail('Expected an error to be thrown'); + } catch(error: any) { + expect(error.message).to.equal('CachedPermissions: No permissions found for MessagesQuery: undefined'); + } + + // create a permission grant to fetch + const messagesQueryGrant = await permissions.createGrant({ + store : true, + author : aliceDid.uri, + grantedTo : bobDid.uri, + dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), + scope : { + interface : DwnInterfaceName.Messages, + method : DwnMethodName.Query, + } + }); + + // store the grant as owner from bob so that it can be fetched + const { encodedData, ...messagesQueryGrantMessage } = messagesQueryGrant.message; + const grantReply = await testHarness.agent.processDwnRequest({ + target : bobDid.uri, + author : bobDid.uri, + signAsOwner : true, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesQueryGrantMessage, + dataStream : new Blob([ Convert.base64Url(encodedData).toUint8Array() ]) + }); + expect(grantReply.reply.status.code).to.equal(202); + + // fetch the grant + const fetchedMessagesQueryGrant = await cachedPermissions.getPermission({ + connectedDid : aliceDid.uri, + delegateDid : bobDid.uri, + messageType : DwnInterface.MessagesQuery, + }); + expect(fetchedMessagesQueryGrant.message.recordId).to.equal(messagesQueryGrant.message.recordId); + }); + }); +}); \ No newline at end of file diff --git a/packages/agent/tests/dwn-api.spec.ts b/packages/agent/tests/dwn-api.spec.ts index 9ab4aca3f..7983204ce 100644 --- a/packages/agent/tests/dwn-api.spec.ts +++ b/packages/agent/tests/dwn-api.spec.ts @@ -11,13 +11,13 @@ import { expect } from 'chai'; import type { PortableIdentity } from '../src/types/identity.js'; -import { DwnInterface } from '../src/types/dwn.js'; +import { DwnInterface, DwnPermissionScope } from '../src/types/dwn.js'; import { BearerIdentity } from '../src/bearer-identity.js'; import emailProtocolDefinition from './fixtures/protocol-definitions/email.json' assert { type: 'json' }; import { PlatformAgentTestHarness } from '../src/test-harness.js'; import { TestAgent } from './utils/test-agent.js'; import { testDwnUrl } from './utils/test-config.js'; -import { AgentDwnApi, isDwnMessage } from '../src/dwn-api.js'; +import { AgentDwnApi, isDwnMessage, isMessagesPermissionScope, isRecordPermissionScope } from '../src/dwn-api.js'; // NOTE: @noble/secp256k1 requires globalThis.crypto polyfill for node.js <=18: https://github.com/paulmillr/noble-secp256k1/blob/main/README.md#usage // Remove when we move off of node.js v18 to v20, earliest possible time would be Oct 2023: https://github.com/nodejs/release#release-schedule @@ -1935,4 +1935,48 @@ describe('isDwnMessage', () => { expect(isDwnMessage(DwnInterface.RecordsQuery, recordsWriteMessage)).to.be.false; expect(isDwnMessage(DwnInterface.RecordsWrite, recordsQueryMessage)).to.be.false; }); +}); + +describe('isRecordPermissionScope', () => { + it('asserts the type of RecordPermissionScope', async () => { + // messages read scope to test negative case + const messagesReadScope:DwnPermissionScope = { + interface : DwnInterfaceName.Messages, + method : DwnMethodName.Read + }; + + expect(isRecordPermissionScope(messagesReadScope)).to.be.false; + + // records read scope to test positive case + const recordsReadScope:DwnPermissionScope = { + interface : DwnInterfaceName.Records, + method : DwnMethodName.Read, + protocol : 'https://schemas.xyz/example' + }; + + expect(isRecordPermissionScope(recordsReadScope)).to.be.true; + }); +}); + +describe('isMessagesPermissionScope', () => { + it('asserts the type of RecordPermissionScope', async () => { + + // records read scope to test negative case + const recordsReadScope:DwnPermissionScope = { + interface : DwnInterfaceName.Records, + method : DwnMethodName.Read, + protocol : 'https://schemas.xyz/example' + }; + + expect(isMessagesPermissionScope(recordsReadScope)).to.be.false; + + // messages read scope to test positive case + const messagesReadScope:DwnPermissionScope = { + interface : DwnInterfaceName.Messages, + method : DwnMethodName.Read + }; + + expect(isMessagesPermissionScope(messagesReadScope)).to.be.true; + + }); }); \ No newline at end of file diff --git a/packages/agent/tests/store-data.spec.ts b/packages/agent/tests/store-data.spec.ts index 97d706ce7..8340a541b 100644 --- a/packages/agent/tests/store-data.spec.ts +++ b/packages/agent/tests/store-data.spec.ts @@ -605,7 +605,7 @@ describe('AgentDataStore', () => { expect.fail('Expected an error to be thrown'); } catch (error: any) { - expect(error.message).to.include(`Failed to write data to store for: test-1`); + expect(error.message).to.include(`Failed to write data to store for test-1`); } finally { dwnApiStub.restore(); } diff --git a/packages/agent/tests/sync-engine-level.spec.ts b/packages/agent/tests/sync-engine-level.spec.ts index aa097d1f7..7365148a6 100644 --- a/packages/agent/tests/sync-engine-level.spec.ts +++ b/packages/agent/tests/sync-engine-level.spec.ts @@ -1,7 +1,7 @@ import sinon from 'sinon'; import { expect } from 'chai'; import { CryptoUtils } from '@web5/crypto'; -import { DwnConstant, ProtocolDefinition } from '@tbd54566975/dwn-sdk-js'; +import { DwnConstant, DwnInterfaceName, DwnMethodName, Jws, Message, ProtocolDefinition, Time } from '@tbd54566975/dwn-sdk-js'; import type { BearerIdentity } from '../src/bearer-identity.js'; @@ -11,6 +11,7 @@ import { DwnInterface } from '../src/types/dwn.js'; import { testDwnUrl } from './utils/test-config.js'; import { SyncEngineLevel } from '../src/sync-engine-level.js'; import { PlatformAgentTestHarness } from '../src/test-harness.js'; +import { Convert } from '@web5/common'; let testDwnUrls: string[] = [testDwnUrl]; @@ -25,6 +26,7 @@ describe('SyncEngineLevel', () => { }); after(async () => { + sinon.restore(); await testHarness.closeStorage(); }); @@ -48,6 +50,85 @@ describe('SyncEngineLevel', () => { }); }); + describe('generateSyncMessageParamsKey & parseSyncMessageParamsKey', () => { + it('parses key into sync params', () => { + const did = 'did:example:alice'; + const delegateDid = 'did:example:bob'; + const dwnUrl = 'https://dwn.example.com'; + const protocol = 'https://protocol.example.com'; + const watermark = '1234567890'; + const messageCid = 'abc123'; + + const key = SyncEngineLevel['generateSyncMessageParamsKey']({ + did, + delegateDid, + dwnUrl, + protocol, + watermark, + messageCid + }); + + const syncParams = SyncEngineLevel['parseSyncMessageParamsKey'](key); + expect(syncParams.did).to.equal(did); + expect(syncParams.delegateDid).to.equal(delegateDid); + expect(syncParams.dwnUrl).to.equal(dwnUrl); + expect(syncParams.protocol).to.equal(protocol); + expect(syncParams.watermark).to.equal(watermark); + expect(syncParams.messageCid).to.equal(messageCid); + }); + + it('returns undefined protocol if not present', () => { + const did = 'did:example:alice'; + const delegateDid = 'did:example:bob'; + const dwnUrl = 'https://dwn.example.com'; + const watermark = '1234567890'; + const messageCid = 'abc123'; + + const key = SyncEngineLevel['generateSyncMessageParamsKey']({ + did, + delegateDid, + dwnUrl, + watermark, + messageCid + }); + console.log('key', key); + + const syncParams = SyncEngineLevel['parseSyncMessageParamsKey'](key); + expect(syncParams.protocol).to.be.undefined; + + expect(syncParams.did).to.equal(did); + expect(syncParams.delegateDid).to.equal(delegateDid); + expect(syncParams.dwnUrl).to.equal(dwnUrl); + expect(syncParams.watermark).to.equal(watermark); + expect(syncParams.messageCid).to.equal(messageCid); + }); + + it('returns undefined delegateDid if not present', () => { + const did = 'did:example:alice'; + const dwnUrl = 'https://dwn.example.com'; + const protocol = 'https://protocol.example.com'; + const watermark = '1234567890'; + const messageCid = 'abc123'; + + const key = SyncEngineLevel['generateSyncMessageParamsKey']({ + did, + dwnUrl, + protocol, + watermark, + messageCid + }); + + const syncParams = SyncEngineLevel['parseSyncMessageParamsKey'](key); + expect(syncParams.delegateDid).to.be.undefined; + + expect(syncParams.did).to.equal(did); + expect(syncParams.dwnUrl).to.equal(dwnUrl); + expect(syncParams.protocol).to.equal(protocol); + expect(syncParams.watermark).to.equal(watermark); + expect(syncParams.messageCid).to.equal(messageCid); + }); + }); + describe('with Web5 Platform Agent', () => { let alice: BearerIdentity; let randomSchema: string; @@ -70,6 +151,7 @@ describe('SyncEngineLevel', () => { sinon.restore(); + syncEngine.stopSync(); await syncEngine.clear(); await testHarness.syncStore.clear(); await testHarness.dwnDataStore.clear(); @@ -271,12 +353,14 @@ describe('SyncEngineLevel', () => { // Register Alice's DID to be synchronized. await testHarness.agent.sync.registerIdentity({ - did: alice.did.uri + did : alice.did.uri, + options : { + protocols: [] + } }); // Execute Sync to pull all records from Alice's remote DWN to Alice's local DWN. - await syncEngine.push(); - await syncEngine.pull(); + await syncEngine.sync(); // query local to see all protocols localProtocolsQueryResponse = await testHarness.agent.dwn.processRequest({ @@ -339,7 +423,721 @@ describe('SyncEngineLevel', () => { expect(remoteRecordsFromQuery).to.have.members([...localRecords, ...remoteRecords]); }).slow(1000); // Yellow at 500ms, Red at 1000ms. + describe('sync()', () => { + it('syncs only specified direction, or if non specified syncs both directions', async () => { + // spy on push and pull and stub their response + const pushSpy = sinon.stub(syncEngine as any, 'push').resolves(); + const pullSpy = sinon.stub(syncEngine as any, 'pull').resolves(); + + // Register Alice's DID to be synchronized. + await testHarness.agent.sync.registerIdentity({ + did : alice.did.uri, + options : { + protocols: [] + } + }); + + // Execute Sync to push and pull all records from Alice's remote DWN to Alice's local DWN. + await syncEngine.sync(); + + // Verify push and pull were called once + expect(pushSpy.calledOnce).to.be.true; + expect(pullSpy.calledOnce).to.be.true; + + + // reset counts + pushSpy.reset(); + pullSpy.reset(); + + // Execute only push sync + await syncEngine.sync('push'); + + // Verify push was called once and pull was not called + expect(pushSpy.calledOnce).to.be.true; + expect(pullSpy.notCalled).to.be.true; + + // reset counts + pushSpy.reset(); + pullSpy.reset(); + + // Execute only pull sync + await syncEngine.sync('pull'); + + // Verify pull was called once and push was not called + expect(pushSpy.notCalled).to.be.true; + expect(pullSpy.calledOnce).to.be.true; + }); + + it('throws if sync is attempted while an interval sync is running', async () => { + // Register Alice's DID to be synchronized. + await testHarness.agent.sync.registerIdentity({ + did : alice.did.uri, + options : { + protocols: [] + } + }); + + // start the sync engine with an interval of 10 seconds + syncEngine.startSync({ interval: '10s' }); + + try { + // Execute Sync to push and pull all records from Alice's remote DWN to Alice's local DWN. + await syncEngine.sync(); + + expect.fail('Expected an error to be thrown'); + } catch (error: any) { + // Execute Sync to push and pull all records from Alice's remote DWN to Alice's local DWN. + expect(error.message).to.equal('SyncEngineLevel: Cannot call sync while a sync interval is active. Call `stopSync()` first.'); + } + }); + }); + + describe('registerIdentity()', () => { + it('syncs only specified protocols', async () => { + // create new identity to not conflict the previous tests's remote records + const aliceSync = await testHarness.createIdentity({ name: 'Alice', testDwnUrls }); + + // create 3 local protocols + const protocolFoo: ProtocolDefinition = { + published : true, + protocol : 'https://protocol.xyz/foo', + types : { + foo: { + schema : 'https://schemas.xyz/foo', + dataFormats : ['text/plain', 'application/json'] + } + }, + structure: { + foo: {} + } + }; + + const protocolBar: ProtocolDefinition = { + published : true, + protocol : 'https://protocol.xyz/bar', + types : { + bar: { + schema : 'https://schemas.xyz/bar', + dataFormats : ['text/plain', 'application/json'] + } + }, + structure: { + bar: {} + } + }; + + const protocolBaz: ProtocolDefinition = { + published : true, + protocol : 'https://protocol.xyz/baz', + types : { + baz: { + schema : 'https://schemas.xyz/baz', + dataFormats : ['text/plain', 'application/json'] + } + }, + structure: { + baz: {} + } + }; + + const protocolsFoo = await testHarness.agent.processDwnRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.ProtocolsConfigure, + messageParams : { + definition: protocolFoo + } + }); + expect(protocolsFoo.reply.status.code).to.equal(202); + + const protocolsBar = await testHarness.agent.processDwnRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.ProtocolsConfigure, + messageParams : { + definition: protocolBar + } + }); + expect(protocolsBar.reply.status.code).to.equal(202); + + const protocolsBaz = await testHarness.agent.processDwnRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.ProtocolsConfigure, + messageParams : { + definition: protocolBaz + } + }); + expect(protocolsBaz.reply.status.code).to.equal(202); + + // write a record for each protocol + const recordFoo = await testHarness.agent.processDwnRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + dataFormat : 'text/plain', + protocol : protocolFoo.protocol, + protocolPath : 'foo', + schema : protocolFoo.types.foo.schema + }, + dataStream: new Blob(['Hello, foo!']) + }); + expect(recordFoo.reply.status.code).to.equal(202); + + const recordBar = await testHarness.agent.processDwnRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + dataFormat : 'text/plain', + protocol : protocolBar.protocol, + protocolPath : 'bar', + schema : protocolBar.types.bar.schema + }, + dataStream: new Blob(['Hello, bar!']) + }); + expect(recordBar.reply.status.code).to.equal(202); + + const recordBaz = await testHarness.agent.processDwnRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + dataFormat : 'text/plain', + protocol : protocolBaz.protocol, + protocolPath : 'baz', + schema : protocolBaz.types.baz.schema + }, + dataStream: new Blob(['Hello, baz!']) + }); + expect(recordBaz.reply.status.code).to.equal(202); + + // Register Alice's DID to be synchronized with only foo and bar protocols + await testHarness.agent.sync.registerIdentity({ + did : aliceSync.did.uri, + options : { + protocols: [ 'https://protocol.xyz/foo', 'https://protocol.xyz/bar' ] + } + }); + + // Execute Sync to push sync, only foo protocol should be synced + await syncEngine.sync('push'); + + // query remote to see foo protocol + const remoteProtocolsQueryResponse = await testHarness.agent.dwn.sendRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.ProtocolsQuery, + messageParams : {} + }); + const remoteProtocolsQueryReply = remoteProtocolsQueryResponse.reply; + expect(remoteProtocolsQueryReply.status.code).to.equal(200); + expect(remoteProtocolsQueryReply.entries?.length).to.equal(2); + expect(remoteProtocolsQueryReply.entries).to.have.deep.equal([ protocolsFoo.message, protocolsBar.message ]); + + // query remote to see foo record + let remoteFooRecordsResponse = await testHarness.agent.dwn.sendRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + filter: { + protocol: protocolFoo.protocol, + } + } + }); + let remoteFooRecordsReply = remoteFooRecordsResponse.reply; + expect(remoteFooRecordsReply.status.code).to.equal(200); + expect(remoteFooRecordsReply.entries).to.have.length(1); + let remoteFooRecordIds = remoteFooRecordsReply.entries?.map(entry => entry.recordId); + expect(remoteFooRecordIds).to.have.members([ recordFoo.message!.recordId ]); + + // query remote to see bar record + let remoteBarRecordsResponse = await testHarness.agent.dwn.sendRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + filter: { + protocol: protocolBar.protocol, + } + } + }); + let remoteBarRecordsReply = remoteBarRecordsResponse.reply; + expect(remoteBarRecordsReply.status.code).to.equal(200); + expect(remoteBarRecordsReply.entries).to.have.length(1); + let remoteBarRecordIds = remoteBarRecordsReply.entries?.map(entry => entry.recordId); + expect(remoteBarRecordIds).to.have.members([ recordBar.message!.recordId ]); + + // query remote to see baz record, none should be returned + let remoteBazRecordsResponse = await testHarness.agent.dwn.sendRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + filter: { + protocol: protocolBaz.protocol, + } + } + }); + let remoteBazRecordsReply = remoteBazRecordsResponse.reply; + expect(remoteBazRecordsReply.status.code).to.equal(200); + expect(remoteBazRecordsReply.entries).to.have.length(0); + + + // now write a foo record remotely, and a bar record locally + // initiate a sync to both push and pull the records respectively + + // write a record to the remote for the foo protocol + const recordFoo2 = await testHarness.agent.sendDwnRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + dataFormat : 'text/plain', + protocol : protocolFoo.protocol, + protocolPath : 'foo', + schema : protocolFoo.types.foo.schema + }, + dataStream: new Blob(['Hello, foo 2!']) + }); + expect(recordFoo2.reply.status.code).to.equal(202); + + // write a local record to the bar protocol + const recordBar2 = await testHarness.agent.processDwnRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + dataFormat : 'text/plain', + protocol : protocolBar.protocol, + protocolPath : 'bar', + schema : protocolBar.types.bar.schema + }, + dataStream: new Blob(['Hello, bar 2!']) + }); + expect(recordBar2.reply.status.code).to.equal(202); + + // confirm that the foo record is not yet in the local DWN + let localFooRecordsResponse = await testHarness.agent.dwn.processRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + filter: { + protocol: protocolFoo.protocol, + } + } + }); + let localFooRecordsReply = localFooRecordsResponse.reply; + expect(localFooRecordsReply.status.code).to.equal(200); + expect(localFooRecordsReply.entries).to.have.length(1); + let localFooRecordIds = localFooRecordsReply.entries?.map(entry => entry.recordId); + expect(localFooRecordIds).to.not.include(recordFoo2.message!.recordId); + + + // confirm that the bar record is not yet in the remote DWN + remoteBarRecordsResponse = await testHarness.agent.dwn.sendRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + filter: { + protocol: protocolBar.protocol, + } + } + }); + remoteBarRecordsReply = remoteBarRecordsResponse.reply; + expect(remoteBarRecordsReply.status.code).to.equal(200); + expect(remoteBarRecordsReply.entries).to.have.length(1); + remoteBarRecordIds = remoteBarRecordsReply.entries?.map(entry => entry.recordId); + expect(remoteBarRecordIds).to.not.include(recordBar2.message!.recordId); + + // preform a pull and push sync + await syncEngine.sync(); + + // query local to see foo records with the new record + localFooRecordsResponse = await testHarness.agent.dwn.processRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + filter: { + protocol: protocolFoo.protocol, + } + } + }); + localFooRecordsReply = localFooRecordsResponse.reply; + expect(localFooRecordsReply.status.code).to.equal(200); + expect(localFooRecordsReply.entries).to.have.length(2); + localFooRecordIds = localFooRecordsReply.entries?.map(entry => entry.recordId); + expect(localFooRecordIds).to.have.members([ recordFoo.message!.recordId, recordFoo2.message!.recordId ]); + + // query remote to see bar records with the new record + remoteBarRecordsResponse = await testHarness.agent.dwn.sendRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + filter: { + protocol: protocolBar.protocol, + } + } + }); + remoteBarRecordsReply = remoteBarRecordsResponse.reply; + expect(remoteBarRecordsReply.status.code).to.equal(200); + expect(remoteBarRecordsReply.entries).to.have.length(2); + remoteBarRecordIds = remoteBarRecordsReply.entries?.map(entry => entry.recordId); + expect(remoteBarRecordIds).to.have.members([ recordBar.message!.recordId, recordBar2.message!.recordId ]); + + // confirm that still no baz records exist remotely + remoteBazRecordsResponse = await testHarness.agent.dwn.sendRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + filter: { + protocol: protocolBaz.protocol, + } + } + }); + remoteBazRecordsReply = remoteBazRecordsResponse.reply; + expect(remoteBazRecordsReply.status.code).to.equal(200); + expect(remoteBazRecordsReply.entries).to.have.length(0); + }); + + it('syncs only specified protocols and delegates', async () => { + const alice = await testHarness.createIdentity({ name: 'Alice', testDwnUrls }); + + const aliceDeviceXHarness = await PlatformAgentTestHarness.setup({ + agentClass : TestAgent, + agentStores : 'memory', + testDataLocation : '__TESTDATA__/alice-device', + }); + await aliceDeviceXHarness.clearStorage(); + await aliceDeviceXHarness.createAgentDid(); + + // create a connected DID + const aliceDeviceX = await aliceDeviceXHarness.agent.identity.create({ + store : true, + didMethod : 'jwk', + metadata : { name: 'Alice Device X', connectedDid: alice.did.uri } + }); + + // Alice create 2 protocols on alice's remote DWN + const protocolFoo: ProtocolDefinition = { + published : true, + protocol : 'https://protocol.xyz/foo', + types : { + foo: { + schema : 'https://schemas.xyz/foo', + dataFormats : ['text/plain', 'application/json'] + } + }, + structure: { + foo: {} + } + }; + + const protocolBar: ProtocolDefinition = { + published : true, + protocol : 'https://protocol.xyz/bar', + types : { + bar: { + schema : 'https://schemas.xyz/bar', + dataFormats : ['text/plain', 'application/json'] + } + }, + structure: { + bar: {} + } + }; + + // configure the protocols + const protocolsFoo = await testHarness.agent.sendDwnRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.ProtocolsConfigure, + messageParams : { + definition: protocolFoo + } + }); + expect(protocolsFoo.reply.status.code).to.equal(202); + + const protocolsBar = await testHarness.agent.sendDwnRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.ProtocolsConfigure, + messageParams : { + definition: protocolBar + } + }); + expect(protocolsBar.reply.status.code).to.equal(202); + + // create grants for foo protocol, granted to aliceDeviceX + const messagesReadGrant = await testHarness.agent.permissions.createGrant({ + store : true, + author : alice.did.uri, + grantedTo : aliceDeviceX.did.uri, + dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), + scope : { protocol: protocolFoo.protocol, interface: DwnInterfaceName.Messages, method: DwnMethodName.Read } + }); + + const messagesQueryGrant = await testHarness.agent.permissions.createGrant({ + store : true, + author : alice.did.uri, + grantedTo : aliceDeviceX.did.uri, + dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), + scope : { protocol: protocolFoo.protocol, interface: DwnInterfaceName.Messages, method: DwnMethodName.Query } + }); + + const recordsQueryGrant = await testHarness.agent.permissions.createGrant({ + store : true, + author : alice.did.uri, + grantedTo : aliceDeviceX.did.uri, + dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), + delegated : true, + scope : { protocol: protocolFoo.protocol, interface: DwnInterfaceName.Records, method: DwnMethodName.Query } + }); + + const { encodedData: readGrantData, ... messagesReadGrantMessage } = messagesReadGrant.message; + const processMessagesReadGrantAsOwner = await aliceDeviceXHarness.agent.processDwnRequest({ + author : aliceDeviceX.did.uri, + target : aliceDeviceX.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesReadGrantMessage, + dataStream : new Blob([ Convert.base64Url(readGrantData).toUint8Array() ]), + signAsOwner : true + }); + expect(processMessagesReadGrantAsOwner.reply.status.code).to.equal(202); + + const processMessagesReadGrant = await aliceDeviceXHarness.agent.processDwnRequest({ + author : aliceDeviceX.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesReadGrantMessage, + dataStream : new Blob([ Convert.base64Url(readGrantData).toUint8Array() ]) + }); + expect(processMessagesReadGrant.reply.status.code).to.equal(202); + + const { encodedData: queryGrantData, ... messagesQueryGrantMessage } = messagesQueryGrant.message; + const processMessagesQueryGrantAsOwner = await aliceDeviceXHarness.agent.processDwnRequest({ + author : aliceDeviceX.did.uri, + target : aliceDeviceX.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesQueryGrantMessage, + dataStream : new Blob([ Convert.base64Url(queryGrantData).toUint8Array() ]), + signAsOwner : true + }); + expect(processMessagesQueryGrantAsOwner.reply.status.code).to.equal(202); + + const processMessagesQueryGrant = await aliceDeviceXHarness.agent.processDwnRequest({ + author : aliceDeviceX.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesQueryGrantMessage, + dataStream : new Blob([ Convert.base64Url(queryGrantData).toUint8Array() ]), + }); + expect(processMessagesQueryGrant.reply.status.code).to.equal(202); + + // send the grants to the remote DWN + const remoteMessagesQueryGrant = await testHarness.agent.sendDwnRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesQueryGrantMessage, + dataStream : new Blob([ Convert.base64Url(queryGrantData).toUint8Array() ]), + }); + expect(remoteMessagesQueryGrant.reply.status.code).to.equal(202); + + const remoteMessagesReadGrant = await testHarness.agent.sendDwnRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesReadGrantMessage, + dataStream : new Blob([ Convert.base64Url(readGrantData).toUint8Array() ]), + }); + expect(remoteMessagesReadGrant.reply.status.code).to.equal(202); + + const { encodedData: recordsQueryGrantData, ... recordsQueryGrantMessage } = recordsQueryGrant.message; + const processRecordsQueryGrant = await testHarness.agent.sendDwnRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : recordsQueryGrantMessage, + dataStream : new Blob([ Convert.base64Url(recordsQueryGrantData).toUint8Array() ]), + }); + expect(processRecordsQueryGrant.reply.status.code).to.equal(202); + + + // create a record for each protocol + const recordFoo = await testHarness.agent.sendDwnRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + dataFormat : 'text/plain', + protocol : protocolFoo.protocol, + protocolPath : 'foo', + schema : protocolFoo.types.foo.schema + }, + dataStream: new Blob(['Hello, foo!']) + }); + expect(recordFoo.reply.status.code).to.equal(202); + + const recordBar = await testHarness.agent.sendDwnRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + dataFormat : 'text/plain', + protocol : protocolBar.protocol, + protocolPath : 'bar', + schema : protocolBar.types.bar.schema + }, + dataStream: new Blob(['Hello, bar!']) + }); + expect(recordBar.reply.status.code).to.equal(202); + + // Register Alice's DID to be synchronized with only foo protocol + await aliceDeviceXHarness.agent.sync.registerIdentity({ + did : alice.did.uri, + options : { + protocols : [ protocolFoo.protocol ], + delegateDid : aliceDeviceX.did.uri + } + }); + + // Execute Sync, only foo protocol should be synced + await aliceDeviceXHarness.agent.sync.sync(); + + // query aliceDeviceX to see foo records + const localFooRecords = await aliceDeviceXHarness.agent.dwn.processRequest({ + author : alice.did.uri, + target : alice.did.uri, + granteeDid : aliceDeviceX.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + delegatedGrant : recordsQueryGrant.message, + filter : { + protocol: protocolFoo.protocol, + } + } + }); + const didAuthor = Jws.getSignerDid(localFooRecords.message!.authorization?.signature.signatures[0]!); + expect(didAuthor).to.equal(aliceDeviceX.did.uri); + expect(localFooRecords.reply.status.code).to.equal(200); + expect(localFooRecords.reply.entries).to.have.length(1); + expect(localFooRecords.reply.entries?.map(entry => entry.recordId)).to.have.deep.equal([ recordFoo.message?.recordId ]); + + // sanity check that bar records do not exist on aliceDeviceX + // since aliceDeviceX does not have a grant for the bar protocol, query the records using alice's signatures. + // confirm that the query was successful on alice's remote DWN and returns the message + const localBarRecordsQuery = await testHarness.agent.dwn.sendRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + filter: { + protocol: protocolBar.protocol, + } + } + }); + expect(localBarRecordsQuery.reply.status.code).to.equal(200); + expect(localBarRecordsQuery.reply.entries).to.have.length(1); + + // use the same message to query `aliceDeviceXHarness` DWN, should return zero results because they were not synced + const localBarRecords = await aliceDeviceXHarness.agent.dwn.processRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsQuery, + rawMessage : localBarRecordsQuery.message, + }); + expect(localBarRecords.reply.status.code).to.equal(200); + expect(localBarRecords.reply.entries).to.have.length(0); + }); + }); + describe('pull()', () => { + it('synchronizes records that have been updated', async () => { + // Write a test record to Alice's remote DWN. + let writeResponse1 = await testHarness.agent.dwn.sendRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + dataFormat : 'text/plain', + schema : randomSchema + }, + dataStream: new Blob(['Hello, world!']) + }); + + // Get the record ID of the test record. + const testRecordId = writeResponse1.message!.recordId; + + // const update the record + let updateResponse = await testHarness.agent.dwn.sendRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + recordId : testRecordId, + dataFormat : 'text/plain', + schema : randomSchema, + dateCreated : writeResponse1.message!.descriptor.dateCreated + }, + dataStream: new Blob(['Hello, world updated!']) + }); + expect(updateResponse.reply.status.code).to.equal(202); + expect(updateResponse.message!.recordId).to.equal(testRecordId); + + const updateMessageCid = updateResponse.messageCid; + + // Confirm the record does NOT exist on Alice's local DWN. + let queryResponse = await testHarness.agent.dwn.processRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + filter: { + dataFormat : 'text/plain', + schema : randomSchema + } + } + }); + let localDwnQueryReply = queryResponse.reply; + expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(localDwnQueryReply.entries).to.have.length(0); // Record doesn't exist on local DWN. + + // Register Alice's DID to be synchronized. + await testHarness.agent.sync.registerIdentity({ + did : alice.did.uri, + options : { + protocols: [] + } + }); + + // Execute Sync to pull all records from Alice's remote DWN to Alice's local DWN. + await syncEngine.sync('pull'); + + // Confirm the record now DOES exist on Alice's local DWN. + queryResponse = await testHarness.agent.dwn.processRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { filter: { recordId: testRecordId } } + }); + localDwnQueryReply = queryResponse.reply; + expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(localDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN. + + // remove `initialWrite` from the response to generate an accurate messageCid + const { initialWrite, ...rawMessage } = localDwnQueryReply.entries![0]; + const queriedMessageCid = await Message.getCid(rawMessage); + expect(queriedMessageCid).to.equal(updateMessageCid); + }); + it('silently ignores sendDwnRequest for a messageCid that does not exist on a remote DWN', async () => { // scenario: The messageCids returned from the remote eventLog contains a Cid that is not found in the remote DWN // this could happen when a record is updated, only the initial write and the most recent state are kept. @@ -410,11 +1208,14 @@ describe('SyncEngineLevel', () => { // Register Alice's DID to be synchronized. await testHarness.agent.sync.registerIdentity({ - did: alice.did.uri + did : alice.did.uri, + options : { + protocols: [] + } }); // Execute Sync to pull all records from Alice's remote DWNs - await syncEngine.pull(); + await syncEngine.sync('pull'); // Verify sendDwnRequest was called once for each record, including the invalid record // @@ -536,7 +1337,10 @@ describe('SyncEngineLevel', () => { // Register Alice's DID to be synchronized. await testHarness.agent.sync.registerIdentity({ - did: alice.did.uri + did : alice.did.uri, + options : { + protocols: [] + } }); // spy on sendDwnRequest to the remote DWN @@ -544,7 +1348,7 @@ describe('SyncEngineLevel', () => { const processMessageSpy = sinon.spy(testHarness.agent.dwn.node, 'processMessage'); // Execute Sync to push records to Alice's remote node - await syncEngine.pull(); + await syncEngine.sync('pull'); // Verify sendDwnRequest is called for all 4 messages expect(sendDwnRequestSpy.callCount).to.equal(4, 'sendDwnRequestSpy'); @@ -583,7 +1387,7 @@ describe('SyncEngineLevel', () => { const didResolveSpy = sinon.spy(testHarness.agent.did, 'resolve'); const sendDwnRequestSpy = sinon.spy(testHarness.agent.rpc, 'sendDwnRequest'); - await syncEngine.pull(); + await syncEngine.sync('pull'); // Verify DID resolution and DWN requests did not occur. expect(didResolveSpy.notCalled).to.be.true; @@ -593,8 +1397,138 @@ describe('SyncEngineLevel', () => { sendDwnRequestSpy.restore(); }); + it('logs an error if could not fetch MessagesQuery permission needed for a sync', async () => { + // create new identity to not conflict the previous tests's remote records + const aliceSync = await testHarness.createIdentity({ name: 'Alice', testDwnUrls }); + + const delegateDid = await testHarness.agent.identity.create({ + store : true, + didMethod : 'jwk', + metadata : { name: 'Alice Delegate', connectedDid: aliceSync.did.uri } + }); + + await testHarness.agent.sync.registerIdentity({ + did : aliceSync.did.uri, + options : { + delegateDid : delegateDid.did.uri, + protocols : [ 'https://protocol.xyz/foo' ] + } + }); + + // spy on console.error to check if the error message is logged + const consoleErrorSpy = sinon.stub(console, 'error').resolves(); + + await syncEngine.sync('pull'); + expect(consoleErrorSpy.called).to.be.true; + expect(consoleErrorSpy.args[0][0]).to.include('SyncEngineLevel: Error fetching MessagesQuery permission grant for delegate DID'); + }); + + it('logs an error if could not fetch MessagesRead permission needed for a sync', async () => { + // create new identity to not conflict the previous tests's remote records + const aliceSync = await testHarness.createIdentity({ name: 'Alice', testDwnUrls }); + + // create 3 local protocols + const protocolFoo: ProtocolDefinition = { + published : true, + protocol : 'https://protocol.xyz/foo', + types : { + foo: { + schema : 'https://schemas.xyz/foo', + dataFormats : ['text/plain', 'application/json'] + } + }, + structure: { + foo: {} + } + }; + + // install a protocol on the remote node for aliceSync + const protocolsFoo = await testHarness.agent.sendDwnRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.ProtocolsConfigure, + messageParams : { + definition: protocolFoo + } + }); + expect(protocolsFoo.reply.status.code).to.equal(202); + + + // create a record that will be read as a part of sync + const record1 = await testHarness.agent.sendDwnRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + protocol : 'https://protocol.xyz/foo', + protocolPath : 'foo', + schema : 'https://schemas.xyz/foo', + dataFormat : 'text/plain', + }, + dataStream: new Blob(['Hello, world!']) + }); + expect(record1.reply.status.code).to.equal(202); + + + const delegateDid = await testHarness.agent.identity.create({ + store : true, + didMethod : 'jwk', + metadata : { name: 'Alice Delegate', connectedDid: aliceSync.did.uri } + }); + + // write a MessagesQuery permission grant for the delegate DID + const messagesQueryGrant = await testHarness.agent.permissions.createGrant({ + store : true, + author : aliceSync.did.uri, + grantedTo : delegateDid.did.uri, + dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), + scope : { + interface : DwnInterfaceName.Messages, + method : DwnMethodName.Query, + protocol : 'https://protocol.xyz/foo' + } + }); + + const { encodedData: messagesQueryGrantData, ...messagesQueryGrantMessage } = messagesQueryGrant.message; + // send to the remote node + const sendGrant = await testHarness.agent.sendDwnRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesQueryGrantMessage, + dataStream : new Blob([ Convert.base64Url(messagesQueryGrantData).toUint8Array() ]), + }); + expect(sendGrant.reply.status.code).to.equal(202); + + // store it as the delegate DID so that it can be fetched during sync + const processGrant = await testHarness.agent.processDwnRequest({ + author : delegateDid.did.uri, + target : delegateDid.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesQueryGrantMessage, + dataStream : new Blob([ Convert.base64Url(messagesQueryGrantData).toUint8Array() ]), + signAsOwner : true + }); + expect(processGrant.reply.status.code).to.equal(202); + + await testHarness.agent.sync.registerIdentity({ + did : aliceSync.did.uri, + options : { + delegateDid : delegateDid.did.uri, + protocols : [ 'https://protocol.xyz/foo' ] + } + }); + + // spy on console.error to check if the error message is logged + const consoleErrorSpy = sinon.stub(console, 'error').resolves(); + + await syncEngine.sync('pull'); + expect(consoleErrorSpy.called).to.be.true; + expect(consoleErrorSpy.args[0][0]).to.include('SyncEngineLevel: pull - Error fetching MessagesRead permission grant for delegate DID'); + }); + it('synchronizes records for 1 identity from remote DWN to local DWN', async () => { - // Write a test record to Alice's remote DWN. + // Write a test record to Alice's remote DWN. let writeResponse = await testHarness.agent.dwn.sendRequest({ author : alice.did.uri, target : alice.did.uri, @@ -627,11 +1561,14 @@ describe('SyncEngineLevel', () => { // Register Alice's DID to be synchronized. await testHarness.agent.sync.registerIdentity({ - did: alice.did.uri + did : alice.did.uri, + options : { + protocols: [] + } }); // Execute Sync to pull all records from Alice's remote DWN to Alice's local DWN. - await syncEngine.pull(); + await syncEngine.sync('pull'); // Confirm the record now DOES exist on Alice's local DWN. queryResponse = await testHarness.agent.dwn.processRequest({ @@ -645,8 +1582,6 @@ describe('SyncEngineLevel', () => { expect(localDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN. - - // Add another record for a subsequent sync. let writeResponse2 = await testHarness.agent.dwn.sendRequest({ author : alice.did.uri, @@ -671,7 +1606,7 @@ describe('SyncEngineLevel', () => { expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. expect(localDwnQueryReply.entries).to.have.length(0); // New Record doesn't exist on local DWN. - await syncEngine.pull(); + await syncEngine.sync('pull'); // Confirm the new record DOES exist on Alice's local DWN. queryResponse = await testHarness.agent.dwn.processRequest({ @@ -691,7 +1626,10 @@ describe('SyncEngineLevel', () => { // register alice await testHarness.agent.sync.registerIdentity({ - did: alice.did.uri + did : alice.did.uri, + options : { + protocols: [] + } }); // create a remote record @@ -718,7 +1656,7 @@ describe('SyncEngineLevel', () => { expect(localReply.entries?.length).to.equal(0); // initiate sync - await syncEngine.pull(); + await syncEngine.sync('pull'); // query that the local record exists const { reply: localReply2 } = await testHarness.agent.dwn.processRequest({ @@ -782,16 +1720,22 @@ describe('SyncEngineLevel', () => { // Register Alice's DID to be synchronized. await testHarness.agent.sync.registerIdentity({ - did: alice.did.uri + did : alice.did.uri, + options : { + protocols: [] + } }); // Register Bob's DID to be synchronized. await testHarness.agent.sync.registerIdentity({ - did: bob.did.uri + did : bob.did.uri, + options : { + protocols: [] + } }); // Execute Sync to pull all records from Alice's and Bob's remove DWNs to their local DWNs. - await syncEngine.pull(); + await syncEngine.sync('pull'); // Confirm the Alice test record exist on Alice's local DWN. let queryResponse = await testHarness.agent.dwn.processRequest({ @@ -818,6 +1762,84 @@ describe('SyncEngineLevel', () => { }); describe('push()', () => { + it('synchronizes records that have been updated', async () => { + // Write a test record to Alice's local DWN. + let writeResponse1 = await testHarness.agent.dwn.processRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + dataFormat : 'text/plain', + schema : randomSchema + }, + dataStream: new Blob(['Hello, world!']) + }); + + // Get the record ID of the test record. + const testRecordId = writeResponse1.message!.recordId; + + // const update the record + let updateResponse = await testHarness.agent.dwn.processRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + recordId : testRecordId, + dataFormat : 'text/plain', + schema : randomSchema, + dateCreated : writeResponse1.message!.descriptor.dateCreated + }, + dataStream: new Blob(['Hello, world updated!']) + }); + expect(updateResponse.reply.status.code).to.equal(202); + expect(updateResponse.message!.recordId).to.equal(testRecordId); + + const updateMessageCid = updateResponse.messageCid; + + // Confirm the record does NOT exist on Alice's remote DWN. + let queryResponse = await testHarness.agent.dwn.sendRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { + filter: { + dataFormat : 'text/plain', + schema : randomSchema + } + } + }); + let remoteDwnQueryReply = queryResponse.reply; + expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(remoteDwnQueryReply.entries).to.have.length(0); // Record doesn't exist on local DWN. + + // Register Alice's DID to be synchronized. + await testHarness.agent.sync.registerIdentity({ + did : alice.did.uri, + options : { + protocols: [] + } + }); + + // Execute Sync to pull all records from Alice's remote DWN to Alice's local DWN. + await syncEngine.sync('push'); + + // Confirm the record now DOES exist on Alice's local DWN. + queryResponse = await testHarness.agent.dwn.sendRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsQuery, + messageParams : { filter: { recordId: testRecordId } } + }); + remoteDwnQueryReply = queryResponse.reply; + expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(remoteDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN. + + // remove `initialWrite` from the response to generate an accurate messageCid + const { initialWrite, ...rawMessage } = remoteDwnQueryReply.entries![0]; + const queriedMessageCid = await Message.getCid(rawMessage); + expect(queriedMessageCid).to.equal(updateMessageCid); + }); + it('silently ignores a messageCid from the eventLog that does not exist on the local DWN', async () => { // It's important to create a new DID here to avoid conflicts with the previous test on the remote DWN, // since we are not clearing the remote DWN's storage before each test. @@ -895,11 +1917,14 @@ describe('SyncEngineLevel', () => { // Register Alice's DID to be synchronized. await testHarness.agent.sync.registerIdentity({ - did: alice.did.uri + did : alice.did.uri, + options : { + protocols: [] + } }); // Execute Sync to pull all records from Alice's remote DWNs - await syncEngine.push(); + await syncEngine.sync('push'); // verify that sendDwnRequest was called once only for each valid record // and getDwnMessage was called for each record, including the invalid record @@ -931,7 +1956,10 @@ describe('SyncEngineLevel', () => { // Register Alice's DID to be synchronized. await testHarness.agent.sync.registerIdentity({ - did: alice.did.uri + did : alice.did.uri, + options : { + protocols: [] + } }); // scenario: The messageCids returned from the local eventLog contains a Cid that already exists in the remote DWN. @@ -1028,7 +2056,7 @@ describe('SyncEngineLevel', () => { const sendDwnRequestSpy = sinon.spy(testHarness.agent.rpc, 'sendDwnRequest'); // Execute Sync to push records to Alice's remote node - await syncEngine.push(); + await syncEngine.sync('push'); // Verify sendDwnRequest was called once for each record including the ones that already exist remotely expect(sendDwnRequestSpy.callCount).to.equal(4); @@ -1065,7 +2093,7 @@ describe('SyncEngineLevel', () => { const didResolveSpy = sinon.spy(testHarness.agent.did, 'resolve'); const processRequestSpy = sinon.spy(testHarness.agent.dwn, 'processRequest'); - await syncEngine.push(); + await syncEngine.sync('push'); // Verify DID resolution and DWN requests did not occur. expect(didResolveSpy.notCalled).to.be.true; @@ -1075,6 +2103,126 @@ describe('SyncEngineLevel', () => { processRequestSpy.restore(); }); + it('logs an error if could not fetch MessagesQuery permission needed for a sync', async () => { + // create new identity to not conflict the previous tests's remote records + const aliceSync = await testHarness.createIdentity({ name: 'Alice', testDwnUrls }); + + const delegateDid = await testHarness.agent.identity.create({ + store : true, + didMethod : 'jwk', + metadata : { name: 'Alice Delegate', connectedDid: aliceSync.did.uri } + }); + + await testHarness.agent.sync.registerIdentity({ + did : aliceSync.did.uri, + options : { + delegateDid : delegateDid.did.uri, + protocols : [ 'https://protocol.xyz/foo' ] + } + }); + + // spy on console.error to check if the error message is logged + const consoleErrorSpy = sinon.stub(console, 'error').resolves(); + + await syncEngine.sync('push'); + expect(consoleErrorSpy.called).to.be.true; + expect(consoleErrorSpy.args[0][0]).to.include('SyncEngineLevel: Error fetching MessagesQuery permission grant for delegate DID'); + }); + + it('logs an error if could not fetch MessagesRead permission needed for a sync', async () => { + // create new identity to not conflict the previous tests's remote records + const aliceSync = await testHarness.createIdentity({ name: 'Alice', testDwnUrls }); + + // create 3 local protocols + const protocolFoo: ProtocolDefinition = { + published : true, + protocol : 'https://protocol.xyz/foo', + types : { + foo: { + schema : 'https://schemas.xyz/foo', + dataFormats : ['text/plain', 'application/json'] + } + }, + structure: { + foo: {} + } + }; + + // install a protocol on the local node for aliceSync + const protocolsFoo = await testHarness.agent.processDwnRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.ProtocolsConfigure, + messageParams : { + definition: protocolFoo + } + }); + expect(protocolsFoo.reply.status.code).to.equal(202); + + + // create a record that will be read as a part of sync + const record1 = await testHarness.agent.processDwnRequest({ + author : aliceSync.did.uri, + target : aliceSync.did.uri, + messageType : DwnInterface.RecordsWrite, + messageParams : { + protocol : 'https://protocol.xyz/foo', + protocolPath : 'foo', + schema : 'https://schemas.xyz/foo', + dataFormat : 'text/plain', + }, + dataStream: new Blob(['Hello, world!']) + }); + expect(record1.reply.status.code).to.equal(202); + + + const delegateDid = await testHarness.agent.identity.create({ + store : true, + didMethod : 'jwk', + metadata : { name: 'Alice Delegate', connectedDid: aliceSync.did.uri } + }); + + // write a MessagesQuery permission grant for the delegate DID + const messagesQueryGrant = await testHarness.agent.permissions.createGrant({ + store : true, + author : aliceSync.did.uri, + grantedTo : delegateDid.did.uri, + dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), + scope : { + interface : DwnInterfaceName.Messages, + method : DwnMethodName.Query, + protocol : 'https://protocol.xyz/foo' + } + }); + + // store it as the delegate DID so that it can be fetched during sync + const { encodedData: messagesQueryGrantData, ...messagesQueryGrantMessage } = messagesQueryGrant.message; + const processGrant = await testHarness.agent.processDwnRequest({ + author : delegateDid.did.uri, + target : delegateDid.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesQueryGrantMessage, + dataStream : new Blob([ Convert.base64Url(messagesQueryGrantData).toUint8Array() ]), + signAsOwner : true + }); + expect(processGrant.reply.status.code).to.equal(202); + + await testHarness.agent.sync.registerIdentity({ + did : aliceSync.did.uri, + options : { + delegateDid : delegateDid.did.uri, + protocols : [ 'https://protocol.xyz/foo' ] + } + }); + + // spy on console.error to check if the error message is logged + const consoleErrorSpy = sinon.stub(console, 'error').resolves(); + + await syncEngine.sync('push'); + expect(consoleErrorSpy.called).to.be.true; + expect(consoleErrorSpy.args[0][0]).to.include('SyncEngineLevel: push - Error fetching MessagesRead permission grant for delegate DID'); + }); + it('synchronizes records for 1 identity from local DWN to remote DWN', async () => { // Write a record that we can use for this test. let writeResponse = await testHarness.agent.dwn.processRequest({ @@ -1103,11 +2251,14 @@ describe('SyncEngineLevel', () => { // Register Alice's DID to be synchronized. await testHarness.agent.sync.registerIdentity({ - did: alice.did.uri + did : alice.did.uri, + options : { + protocols: [] + } }); // Execute Sync to push all records from Alice's local DWN to Alice's remote DWN. - await syncEngine.push(); + await syncEngine.sync('push'); // Confirm the record now DOES exist on Alice's remote DWN. queryResponse = await testHarness.agent.dwn.sendRequest({ @@ -1144,7 +2295,7 @@ describe('SyncEngineLevel', () => { expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. expect(remoteDwnQueryReply.entries).to.have.length(0); // New Record doesn't exist on local DWN. - await syncEngine.push(); + await syncEngine.sync('push'); // Confirm the new record DOES exist on Alice's local DWN. queryResponse = await testHarness.agent.dwn.sendRequest({ @@ -1164,7 +2315,10 @@ describe('SyncEngineLevel', () => { //register alice await testHarness.agent.sync.registerIdentity({ - did: alice.did.uri + did : alice.did.uri, + options : { + protocols: [] + } }); // create a local record @@ -1190,7 +2344,7 @@ describe('SyncEngineLevel', () => { expect(remoteReply.entries?.length).to.equal(0); // initiate sync - await syncEngine.push(); + await syncEngine.sync('push'); // query for remote REcords const { reply: remoteReply2 } = await testHarness.agent.dwn.sendRequest({ @@ -1253,16 +2407,22 @@ describe('SyncEngineLevel', () => { // Register Alice's DID to be synchronized. await testHarness.agent.sync.registerIdentity({ - did: alice.did.uri + did : alice.did.uri, + options : { + protocols: [] + } }); // Register Bob's DID to be synchronized. await testHarness.agent.sync.registerIdentity({ - did: bob.did.uri + did : bob.did.uri, + options : { + protocols: [] + } }); // Execute Sync to push all records from Alice's and Bob's local DWNs to their remote DWNs. - await syncEngine.push(); + await syncEngine.sync('push'); // Confirm the Alice test record exist on Alice's remote DWN. let queryResponse = await testHarness.agent.dwn.sendRequest({ @@ -1289,60 +2449,56 @@ describe('SyncEngineLevel', () => { }); describe('startSync()', () => { - it('calls push/pull in each interval', async () => { + it('calls sync() in each interval', async () => { await testHarness.agent.sync.registerIdentity({ - did: alice.did.uri + did : alice.did.uri, + options : { + protocols: [] + } }); - const pushSpy = sinon.stub(SyncEngineLevel.prototype, 'push'); - pushSpy.resolves(); - - const pullSpy = sinon.stub(SyncEngineLevel.prototype, 'pull'); - pullSpy.resolves(); + const syncSpy = sinon.stub(SyncEngineLevel.prototype, 'sync'); + syncSpy.resolves(); const clock = sinon.useFakeTimers(); testHarness.agent.sync.startSync({ interval: '500ms' }); await clock.tickAsync(1_400); // just under 3 intervals - pushSpy.restore(); - pullSpy.restore(); + syncSpy.restore(); clock.restore(); - expect(pushSpy.callCount).to.equal(2, 'push'); - expect(pullSpy.callCount).to.equal(2, 'pull'); + expect(syncSpy.callCount).to.equal(2, 'push'); }); - it('does not call push/pull again until a push/pull finishes', async () => { + it('does not call sync() again until a sync round finishes', async () => { await testHarness.agent.sync.registerIdentity({ - did: alice.did.uri + did : alice.did.uri, + options : { + protocols: [] + } }); const clock = sinon.useFakeTimers(); - const pushSpy = sinon.stub(SyncEngineLevel.prototype, 'push'); - pushSpy.returns(new Promise((resolve) => { + const syncSpy = sinon.stub(SyncEngineLevel.prototype, 'sync'); + syncSpy.returns(new Promise((resolve) => { clock.setTimeout(() => { resolve(); }, 1_500); // more than the interval })); - const pullSpy = sinon.stub(SyncEngineLevel.prototype, 'pull'); - pullSpy.resolves(); - testHarness.agent.sync.startSync({ interval: '500ms' }); await clock.tickAsync(1_400); // less time than the push - expect(pushSpy.callCount).to.equal(1, 'push'); - expect(pullSpy.callCount).to.equal(0, 'pull'); // not called yet + expect(syncSpy.callCount).to.equal(1, 'sync'); - await clock.tickAsync(100); //remaining time for pull to be called + await clock.tickAsync(600); //remaining time for a 2nd sync - expect(pullSpy.callCount).to.equal(1, 'pull'); + expect(syncSpy.callCount).to.equal(2, 'sync'); - pushSpy.restore(); - pullSpy.restore(); + syncSpy.restore(); clock.restore(); }); }); diff --git a/packages/api/src/dwn-api.ts b/packages/api/src/dwn-api.ts index c8661fe99..7b007e93d 100644 --- a/packages/api/src/dwn-api.ts +++ b/packages/api/src/dwn-api.ts @@ -17,13 +17,14 @@ import { DwnResponse, DwnMessageParams, DwnResponseStatus, + CachedPermissions, ProcessDwnRequest, DwnPaginationCursor, DwnDataEncodedRecordsWriteMessage, AgentPermissionsApi } from '@web5/agent'; -import { isEmptyObject, TtlCache } from '@web5/common'; +import { isEmptyObject } from '@web5/common'; import { DwnInterface, getRecordAuthor } from '@web5/agent'; import { Record } from './record.js'; @@ -31,6 +32,8 @@ import { dataToBlob } from './utils.js'; import { Protocol } from './protocol.js'; import { PermissionGrant } from './permission-grant.js'; import { PermissionRequest } from './permission-request.js'; +import { DwnMessagesPermissionScope } from '@web5/agent'; +import { DwnRecordsPermissionScope } from '@web5/agent'; /** * Represents the request payload for fetching permission requests from a Decentralized Web Node (DWN). @@ -271,13 +274,14 @@ export class DwnApi { private permissionsApi: AgentPermissionsApi; /** cache for fetching a permission {@link PermissionGrant}, keyed by a specific MessageType and protocol */ - private cachedPermissions: TtlCache = new TtlCache({ ttl: 60 * 1000 }); + private cachedPermissionsApi: CachedPermissions; constructor(options: { agent: Web5Agent, connectedDid: string, delegateDid?: string }) { this.agent = options.agent; this.connectedDid = options.connectedDid; this.delegateDid = options.delegateDid; this.permissionsApi = new AgentPermissionsApi({ agent: this.agent }); + this.cachedPermissionsApi = new CachedPermissions({ agent: this.agent, cachedDefault: true }); } /** @@ -304,33 +308,16 @@ export class DwnApi { throw new Error('AgentDwnApi: Cannot find connected grants without a signer DID'); } - // Currently we only support finding grants based on protocols - // A different approach may be necessary when we introduce `protocolPath` and `contextId` specific impersonation - const cacheKey = [ this.connectedDid, messageParams.messageType, messageParams.protocol ].join('~'); - const cachedGrant = cached ? this.cachedPermissions.get(cacheKey) : undefined; - if (cachedGrant) { - return cachedGrant; - } - - const permissionGrants = await this.permissions.queryGrants({ checkRevoked: true, grantor: this.connectedDid }); - - const grantEntries = permissionGrants.map(grant => ({ message: grant.rawMessage, grant: grant.toJSON() })); - - // get the delegate grants that match the messageParams and are associated with the connectedDid as the grantor - const delegateGrant = await AgentPermissionsApi.matchGrantFromArray( - this.connectedDid, - this.delegateDid, - messageParams, - grantEntries, - true - ); - - if (!delegateGrant) { - throw new Error(`AgentDwnApi: No permissions found for ${messageParams.messageType}: ${messageParams.protocol}`); - } + const delegateGrant = await this.cachedPermissionsApi.getPermission({ + connectedDid : this.connectedDid, + delegateDid : this.delegateDid, + messageType : messageParams.messageType, + protocol : messageParams.protocol, + delegate : true, + cached, + }); const grant = await PermissionGrant.parse({ connectedDid: this.delegateDid, agent: this.agent, message: delegateGrant.message }); - this.cachedPermissions.set(cacheKey, grant); return grant; } }; @@ -813,25 +800,4 @@ export class DwnApi { }, }; } - - /** - * A static method to process connected grants for a delegate DID. - * - * This will store the grants as the DWN owner to be used later when impersonating the connected DID. - */ - static async processConnectedGrants({ grants, agent, delegateDid }: { - grants: DwnDataEncodedRecordsWriteMessage[], - agent: Web5Agent, - delegateDid: string, - }): Promise { - for (const grantMessage of grants) { - // use the delegateDid as the connectedDid of the grant as they do not yet support impersonation/delegation - const grant = await PermissionGrant.parse({ connectedDid: delegateDid, agent, message: grantMessage }); - // store the grant as the owner of the DWN, this will allow the delegateDid to use the grant when impersonating the connectedDid - const { status } = await grant.store(true); - if (status.code !== 202) { - throw new Error(`AgentDwnApi: Failed to process connected grant: ${status.detail}`); - } - } - } } \ No newline at end of file diff --git a/packages/api/src/web5.ts b/packages/api/src/web5.ts index e905332c5..ff9c6827a 100644 --- a/packages/api/src/web5.ts +++ b/packages/api/src/web5.ts @@ -6,17 +6,21 @@ import type { BearerIdentity, + DwnDataEncodedRecordsWriteMessage, + DwnMessagesPermissionScope, + DwnRecordsPermissionScope, HdIdentityVault, WalletConnectOptions, Web5Agent, } from '@web5/agent'; import { Web5UserAgent } from '@web5/user-agent'; -import { DwnRegistrar, WalletConnect } from '@web5/agent'; +import { DwnRegistrar, isMessagesPermissionScope, isRecordPermissionScope, WalletConnect } from '@web5/agent'; import { DidApi } from './did-api.js'; import { DwnApi } from './dwn-api.js'; import { VcApi } from './vc-api.js'; +import { PermissionGrant } from './permission-grant.js'; /** Override defaults configured during the technical preview phase. */ export type TechPreviewOptions = { @@ -227,6 +231,7 @@ export class Web5 { }: Web5ConnectOptions = {}): Promise { let delegateDid: string | undefined; if (agent === undefined) { + let registerSync = false; // A custom Web5Agent implementation was not specified, so use default managed user agent. const userAgent = await Web5UserAgent.create({ agentVault }); agent = userAgent; @@ -252,11 +257,21 @@ export class Web5 { // Attempt to retrieve the connected Identity if it exists. const connectedIdentity: BearerIdentity = await userAgent.identity.connectedIdentity(); let identity: BearerIdentity; + let connectedProtocols: string[] = []; if (connectedIdentity) { // if a connected identity is found, use it // TODO: In the future, implement a way to re-connect an already connected identity and apply additional grants/protocols identity = connectedIdentity; } else if (walletConnectOptions) { + if (sync === 'off') { + // Currently we require sync to be enabled when using WalletConnect + // This is to ensure a connected app is not in a disjointed state from any other clients/app using the connectedDid + throw new Error('Sync must not be disabled when using WalletConnect'); + } + + // Since we are connecting a new identity, we will want to register sync for the connectedDid + registerSync = true; + // No connected identity found and connectOptions are provided, attempt to import a delegated DID from an external wallet try { // TEMPORARY: Placeholder for WalletConnect integration @@ -277,7 +292,8 @@ export class Web5 { // Attempts to process the connected grants to be used by the delegateDID // If the process fails, we want to clean up the identity - await DwnApi.processConnectedGrants({ agent, delegateDid: delegateDid.uri, grants: delegateGrants }); + // the connected grants will return a de-duped array of protocol URIs that are used to register sync for those protocols + connectedProtocols = await this.processConnectedGrants({ agent, delegateDid: delegateDid.uri, grants: delegateGrants }); } catch (error:any) { // clean up the DID and Identity if import fails and throw // TODO: Implement the ability to purge all of our messages as a tenant @@ -292,6 +308,9 @@ export class Web5 { // If an existing identity is not found found, create a new one. const existingIdentityCount = identities.length; if (existingIdentityCount === 0) { + // since we are creating a new identity, we will want to register sync for the created Did + registerSync = true; + // Use the specified DWN endpoints or the latest TBD hosted DWN const serviceEndpointNodes = techPreview?.dwnEndpoints ?? didCreateOptions?.dwnEndpoints ?? ['https://dwn.tbddev.org/beta']; @@ -370,7 +389,23 @@ export class Web5 { // Enable sync, unless explicitly disabled. if (sync !== 'off') { // First, register the user identity for sync. - await userAgent.sync.registerIdentity({ did: connectedDid }); + // The connected protocols are used to register sync for only a subset of protocols from the connectedDid's DWN + + if (registerSync) { + await userAgent.sync.registerIdentity({ + did : connectedDid, + options : { + delegateDid, + protocols: connectedProtocols + } + }); + + if(walletConnectOptions !== undefined) { + // If we are using WalletConnect, we should do a one-shot sync to pull down any messages that are associated with the connectedDid + await userAgent.sync.sync('pull'); + } + + } // Enable sync using the specified interval or default. sync ??= '2m'; @@ -412,4 +447,35 @@ export class Web5 { console.error(`Failed to delete Identity ${identity.metadata.name}: ${error.message}`); } } + + /** + * A static method to process connected grants for a delegate DID. + * + * This will store the grants as the DWN owner to be used later when impersonating the connected DID. + */ + static async processConnectedGrants({ grants, agent, delegateDid }: { + grants: DwnDataEncodedRecordsWriteMessage[], + agent: Web5Agent, + delegateDid: string, + }): Promise { + const connectedProtocols = new Set(); + for (const grantMessage of grants) { + // use the delegateDid as the connectedDid of the grant as they do not yet support impersonation/delegation + const grant = await PermissionGrant.parse({ connectedDid: delegateDid, agent, message: grantMessage }); + // store the grant as the owner of the DWN, this will allow the delegateDid to use the grant when impersonating the connectedDid + const { status } = await grant.store(true); + if (status.code !== 202) { + throw new Error(`AgentDwnApi: Failed to process connected grant: ${status.detail}`); + } + + const protocol = (grant.scope as DwnMessagesPermissionScope | DwnRecordsPermissionScope).protocol; + if (protocol) { + connectedProtocols.add(protocol); + } + } + + // currently we return a de-duped set of protocols represented by these grants, this is used to register protocols for sync + // we expect that any connected protocols will include MessagesQuery and MessagesRead grants that will allow it to sync + return [...connectedProtocols]; + } } diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 16ae2f8bc..984b92bbc 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -11,6 +11,7 @@ import emailProtocolDefinition from './fixtures/protocol-definitions/email.json' import photosProtocolDefinition from './fixtures/protocol-definitions/photos.json' assert { type: 'json' }; import { DwnInterfaceName, DwnMethodName, PermissionsProtocol, Time } from '@tbd54566975/dwn-sdk-js'; import { PermissionGrant } from '../src/permission-grant.js'; +import { Web5 } from '../src/web5.js'; let testDwnUrls: string[] = [testDwnUrl]; @@ -46,6 +47,10 @@ describe('DwnApi', () => { // Instantiate DwnApi for both test identities. dwnAlice = new DwnApi({ agent: testHarness.agent, connectedDid: aliceDid.uri }); dwnBob = new DwnApi({ agent: testHarness.agent, connectedDid: bobDid.uri }); + + // clear cached permissions between test runs + dwnAlice['cachedPermissionsApi'].clear(); + dwnBob['cachedPermissionsApi'].clear(); }); after(async () => { @@ -1385,7 +1390,7 @@ describe('DwnApi', () => { // simulate a connect where bobDid can impersonate aliceDid dwnBob['connectedDid'] = aliceDid.uri; dwnBob['delegateDid'] = bobDid.uri; - await DwnApi.processConnectedGrants({ + await Web5.processConnectedGrants({ agent : testHarness.agent, delegateDid : bobDid.uri, grants : [ deviceXGrant.rawMessage ] @@ -1445,7 +1450,7 @@ describe('DwnApi', () => { }); expect.fail('Should have thrown an error'); } catch(error:any) { - expect(error.message).to.equal('AgentDwnApi: No permissions found for RecordsRead: http://example.com/protocol'); + expect(error.message).to.equal('CachedPermissions: No permissions found for RecordsRead: http://example.com/protocol'); } expect(fetchGrantsSpy.callCount).to.equal(1); @@ -1459,7 +1464,7 @@ describe('DwnApi', () => { }); expect.fail('Should have thrown an error'); } catch(error:any) { - expect(error.message).to.equal('AgentDwnApi: No permissions found for RecordsRead: http://example.com/protocol'); + expect(error.message).to.equal('CachedPermissions: No permissions found for RecordsRead: http://example.com/protocol'); } expect(fetchGrantsSpy.callCount).to.equal(2); // should have been called again diff --git a/packages/api/tests/web5.spec.ts b/packages/api/tests/web5.spec.ts index b91429095..d3449ffab 100644 --- a/packages/api/tests/web5.spec.ts +++ b/packages/api/tests/web5.spec.ts @@ -16,9 +16,22 @@ import { Web5 } from '../src/web5.js'; import { DwnInterfaceName, DwnMethodName, Jws, Time } from '@tbd54566975/dwn-sdk-js'; import { testDwnUrl } from './utils/test-config.js'; import { DidJwk } from '@web5/dids'; -import { DwnApi } from '../src/dwn-api.js'; +import { Convert } from '@web5/common'; describe('web5 api', () => { + let consoleWarn; + + before(() => { + // Suppress console.warn output due to default password warnings + consoleWarn = console.warn; + console.warn = () => {}; + }); + + after(() => { + // Restore console.warn output + console.warn = consoleWarn; + }); + describe('using Test Harness', () => { let testHarness: PlatformAgentTestHarness; @@ -41,24 +54,252 @@ describe('web5 api', () => { await testHarness.closeStorage(); }); - describe('connect()', () => { - it('accepts an externally created DID with an external agent', async () => { - const testIdentity = await testHarness.createIdentity({ - name : 'Test', - testDwnUrls : ['https://dwn.example.com'] + describe('constructor', () => { + it('instantiates Web5 API with provided Web5Agent and connectedDid', async () => { + // Create a new Identity. + const socialIdentity = await testHarness.agent.identity.create({ + metadata : { name: 'Social' }, + didMethod : 'jwk', }); - // Call connect() with the custom agent. - const { web5, did } = await Web5.connect({ + // Instantiates Web5 instance with test agent and new Identity's DID. + const web5 = new Web5({ agent : testHarness.agent, - connectedDid : testIdentity.did.uri + connectedDid : socialIdentity.did.uri, }); - - expect(did).to.exist; expect(web5).to.exist; - expect(did).to.equal(testIdentity.did.uri); + expect(web5).to.have.property('did'); + expect(web5).to.have.property('dwn'); + expect(web5).to.have.property('vc'); + }); + + it('supports a single agent with multiple Web5 instances and different DIDs', async () => { + // Create two identities, each of which is stored in a new tenant. + const careerIdentity = await testHarness.agent.identity.create({ + metadata : { name: 'Social' }, + didMethod : 'jwk', + }); + const socialIdentity = await testHarness.agent.identity.create({ + metadata : { name: 'Social' }, + didMethod : 'jwk', + }); + + // Instantiate a Web5 instance with the "Career" Identity, write a record, and verify the result. + const web5Career = new Web5({ + agent : testHarness.agent, + connectedDid : careerIdentity.did.uri, + }); + expect(web5Career).to.exist; + + // Instantiate a Web5 instance with the "Social" Identity, write a record, and verify the result. + const web5Social = new Web5({ + agent : testHarness.agent, + connectedDid : socialIdentity.did.uri, + }); + expect(web5Social).to.exist; + }); + }); + + describe('scenarios', () => { + it('writes records with multiple identities under management', async () => { + // First launch and initialization. + await testHarness.agent.initialize({ password: 'test' }); + + // Start the Agent, which will decrypt and load the Agent's DID from the vault. + await testHarness.agent.start({ password: 'test' }); + + // Create two identities, each of which is stored in a new tenant. + const careerIdentity = await testHarness.agent.identity.create({ + metadata : { name: 'Social' }, + didMethod : 'jwk', + }); + const socialIdentity = await testHarness.agent.identity.create({ + metadata : { name: 'Social' }, + didMethod : 'jwk', + }); + + // Instantiate a Web5 instance with the "Career" Identity, write a record, and verify the result. + const web5Career = new Web5({ + agent : testHarness.agent, + connectedDid : careerIdentity.did.uri, + }); + const careerResult = await web5Career.dwn.records.write({ + data : 'Hello, world!', + message : { + schema : 'foo/bar', + dataFormat : 'text/plain', + }, + }); + expect(careerResult.status.code).to.equal(202); + expect(careerResult.record).to.exist; + expect(careerResult.record?.author).to.equal(careerIdentity.did.uri); + expect(await careerResult.record?.data.text()).to.equal( + 'Hello, world!' + ); + + // Instantiate a Web5 instance with the "Social" Identity, write a record, and verify the result. + const web5Social = new Web5({ + agent : testHarness.agent, + connectedDid : socialIdentity.did.uri, + }); + const socialResult = await web5Social.dwn.records.write({ + data : 'Hello, everyone!', + message : { + schema : 'foo/bar', + dataFormat : 'text/plain', + }, + }); + expect(socialResult.status.code).to.equal(202); + expect(socialResult.record).to.exist; + expect(socialResult.record?.author).to.equal(socialIdentity.did.uri); + expect(await socialResult.record?.data.text()).to.equal( + 'Hello, everyone!' + ); + }); + }); + }); + + describe('connect()', () => { + let testHarness: PlatformAgentTestHarness; + + before(async () => { + testHarness = await PlatformAgentTestHarness.setup({ + agentClass : Web5UserAgent, + agentStores : 'memory', + }); + }); + + beforeEach(async () => { + sinon.restore(); + testHarness.agent.sync.stopSync(); + await testHarness.clearStorage(); + await testHarness.createAgentDid(); + }); + + after(async () => { + sinon.restore(); + await testHarness.clearStorage(); + await testHarness.closeStorage(); + }); + + it('uses Web5UserAgent, by default', async () => { + // stub the create method of the Web5UserAgent to use the test harness agent + // this avoids DB locks when the agent is created twice + sinon.stub(Web5UserAgent, 'create').resolves(testHarness.agent as Web5UserAgent); + + const { web5, recoveryPhrase, did } = await Web5.connect(); + expect(web5).to.exist; + expect(web5.agent).to.be.instanceOf(Web5UserAgent); + // Verify recovery phrase is a 12-word string. + expect(recoveryPhrase).to.be.a('string'); + expect(recoveryPhrase.split(' ')).to.have.lengthOf(12); + + // if called again, the same DID is returned, and the recovery phrase is not regenerated + const { recoveryPhrase: recoveryPhraseConnect2, did: didConnect2 } = await Web5.connect(); + expect(recoveryPhraseConnect2).to.be.undefined; + expect(didConnect2).to.equal(did); + }); + + it('accepts an externally created DID', async () => { + const walletConnectSpy = sinon.spy(WalletConnect, 'initClient'); + + const testIdentity = await testHarness.createIdentity({ + name : 'Test', + testDwnUrls : ['https://dwn.example.com'], + }); + + // Call connect() with the custom agent. + const { web5, did } = await Web5.connect({ + agent : testHarness.agent, + connectedDid : testIdentity.did.uri, + }); + + expect(did).to.exist; + expect(web5).to.exist; + expect(walletConnectSpy.called).to.be.false; + }); + + it('creates an identity using the provided techPreview dwnEndpoints', async () => { + sinon + .stub(Web5UserAgent, 'create') + .resolves(testHarness.agent as Web5UserAgent); + const walletConnectSpy = sinon.spy(WalletConnect, 'initClient'); + const identityApiSpy = sinon.spy(AgentIdentityApi.prototype, 'create'); + const { web5, did } = await Web5.connect({ + techPreview: { dwnEndpoints: ['https://dwn.example.com/preview'] }, + }); + expect(web5).to.exist; + expect(did).to.exist; + + expect(identityApiSpy.calledOnce, 'identityApiSpy called').to.be.true; + const serviceEndpoints = ( + identityApiSpy.firstCall.args[0].didOptions as any + ).services[0].serviceEndpoint; + expect(serviceEndpoints).to.deep.equal([ + 'https://dwn.example.com/preview', + ]); + expect(walletConnectSpy.called).to.be.false; + }); + + it('creates an identity using the provided didCreateOptions dwnEndpoints', async () => { + sinon + .stub(Web5UserAgent, 'create') + .resolves(testHarness.agent as Web5UserAgent); + const identityApiSpy = sinon.spy(AgentIdentityApi.prototype, 'create'); + const walletConnectSpy = sinon.spy(WalletConnect, 'initClient'); + const { web5, did } = await Web5.connect({ + didCreateOptions: { dwnEndpoints: ['https://dwn.example.com'] }, + }); + expect(web5).to.exist; + expect(did).to.exist; + + expect(identityApiSpy.calledOnce, 'identityApiSpy called').to.be.true; + const serviceEndpoints = ( + identityApiSpy.firstCall.args[0].didOptions as any + ).services[0].serviceEndpoint; + expect(serviceEndpoints).to.deep.equal(['https://dwn.example.com']); + expect(walletConnectSpy.called).to.be.false; + }); + + it('defaults to the first identity if multiple identities exist', async () => { + // scenario: For some reason more than one identity exists when attempting to re-connect to `Web5` + // the first identity in the array should be the one selected + // TODO: this has happened due to a race condition somewhere. Dig into this issue and implement a better way to select/manage DIDs when using `Web5.connect()` + + // create an identity by connecting + sinon.stub(Web5UserAgent, 'create').resolves(testHarness.agent as Web5UserAgent); + const { web5, did } = await Web5.connect({ techPreview: { dwnEndpoints: [ testDwnUrl ] }}); + expect(web5).to.exist; + expect(did).to.exist; + + // create a second identity + await testHarness.agent.identity.create({ + didMethod : 'jwk', + metadata : { name: 'Second' } }); + // connect again + const { did: did2 } = await Web5.connect(); + expect(did2).to.equal(did); + }); + + it('defaults to `https://dwn.tbddev.org/beta` as the single DWN Service endpoint if non is provided', async () => { + sinon + .stub(Web5UserAgent, 'create') + .resolves(testHarness.agent as Web5UserAgent); + const identityApiSpy = sinon.spy(AgentIdentityApi.prototype, 'create'); + const { web5, did } = await Web5.connect(); + expect(web5).to.exist; + expect(did).to.exist; + + expect(identityApiSpy.calledOnce, 'identityApiSpy called').to.be.true; + const serviceEndpoints = ( + identityApiSpy.firstCall.args[0].didOptions as any + ).services[0].serviceEndpoint; + expect(serviceEndpoints).to.deep.equal(['https://dwn.tbddev.org/beta']); + }); + + describe('walletConnectOptions', () => { it('uses walletConnectOptions to connect to a DID and import the grants', async () => { // Create a new Identity. const alice = await testHarness.createIdentity({ @@ -90,6 +331,16 @@ describe('web5 api', () => { }, }); expect(protocolConfigReply.status.code).to.equal(202); + + // send the protocol to alice's remote DWN + const { reply: protocolSendReply } = await testHarness.agent.dwn.sendRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.ProtocolsConfigure, + rawMessage : protocolConfigureMessage, + }); + expect(protocolSendReply.status.code).to.equal(202); + // create an identity for the app to use const app = await testHarness.agent.did.create({ store : false, @@ -109,6 +360,16 @@ describe('web5 api', () => { } }); + const { encodedData: writeGrantEncodedData, ...writeGrantMessage } = writeGrant.message; + const writeGrantSend = await testHarness.agent.sendDwnRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : writeGrantMessage, + dataStream : new Blob([ Convert.base64Url(writeGrantEncodedData).toUint8Array() ]) + }); + expect(writeGrantSend.reply.status.code).to.equal(202); + const readGrant = await testHarness.agent.permissions.createGrant({ delegated : true, author : alice.did.uri, @@ -121,9 +382,62 @@ describe('web5 api', () => { } }); - // stub the walletInit method of the Connect placeholder class + const { encodedData: readGrantEncodedData, ...readGrantMessage } = readGrant.message; + const readGrantSend = await testHarness.agent.sendDwnRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : readGrantMessage, + dataStream : new Blob([ Convert.base64Url(readGrantEncodedData).toUint8Array() ]) + }); + expect(readGrantSend.reply.status.code).to.equal(202); + + // create MessagesQuery and MessagesRead grants so that sync does not fail + const messagesQueryGrant = await testHarness.agent.permissions.createGrant({ + store : true, + author : alice.did.uri, + grantedTo : app.uri, + dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), + scope : { + interface : DwnInterfaceName.Messages, + method : DwnMethodName.Query, + } + }); + + const { encodedData: messagesQueryGrantEncodedData, ...messagesQueryGrantMessage} = messagesQueryGrant.message; + const messagesQueryGrantSend = await testHarness.agent.sendDwnRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesQueryGrantMessage, + dataStream : new Blob([ Convert.base64Url(messagesQueryGrantEncodedData).toUint8Array() ]) + }); + expect(messagesQueryGrantSend.reply.status.code).to.equal(202); + + const messagesReadGrant = await testHarness.agent.permissions.createGrant({ + store : true, + author : alice.did.uri, + grantedTo : app.uri, + dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), + scope : { + interface : DwnInterfaceName.Messages, + method : DwnMethodName.Read, + } + }); + + const { encodedData: messagesReadEncodedData, ...messagesReadGrantMessage } = messagesReadGrant.message; + const messagesReadGrantSend = await testHarness.agent.sendDwnRequest({ + author : alice.did.uri, + target : alice.did.uri, + messageType : DwnInterface.RecordsWrite, + rawMessage : messagesReadGrantMessage, + dataStream : new Blob([ Convert.base64Url(messagesReadEncodedData).toUint8Array() ]) + }); + expect(messagesReadGrantSend.reply.status.code).to.equal(202); + + // stub the walletInit method sinon.stub(WalletConnect, 'initClient').resolves({ - delegateGrants : [ writeGrant.message, readGrant.message ], + delegateGrants : [ writeGrant.message, readGrant.message, messagesQueryGrant.message, messagesReadGrant.message ], delegateDid : await app.export(), connectedDid : alice.did.uri }); @@ -155,15 +469,6 @@ describe('web5 api', () => { expect(did).to.equal(alice.did.uri); expect(delegateDid).to.equal(app.uri); - // in lieu of sync, we will process the grants and protocol definition on the local connected agent - const { reply: localProtocolReply } = await web5.agent.processDwnRequest({ - author : alice.did.uri, - target : alice.did.uri, - messageType : DwnInterface.ProtocolsConfigure, - rawMessage : protocolConfigureMessage, - }); - expect(localProtocolReply.status.code).to.equal(202); - // use the grant to write a record const writeResult = await web5.dwn.records.write({ data : 'Hello, world!', @@ -203,7 +508,7 @@ describe('web5 api', () => { expect.fail('Should have thrown an error'); } catch(error:any) { - expect(error.message).to.include('AgentDwnApi: No permissions found for RecordsQuery'); + expect(error.message).to.include('CachedPermissions: No permissions found for RecordsQuery'); } try { @@ -216,14 +521,14 @@ describe('web5 api', () => { expect.fail('Should have thrown an error'); } catch(error:any) { - expect(error.message).to.include('AgentDwnApi: No permissions found for RecordsDelete'); + expect(error.message).to.include('CachedPermissions: No permissions found for RecordsDelete'); } // grant query and delete permissions const queryGrant = await testHarness.agent.permissions.createGrant({ - author : alice.did.uri, delegated : true, - grantedTo : app.uri, + author : alice.did.uri, + grantedTo : delegateDid, dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), scope : { interface : DwnInterfaceName.Records, @@ -233,9 +538,9 @@ describe('web5 api', () => { }); const deleteGrant = await testHarness.agent.permissions.createGrant({ - author : alice.did.uri, delegated : true, - grantedTo : app.uri, + author : alice.did.uri, + grantedTo : delegateDid, dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), scope : { interface : DwnInterfaceName.Records, @@ -246,10 +551,10 @@ describe('web5 api', () => { // write the grants to app as owner // this also clears the grants cache - await DwnApi.processConnectedGrants({ - grants : [ queryGrant.message, deleteGrant.message ], - agent : appTestHarness.agent, - delegateDid : app.uri, + await Web5.processConnectedGrants({ + grants : [ queryGrant.message, deleteGrant.message ], + agent : appTestHarness.agent, + delegateDid, }); // attempt to delete using the grant @@ -367,6 +672,9 @@ describe('web5 api', () => { // stub the create method of the Web5UserAgent to use the test harness agent sinon.stub(Web5UserAgent, 'create').resolves(appTestHarness.agent as Web5UserAgent); + // stub console.error so that it doesn't log in the test output and use it as a spy confirming the error messages were logged + const consoleSpy = sinon.stub(console, 'error').returns(); + try { // connect to the app, the options don't matter because we're stubbing the initClient method await Web5.connect({ @@ -385,8 +693,8 @@ describe('web5 api', () => { } // check that the Identity was deleted - const appDid = await appTestHarness.agent.identity.list(); - expect(appDid).to.have.lengthOf(0); + const appIdentities = await appTestHarness.agent.identity.list(); + expect(appIdentities).to.have.lengthOf(0); // close the app test harness storage await appTestHarness.clearStorage(); @@ -414,37 +722,43 @@ describe('web5 api', () => { expect(consoleSpy.calledTwice, 'console.error called twice').to.be.true; }); - it('uses walletConnectOptions to connect to a DID and import the grants', async () => { - // Create a new Identity. - const alice = await testHarness.createIdentity({ - name : 'Alice', - testDwnUrls : [testDwnUrl] + it('throws an error if walletConnectOptions are provided and sync is set to `off`', async () => { + // stub the walletInit method + sinon.stub(WalletConnect, 'initClient').resolves({ + delegateGrants : [ ], + delegateDid : {} as any, + connectedDid : '' }); - // alice installs a protocol definition - const protocol: DwnProtocolDefinition = { - protocol : 'https://example.com/test-protocol', - published : true, - types : { - foo : {}, - bar : {} - }, - structure: { - foo: { - bar: {} + // stub the create method of the Web5UserAgent to use the test harness agent + sinon.stub(Web5UserAgent, 'create').resolves(testHarness.agent as Web5UserAgent); + + try { + + // attempt to connect with sync set to false + await Web5.connect({ + sync : 'off', + walletConnectOptions : { + connectServerUrl : 'https://connect.example.com', + walletUri : 'https://wallet.example.com', + validatePin : async () => { return '1234'; }, + onWalletUriReady : (_walletUri: string) => {}, + permissionRequests : [] } - } - }; + }); - const { reply: protocolConfigReply, message: protocolConfigureMessage } = await testHarness.agent.dwn.processRequest({ - author : alice.did.uri, - target : alice.did.uri, - messageType : DwnInterface.ProtocolsConfigure, - messageParams : { - definition: protocol, - }, + expect.fail('Should have thrown an error'); + } catch(error: any) { + expect(error.message).to.equal('Sync must not be disabled when using WalletConnect'); + } + }); + + it('does not throw an error if walletConnectOptions are provided and sync is not the default', async () => { + // Create a new Identity. + const alice = await testHarness.createIdentity({ + name : 'Alice', + testDwnUrls : [testDwnUrl] }); - expect(protocolConfigReply.status.code).to.equal(202); // create an identity for the app to use const app = await testHarness.agent.did.create({ @@ -452,592 +766,33 @@ describe('web5 api', () => { method : 'jwk', }); - // create grants for the app to use - const writeGrant = await testHarness.agent.permissions.createGrant({ - delegated : true, - author : alice.did.uri, - grantedTo : app.uri, - dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), - scope : { - interface : DwnInterfaceName.Records, - method : DwnMethodName.Write, - protocol : protocol.protocol, - } - }); - - const readGrant = await testHarness.agent.permissions.createGrant({ - delegated : true, - author : alice.did.uri, - grantedTo : app.uri, - dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), - scope : { - interface : DwnInterfaceName.Records, - method : DwnMethodName.Read, - protocol : protocol.protocol, - } - }); - // stub the walletInit method sinon.stub(WalletConnect, 'initClient').resolves({ - delegateGrants : [ writeGrant.message, readGrant.message ], + delegateGrants : [ ], delegateDid : await app.export(), connectedDid : alice.did.uri }); - const appTestHarness = await PlatformAgentTestHarness.setup({ - agentClass : Web5UserAgent, - agentStores : 'memory', - testDataLocation : '__TESTDATA__/web5-connect-app' - }); - await appTestHarness.clearStorage(); - await appTestHarness.createAgentDid(); - // stub the create method of the Web5UserAgent to use the test harness agent - sinon.stub(Web5UserAgent, 'create').resolves(appTestHarness.agent as Web5UserAgent); - - // connect to the app, the options don't matter because we're stubbing the initClient method - const { web5, did, delegateDid } = await Web5.connect({ - walletConnectOptions: { + sinon.stub(Web5UserAgent, 'create').resolves(testHarness.agent as Web5UserAgent); + + // stub the sync records as we are not actually testing sync + sinon.stub(testHarness.agent.sync, 'sync').resolves(); + const startSyncSpy = sinon.stub(testHarness.agent.sync, 'startSync').resolves(); + // attempt to connect with sync set to false + await Web5.connect({ + sync : '1m', + walletConnectOptions : { connectServerUrl : 'https://connect.example.com', walletUri : 'https://wallet.example.com', validatePin : async () => { return '1234'; }, onWalletUriReady : (_walletUri: string) => {}, - permissionRequests : [], - } - }); - expect(web5).to.exist; - expect(did).to.exist; - expect(delegateDid).to.exist; - expect(did).to.equal(alice.did.uri); - expect(delegateDid).to.equal(app.uri); - - // in lieu of sync, we will process the grants and protocol definition on the local connected agent - const { reply: localProtocolReply } = await web5.agent.processDwnRequest({ - author : alice.did.uri, - target : alice.did.uri, - messageType : DwnInterface.ProtocolsConfigure, - rawMessage : protocolConfigureMessage, - }); - expect(localProtocolReply.status.code).to.equal(202); - - // use the grant to write a record - const writeResult = await web5.dwn.records.write({ - data : 'Hello, world!', - message : { - protocol : protocol.protocol, - protocolPath : 'foo', - } - }); - expect(writeResult.status.code).to.equal(202); - expect(writeResult.record).to.exist; - // test that the logical author is the connected DID and the signer is the impersonator DID - expect(writeResult.record.author).to.equal(did); - const writeSigner = Jws.getSignerDid(writeResult.record.authorization.signature.signatures[0]); - expect(writeSigner).to.equal(delegateDid); - - const readResult = await web5.dwn.records.read({ - protocol : protocol.protocol, - message : { - filter: { recordId: writeResult.record.id } - } - }); - expect(readResult.status.code).to.equal(200); - expect(readResult.record).to.exist; - // test that the logical author is the connected DID and the signer is the impersonator DID - expect(readResult.record.author).to.equal(did); - const readSigner = Jws.getSignerDid(readResult.record.authorization.signature.signatures[0]); - expect(readSigner).to.equal(delegateDid); - - // attempt to query or delete, should fail because we did not grant query permissions - try { - await web5.dwn.records.query({ - protocol : protocol.protocol, - message : { - filter: { protocol: protocol.protocol } - } - }); - - expect.fail('Should have thrown an error'); - } catch(error:any) { - expect(error.message).to.include('AgentDwnApi: No permissions found for RecordsQuery'); - } - - try { - await web5.dwn.records.delete({ - protocol : protocol.protocol, - message : { - recordId: writeResult.record.id - } - }); - - expect.fail('Should have thrown an error'); - } catch(error:any) { - expect(error.message).to.include('AgentDwnApi: No permissions found for RecordsDelete'); - } - - // grant query and delete permissions - const queryGrant = await testHarness.agent.permissions.createGrant({ - delegated : true, - author : alice.did.uri, - grantedTo : delegateDid, - dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), - scope : { - interface : DwnInterfaceName.Records, - method : DwnMethodName.Query, - protocol : protocol.protocol, - } - }); - - const deleteGrant = await testHarness.agent.permissions.createGrant({ - delegated : true, - author : alice.did.uri, - grantedTo : delegateDid, - dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), - scope : { - interface : DwnInterfaceName.Records, - method : DwnMethodName.Delete, - protocol : protocol.protocol, - } - }); - - // write the grants to app as owner - // this also clears the grants cache - await DwnApi.processConnectedGrants({ - grants : [ queryGrant.message, deleteGrant.message ], - agent : appTestHarness.agent, - delegateDid, - }); - - // attempt to delete using the grant - const deleteResult = await web5.dwn.records.delete({ - protocol : protocol.protocol, - message : { - recordId: writeResult.record.id - } - }); - expect(deleteResult.status.code).to.equal(202); - - // attempt to query using the grant - const queryResult = await web5.dwn.records.query({ - protocol : protocol.protocol, - message : { - filter: { protocol: protocol.protocol } + permissionRequests : [] } }); - expect(queryResult.status.code).to.equal(200); - expect(queryResult.records).to.have.lengthOf(0); // record has been deleted - // connecting a 2nd time will return the same connectedDID and delegatedDID - const { did: did2, delegateDid: delegateDid2 } = await Web5.connect(); - expect(did2).to.equal(did); - expect(delegateDid2).to.equal(delegateDid); - - // Close the app test harness storage. - await appTestHarness.clearStorage(); - await appTestHarness.closeStorage(); + expect(startSyncSpy.args[0][0].interval).to.equal('1m'); }); - - it('cleans up imported Identity from walletConnectOptions flow if grants cannot be processed', async () => { - const alice = await testHarness.createIdentity({ - name : 'Alice', - testDwnUrls : [testDwnUrl] - }); - - // alice installs a protocol definition - const protocol: DwnProtocolDefinition = { - protocol : 'https://example.com/test-protocol', - published : true, - types : { - foo : {}, - bar : {} - }, - structure: { - foo: { - bar: {} - } - } - }; - - const { reply: protocolConfigReply } = await testHarness.agent.dwn.processRequest({ - author : alice.did.uri, - target : alice.did.uri, - messageType : DwnInterface.ProtocolsConfigure, - messageParams : { - definition: protocol, - }, - }); - expect(protocolConfigReply.status.code).to.equal(202); - // create an identity for the app to use - const app = await testHarness.agent.did.create({ - store : false, - method : 'jwk', - }); - - // create grants for the app to use - const writeGrant = await testHarness.agent.permissions.createGrant({ - delegated : true, - author : alice.did.uri, - grantedTo : app.uri, - dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), - scope : { - interface : DwnInterfaceName.Records, - method : DwnMethodName.Write, - protocol : protocol.protocol, - } - }); - - const readGrant = await testHarness.agent.permissions.createGrant({ - delegated : true, - author : alice.did.uri, - grantedTo : app.uri, - dateExpires : Time.createOffsetTimestamp({ seconds: 60 }), - scope : { - interface : DwnInterfaceName.Records, - method : DwnMethodName.Read, - protocol : protocol.protocol, - } - }); - - // stub the walletInit method of the Connect placeholder class - sinon.stub(WalletConnect, 'initClient').resolves({ - delegateGrants : [ writeGrant.message, readGrant.message ], - delegateDid : await app.export(), - connectedDid : alice.did.uri - }); - - const appTestHarness = await PlatformAgentTestHarness.setup({ - agentClass : Web5UserAgent, - agentStores : 'memory', - testDataLocation : '__TESTDATA__/web5-connect-app' - }); - await appTestHarness.clearStorage(); - await appTestHarness.createAgentDid(); - - - // stub processDwnRequest to return a non 202 error code - sinon.stub(appTestHarness.agent, 'processDwnRequest').resolves({ - messageCid : '', - reply : { status: { code: 400, detail: 'Bad Request' } } - }); - - // stub the create method of the Web5UserAgent to use the test harness agent - sinon.stub(Web5UserAgent, 'create').resolves(appTestHarness.agent as Web5UserAgent); - - // stub console.error so that it doesn't log in the test output and use it as a spy confirming the error messages were logged - const consoleSpy = sinon.stub(console, 'error').returns(); - - try { - // connect to the app, the options don't matter because we're stubbing the initClient method - await Web5.connect({ - walletConnectOptions: { - connectServerUrl : 'https://connect.example.com', - walletUri : 'https://wallet.example.com', - validatePin : async () => { return '1234'; }, - onWalletUriReady : (_walletUri: string) => {}, - permissionRequests : [] - } - }); - - expect.fail('Should have thrown an error'); - } catch(error:any) { - expect(error.message).to.equal('Failed to connect to wallet: AgentDwnApi: Failed to process connected grant: Bad Request'); - } - - // check that the Identity was deleted - const appIdentities = await appTestHarness.agent.identity.list(); - expect(appIdentities).to.have.lengthOf(0); - - // close the app test harness storage - await appTestHarness.clearStorage(); - await appTestHarness.closeStorage(); - }); - - it('logs an error if there is a failure during cleanup of Identity information, but does not throw', async () => { - // create a DID that is not stored in the agent - const did = await DidJwk.create(); - const identity = new BearerIdentity({ - did, - metadata: { - name : 'Test', - uri : did.uri, - tenant : did.uri - } - }); - - // stub console.error to avoid logging errors into the test output, use as spy to check if the error message is logged - const consoleSpy = sinon.stub(console, 'error').returns(); - - // call identityCleanup on a did that does not exist - await Web5['cleanUpIdentity']({ userAgent: testHarness.agent as Web5UserAgent, identity }); - - expect(consoleSpy.calledTwice, 'console.error called twice').to.be.true; - }); - }); - - describe('constructor', () => { - it('instantiates Web5 API with provided Web5Agent and connectedDid', async () => { - // Create a new Identity. - const socialIdentity = await testHarness.agent.identity.create({ - metadata : { name: 'Social' }, - didMethod : 'jwk', - }); - - // Instantiates Web5 instance with test agent and new Identity's DID. - const web5 = new Web5({ - agent : testHarness.agent, - connectedDid : socialIdentity.did.uri, - }); - expect(web5).to.exist; - expect(web5).to.have.property('did'); - expect(web5).to.have.property('dwn'); - expect(web5).to.have.property('vc'); - }); - - it('supports a single agent with multiple Web5 instances and different DIDs', async () => { - // Create two identities, each of which is stored in a new tenant. - const careerIdentity = await testHarness.agent.identity.create({ - metadata : { name: 'Social' }, - didMethod : 'jwk', - }); - const socialIdentity = await testHarness.agent.identity.create({ - metadata : { name: 'Social' }, - didMethod : 'jwk', - }); - - // Instantiate a Web5 instance with the "Career" Identity, write a record, and verify the result. - const web5Career = new Web5({ - agent : testHarness.agent, - connectedDid : careerIdentity.did.uri, - }); - expect(web5Career).to.exist; - - // Instantiate a Web5 instance with the "Social" Identity, write a record, and verify the result. - const web5Social = new Web5({ - agent : testHarness.agent, - connectedDid : socialIdentity.did.uri, - }); - expect(web5Social).to.exist; - }); - }); - - describe('scenarios', () => { - it('writes records with multiple identities under management', async () => { - // First launch and initialization. - await testHarness.agent.initialize({ password: 'test' }); - - // Start the Agent, which will decrypt and load the Agent's DID from the vault. - await testHarness.agent.start({ password: 'test' }); - - // Create two identities, each of which is stored in a new tenant. - const careerIdentity = await testHarness.agent.identity.create({ - metadata : { name: 'Social' }, - didMethod : 'jwk', - }); - const socialIdentity = await testHarness.agent.identity.create({ - metadata : { name: 'Social' }, - didMethod : 'jwk', - }); - - // Instantiate a Web5 instance with the "Career" Identity, write a record, and verify the result. - const web5Career = new Web5({ - agent : testHarness.agent, - connectedDid : careerIdentity.did.uri, - }); - const careerResult = await web5Career.dwn.records.write({ - data : 'Hello, world!', - message : { - schema : 'foo/bar', - dataFormat : 'text/plain', - }, - }); - expect(careerResult.status.code).to.equal(202); - expect(careerResult.record).to.exist; - expect(careerResult.record?.author).to.equal(careerIdentity.did.uri); - expect(await careerResult.record?.data.text()).to.equal( - 'Hello, world!' - ); - - // Instantiate a Web5 instance with the "Social" Identity, write a record, and verify the result. - const web5Social = new Web5({ - agent : testHarness.agent, - connectedDid : socialIdentity.did.uri, - }); - const socialResult = await web5Social.dwn.records.write({ - data : 'Hello, everyone!', - message : { - schema : 'foo/bar', - dataFormat : 'text/plain', - }, - }); - expect(socialResult.status.code).to.equal(202); - expect(socialResult.record).to.exist; - expect(socialResult.record?.author).to.equal(socialIdentity.did.uri); - expect(await socialResult.record?.data.text()).to.equal( - 'Hello, everyone!' - ); - }); - }); - }); - - describe('connect()', () => { - let testHarness: PlatformAgentTestHarness; - - before(async () => { - testHarness = await PlatformAgentTestHarness.setup({ - agentClass : Web5UserAgent, - agentStores : 'memory', - }); - }); - - beforeEach(async () => { - sinon.restore(); - await testHarness.clearStorage(); - await testHarness.createAgentDid(); - }); - - after(async () => { - sinon.restore(); - await testHarness.clearStorage(); - await testHarness.closeStorage(); - }); - - it('uses Web5UserAgent, by default', async () => { - // stub the create method of the Web5UserAgent to use the test harness agent - // this avoids DB locks when the agent is created twice - sinon.stub(Web5UserAgent, 'create').resolves(testHarness.agent as Web5UserAgent); - - const { web5, recoveryPhrase, did } = await Web5.connect(); - expect(web5).to.exist; - expect(web5.agent).to.be.instanceOf(Web5UserAgent); - // Verify recovery phrase is a 12-word string. - expect(recoveryPhrase).to.be.a('string'); - expect(recoveryPhrase.split(' ')).to.have.lengthOf(12); - - // if called again, the same DID is returned, and the recovery phrase is not regenerated - const { recoveryPhrase: recoveryPhraseConnect2, did: didConnect2 } = await Web5.connect(); - expect(recoveryPhraseConnect2).to.be.undefined; - expect(didConnect2).to.equal(did); - }); - - it('accepts an externally created DID', async () => { - const walletConnectSpy = sinon.spy(WalletConnect, 'initClient'); - - const testIdentity = await testHarness.createIdentity({ - name : 'Test', - testDwnUrls : ['https://dwn.example.com'], - }); - - // Call connect() with the custom agent. - const { web5, did } = await Web5.connect({ - agent : testHarness.agent, - connectedDid : testIdentity.did.uri, - }); - - expect(did).to.exist; - expect(web5).to.exist; - expect(walletConnectSpy.called).to.be.false; - }); - - it('creates an identity using the provided techPreview dwnEndpoints', async () => { - sinon - .stub(Web5UserAgent, 'create') - .resolves(testHarness.agent as Web5UserAgent); - const walletConnectSpy = sinon.spy(WalletConnect, 'initClient'); - const identityApiSpy = sinon.spy(AgentIdentityApi.prototype, 'create'); - const { web5, did } = await Web5.connect({ - techPreview: { dwnEndpoints: ['https://dwn.example.com/preview'] }, - }); - expect(web5).to.exist; - expect(did).to.exist; - - expect(identityApiSpy.calledOnce, 'identityApiSpy called').to.be.true; - const serviceEndpoints = ( - identityApiSpy.firstCall.args[0].didOptions as any - ).services[0].serviceEndpoint; - expect(serviceEndpoints).to.deep.equal([ - 'https://dwn.example.com/preview', - ]); - expect(walletConnectSpy.called).to.be.false; - }); - - it('creates an identity using the provided didCreateOptions dwnEndpoints', async () => { - sinon - .stub(Web5UserAgent, 'create') - .resolves(testHarness.agent as Web5UserAgent); - const identityApiSpy = sinon.spy(AgentIdentityApi.prototype, 'create'); - const walletConnectSpy = sinon.spy(WalletConnect, 'initClient'); - const { web5, did } = await Web5.connect({ - didCreateOptions: { dwnEndpoints: ['https://dwn.example.com'] }, - }); - expect(web5).to.exist; - expect(did).to.exist; - - expect(identityApiSpy.calledOnce, 'identityApiSpy called').to.be.true; - const serviceEndpoints = ( - identityApiSpy.firstCall.args[0].didOptions as any - ).services[0].serviceEndpoint; - expect(serviceEndpoints).to.deep.equal(['https://dwn.example.com']); - expect(walletConnectSpy.called).to.be.false; - }); - - it('defaults to the first identity if multiple identities exist', async () => { - // scenario: For some reason more than one identity exists when attempting to re-connect to `Web5` - // the first identity in the array should be the one selected - // TODO: this has happened due to a race condition somewhere. Dig into this issue and implement a better way to select/manage DIDs when using `Web5.connect()` - - // create an identity by connecting - sinon.stub(Web5UserAgent, 'create').resolves(testHarness.agent as Web5UserAgent); - const { web5, did } = await Web5.connect({ techPreview: { dwnEndpoints: [ testDwnUrl ] }}); - expect(web5).to.exist; - expect(did).to.exist; - - // create a second identity - await testHarness.agent.identity.create({ - didMethod : 'jwk', - metadata : { name: 'Second' } - }); - - // connect again - const { did: did2 } = await Web5.connect(); - expect(did2).to.equal(did); - }); - - it('defaults to the first identity if multiple identities exist', async () => { - // scenario: For some reason more than one identity exists when attempting to re-connect to `Web5` - // the first identity in the array should be the one selected - // TODO: this has happened due to a race condition somewhere. Dig into this issue and implement a better way to select/manage DIDs when using `Web5.connect()` - - // create an identity by connecting - sinon.stub(Web5UserAgent, 'create').resolves(testHarness.agent as Web5UserAgent); - const { web5, did } = await Web5.connect({ techPreview: { dwnEndpoints: [ testDwnUrl ] }}); - expect(web5).to.exist; - expect(did).to.exist; - - // create a second identity - await testHarness.agent.identity.create({ - didMethod : 'jwk', - metadata : { name: 'Second' } - }); - - // connect again - const { did: did2 } = await Web5.connect(); - expect(did2).to.equal(did); - }); - - it('defaults to `https://dwn.tbddev.org/beta` as the single DWN Service endpoint if non is provided', async () => { - sinon - .stub(Web5UserAgent, 'create') - .resolves(testHarness.agent as Web5UserAgent); - const identityApiSpy = sinon.spy(AgentIdentityApi.prototype, 'create'); - const { web5, did } = await Web5.connect(); - expect(web5).to.exist; - expect(did).to.exist; - - expect(identityApiSpy.calledOnce, 'identityApiSpy called').to.be.true; - const serviceEndpoints = ( - identityApiSpy.firstCall.args[0].didOptions as any - ).services[0].serviceEndpoint; - expect(serviceEndpoints).to.deep.equal(['https://dwn.tbddev.org/beta']); }); describe('registration', () => {