Snap sync moving root/healing implementation

packages/client/src/service/fullethereumservice.ts

@@ -11,6 +11,7 @@ import { LesProtocol } from '../net/protocol/lesprotocol.js'
 import { SnapProtocol } from '../net/protocol/snapprotocol.js'
 import { BeaconSynchronizer, FullSynchronizer, SnapSynchronizer } from '../sync/index.js'
 import { Event } from '../types.js'
+import { wait } from '../util/wait.js'
 
 import { Service, type ServiceOptions } from './service.js'
 import { Skeleton } from './skeleton.js'
@@ -243,7 +244,36 @@ export class FullEthereumService extends Service {
  * vm execution
  */
  async buildHeadState(): Promise<void> {
- if (this.building) return
+ if (this.building) {
+ if (this.snapsync !== undefined) {
+ // discover best peer
+ let peer = await this.snapsync.best()
+ let numAttempts = 1
+ while (!peer && this.snapsync.opened) {
+ this.snapsync.config.logger.debug(`Waiting for best peer (attempt #${numAttempts})`)
+ await wait(5000)
+ peer = await this.snapsync.best()
+ numAttempts += 1
+ }
+
+ if (peer === undefined) throw new Error('Unable to find a peer to sync with')
+ const latest = await peer.latest()
+
+ if (latest === undefined || latest.stateRoot === undefined) {
+ // TODO if latest is undefined, latest known root is assumed to not yet have been updated?
+ return
+ }
+
+ // TODO only update root if it is a new one compared to what's already being used
+ const fetcher = this.snapsync.fetcher
+ if (fetcher === null) {
+ return
+ }
+ fetcher.updateStateRoot(latest!.stateRoot as Uint8Array)
+ }
+
+ return
+ }
  this.building = true
 
  try {

packages/client/src/sync/fetcher/accountfetcher.ts

@@ -100,6 +100,7 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
  this.accountTrie = this.stateManager['_getAccountTrie']()
 
  this.debug = createDebugLogger('client:AccountFetcher')
+ this.debug('dbg101')
 
  this.storageFetcher = new StorageFetcher({
  config: this.config,
@@ -276,6 +277,8 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
  }
  break
  case TrieNodeFetcher:
+ // TODO update to check if heal phase completed successfully and then continue with next
+ // healing phase
  fetcherDoneFlags.trieNodeFetcher.done = true
  break
  }
@@ -585,6 +588,11 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
 
  updateStateRoot(stateRoot: Uint8Array) {
  this.root = stateRoot
+
+ this.storageFetcher.updateStateRoot(stateRoot)
+
+ // TODO trieNodeFetcher needs to be constantly healing while other fetchers work
+ this.trieNodeFetcher.updateStateRoot(stateRoot)
  }
 
  nextTasks(): void {

packages/client/src/sync/fetcher/storagefetcher.ts

@@ -617,6 +617,10 @@ export class StorageFetcher extends Fetcher<JobTask, StorageData[][], StorageDat
  return tasks
  }
 
+ updateStateRoot(stateRoot: Uint8Array) {
+ this.root = stateRoot
+ }
+
  nextTasks(): void {
  this.debug(
  `Entering nextTasks with primary queue length of ${this.storageRequests.length} and secondary queue length of ${this.fragmentedRequests.length}`

packages/client/src/sync/fetcher/trienodefetcher.ts

@@ -61,6 +61,7 @@ type FetchedNodeData = {
 }
 
 type NodeRequestData = {
+ requested: boolean
  nodeHash: string
  nodeParentHash: string
  parentAccountHash?: string // for leaf account nodes that contain a storage component
@@ -117,6 +118,7 @@ export class TrieNodeFetcher extends Fetcher<JobTask, Uint8Array[], Uint8Array>
 
  // will always start with root node as first set of node requests
  this.pathToNodeRequestData.setElement('', {
+ requested: false,
  nodeHash: bytesToHex(this.root),
  nodeParentHash: '', // root node does not have a parent
  } as NodeRequestData)
@@ -263,6 +265,7 @@ export class TrieNodeFetcher extends Fetcher<JobTask, Uint8Array[], Uint8Array>
  storagePath,
  ].join('/')
  this.pathToNodeRequestData.setElement(syncPath, {
+ requested: false,
  nodeHash: bytesToHex(storageRoot),
  nodeParentHash: nodeHash,
  parentAccountHash: nodeHash,
@@ -299,6 +302,7 @@ export class TrieNodeFetcher extends Fetcher<JobTask, Uint8Array[], Uint8Array>
  pathString
  ) as NodeRequestData
  this.pathToNodeRequestData.setElement(childNode.path, {
+ requested: false,
  nodeHash: bytesToHex(childNode.nodeHash as Uint8Array),
  nodeParentHash: nodeHash, // TODO root node does not have a parent, so handle that in the leaf callback when checking if dependencies are met recursively
  parentAccountHash,
@@ -411,17 +415,25 @@ export class TrieNodeFetcher extends Fetcher<JobTask, Uint8Array[], Uint8Array>
  if (this.pathToNodeRequestData.size() > 0) {
  let { pathStrings } = this.getSortedPathStrings() // TODO pass in number of paths to return
  while (tasks.length < maxTasks && pathStrings.length > 0) {
- const requestedPathStrings = pathStrings.slice(0, max)
+ const pendingPathStrings = pathStrings.slice(0, max)
  pathStrings = pathStrings.slice(max + 1)
- for (const pathString of requestedPathStrings) {
- const nodeHash = this.pathToNodeRequestData.getElementByKey(pathString)?.nodeHash // TODO return node set too from sorted function and avoid lookups here
- if (nodeHash === undefined) throw Error('Path should exist')
- this.requestedNodeToPath.set(nodeHash as unknown as string, pathString)
+ const neededPathStrings = []
+ for (const pathString of pendingPathStrings) {
+ const requestData = this.pathToNodeRequestData.getElementByKey(pathString) // TODO return node set too from sorted function and avoid lookups here
+ if (requestData === undefined) throw Error('Path should exist')
+ if (requestData.requested === false) {
+ this.requestedNodeToPath.set(requestData.nodeHash as unknown as string, pathString)
+ this.pathToNodeRequestData.setElement(pathString, {
+ ...requestData,
+ requested: true,
+ })
+ neededPathStrings.push(pathString)
+ }
  }
  this.debug('At start of mergeAndFormatPaths')
- const paths = mergeAndFormatKeyPaths(requestedPathStrings) as unknown as Uint8Array[][]
+ const paths = mergeAndFormatKeyPaths(neededPathStrings) as unknown as Uint8Array[][]
  tasks.push({
- pathStrings: requestedPathStrings,
+ pathStrings: neededPathStrings,
  paths,
  })
  this.debug(`Created new tasks num=${tasks.length}`)
@@ -435,6 +447,10 @@ export class TrieNodeFetcher extends Fetcher<JobTask, Uint8Array[], Uint8Array>
  return tasks
  }
 
+ updateStateRoot(stateRoot: Uint8Array) {
+ this.root = stateRoot
+ }
+
  nextTasks(): void {
  try {
  if (this.in.length === 0) {

packages/client/test/sync/fetcher/trienodefetcher.spec.ts

@@ -214,6 +214,7 @@ describe('[TrieNodeFetcher]', async () => {
 
  fetcher.pathToNodeRequestData = new OrderedMap<string, any>()
  fetcher.pathToNodeRequestData.setElement('', {
+ requested: false,
  nodeHash: bytesToHex(new Uint8Array(0)),
  nodeParentHash: '',
  })