Finish SENDFINDCONTENT implementation (#440)

* Add content handling to network scaffolding * Add tests for storing and retrieving beacon content * Integration test scaffolding * Add UTP handling in sendFindContent * WIP * WIP * Finish test for finding content * Turn off logging * Fix method names * Add TODO * Naive changes * Fix npm scripts * fix leftover tape bits * Add history network to provider setup * put tests in util.spec suite * fix async * vitestify wire/types.spec.ts * vitestify history/util.spec * clear linting errors * put tests into suites in beacon.spec * resolve suite/test error in historyProtocol.spec * resolve vitest error in receiptManager.spec * vitestify socket.spec * put tests in suite in protocol.spec * put tests in suite - history/types.spec * Fix integration tests * Fix various imports * lint fixes * Fix parens * Fix import paths * Add more content handling * update package-lock * Add initial RPC stuff for Beacon network * extend gossip test timeout * Fix timeout again * Remove unnecessary timeout hack --------- Co-authored-by: ScottyPoi <[email protected]>
ethereumjs · Aug 22, 2023 · 1ab4651 · 1ab4651
1 parent 4f01afa
commit 1ab4651
Show file tree

Hide file tree

Showing 44 changed files with 2,415 additions and 1,310 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -15,15 +15,13 @@
   "scripts": {
     "preinstall": "npm run checkNpmVersion",
     "checkNpmVersion": "bash ./scripts/check-npm-version.sh",
-    "postinstall": "npm run tapeHotFix && npm run build --workspaces --if-present",
+    "postinstall": "npm run build --workspaces --if-present",
     "start-proxy": "npm run start -w=proxy -- --nat=localhost",
     "start-browser-client": "npm run start-testnet -w=browser-client",
     "start-cli": "npm run dev -w=cli",
     "lint": "npm run lint -w=cli -w=portalnetwork -w=proxy",
     "lint:fix": "npm run lint:fix -w=cli -w=portalnetwork -w=proxy",
-    "clean": "bash ./scripts/clean-root.sh",
-    "tapeHotFix": "sed -i -e 's/js/ts/g' node_modules/tape/bin/import-or-require.js",
-    "ethjsHotFix": "sed -i  '/from \\S\\+\\.json'\\''/ s/;//;/from \\S\\+\\.json/ s/.*/& assert {type: \"json\"};/' node_modules/@ethereumjs/**/dist/esm/**/*.js"
+    "clean": "bash ./scripts/clean-root.sh"
   },
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^5.51.0",

diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts
@@ -115,10 +115,11 @@ const main = async () => {
     //@ts-ignore Because level doesn't know how to get along with itself
     db,
     metrics,
-    supportedProtocols: [ProtocolId.HistoryNetwork],
+    supportedProtocols: [ProtocolId.HistoryNetwork, ProtocolId.BeaconLightClientNetwork],
     dataDir: args.datadir,
   })
   portal.discv5.enableLogs()
+
   portal.enableLog('*ultralight*, *Portal*, *ultralight:RPC*')
   let metricsServer: http.Server | undefined
 
@@ -132,6 +133,7 @@ const main = async () => {
   }
   await portal.start()
 
+  // TODO - make this more intelligent
   const protocol = portal.protocols.get(ProtocolId.HistoryNetwork)
   if (args.bootnode) {
     protocol!.addBootNode(args.bootnode)

diff --git a/packages/cli/src/rpc/modules/portal.ts b/packages/cli/src/rpc/modules/portal.ts
@@ -19,6 +19,8 @@ import {
   ContentMessageType,
   AcceptMessage,
   decodeHistoryNetworkContentKey,
+  BeaconLightClientNetwork,
+  BeaconLightClientNetworkContentType,
 } from 'portalnetwork'
 import { GetEnrResult } from '../schema/types.js'
 import { isValidId } from '../util.js'
@@ -47,6 +49,8 @@ const methods = [
   'portal_historyStore',
   'portal_historyLocalContent',
   'portal_historyGossip',
+  'portal_beaconSendFindContent',
+  'portal_beaconStore',
 
   // not included in portal-network-specs
   'portal_historyAddEnrs',
@@ -57,11 +61,15 @@ const methods = [
 export class portal {
   private _client: PortalNetwork
   private _history: HistoryProtocol
+  private _beacon: BeaconLightClientNetwork
   private logger: Debugger
 
   constructor(client: PortalNetwork, logger: Debugger) {
     this._client = client
     this._history = this._client.protocols.get(ProtocolId.HistoryNetwork) as HistoryProtocol
+    this._beacon = this._client.protocols.get(
+      ProtocolId.BeaconLightClientNetwork,
+    ) as BeaconLightClientNetwork
     this.logger = logger
     this.methods = middleware(this.methods.bind(this), 0, [])
     this.historyNodeInfo = middleware(this.historyNodeInfo.bind(this), 0, [])
@@ -132,6 +140,15 @@ export class portal {
       [validators.contentKey],
       [validators.hex],
     ])
+    this.beaconSendFindContent = middleware(this.beaconSendFindContent.bind(this), 2, [
+      [validators.dstId],
+      [validators.hex],
+    ])
+
+    this.beaconStore = middleware(this.beaconStore.bind(this), 2, [
+      [validators.hex],
+      [validators.hex],
+    ])
   }
   async methods() {
     return methods
@@ -512,4 +529,27 @@ export class portal {
       return false
     }
   }
+
+  async beaconSendFindContent(params: [string, string]) {
+    const [nodeId, contentKey] = params
+    console.log(nodeId)
+    const res = await this._beacon.sendFindContent(nodeId, fromHexString(contentKey))
+    console.log(res)
+    const enr = this._beacon.routingTable.getWithPending(nodeId)?.value
+    return res && enr && '0x' + enr.seq.toString(16)
+  }
+
+  async beaconStore(params: [string, string]) {
+    const [contentKey, content] = params.map((param) => fromHexString(param))
+    try {
+      await this._beacon.store(
+        contentKey[0] as BeaconLightClientNetworkContentType,
+        toHexString(contentKey.slice(1)),
+        content,
+      )
+      return true
+    } catch {
+      return false
+    }
+  }
 }
diff --git a/packages/portalnetwork/.eslintignore b/packages/portalnetwork/.eslintignore
@@ -1 +1,2 @@
-/types/*
+/types/*
+vitest.config.unit.ts
diff --git a/packages/portalnetwork/package.json b/packages/portalnetwork/package.json
@@ -11,10 +11,8 @@
     "dev": "tsc --watch",
     "build": "tsc",
     "docs": "typedoc src/index.ts",
-    "tape": "NODE_OPTIONS='--loader ts-node/esm --experimental-json-modules' tape",
-    "test": "npm run tape -- 'test/!(integration)/**/*.spec.ts'",
-    "test:all": "npm run tape -- 'test/**/*.spec.ts'",
-    "test:integration": "npm run tape -- 'test/integration/**/*.spec.ts'",
+    "test": "npx vitest run test/* -c=./vitest.config.unit.ts",
+    "test:integration": "npx vitest run ./test/integration/*.spec.ts",
     "coverage": "c8 npm run test",
     "coverage:all": "c8 npm run test:all",
     "lint": "../../config/cli/lint.sh",
@@ -60,17 +58,17 @@
     "@ethereumjs/tx": "5.0.0",
     "@ethereumjs/util": "9.0.0",
     "@types/debug": "^4.1.7",
-    "@types/tape": "^4.13.2",
     "@types/ws": "^7.4.7",
+    "@vitest/ui": "^0.34.1",
     "c8": "^7.12.0",
     "eslint": "^8.6.0",
     "prettier": "^2.5.1",
-    "tape": "^5.5.3",
     "testdouble": "^3.16.3",
     "ts-node": "^10.4.0",
     "tslib": "^2.3.1",
     "typedoc": "^0.24.0",
     "typedoc-plugin-markdown": "^3.11.3",
-    "typescript": "^4.4.2"
+    "typescript": "^4.4.2",
+    "vitest": "^0.34.1"
   }
 }
diff --git a/packages/portalnetwork/src/client/client.ts b/packages/portalnetwork/src/client/client.ts
@@ -10,7 +10,7 @@ import { ITalkReqMessage, ITalkRespMessage } from '@chainsafe/discv5/message'
 import { EventEmitter } from 'events'
 import debug, { Debugger } from 'debug'
 import { fromHexString, toHexString } from '@chainsafe/ssz'
-import { BeaconLightClientNetwork, ProtocolId, StateProtocol } from '../subprotocols/index.js'
+import { BeaconLightClientNetwork, StateProtocol } from '../subprotocols/index.js'
 import {
   PortalNetworkEventEmitter,
   PortalNetworkMetrics,
@@ -21,7 +21,7 @@ import type { PeerId, Secp256k1PeerId } from '@libp2p/interface-peer-id'
 import { createSecp256k1PeerId } from '@libp2p/peer-id-factory'
 import { INodeAddress } from '@chainsafe/discv5/lib/session/nodeInfo.js'
 import { PortalNetworkUTP } from '../wire/utp/PortalNetworkUtp/index.js'
-
+import { ProtocolId } from '../types.js'
 import { BaseProtocol } from '../subprotocols/protocol.js'
 import { HistoryProtocol } from '../subprotocols/history/history.js'
 import { Multiaddr, multiaddr } from '@multiformats/multiaddr'

diff --git a/packages/portalnetwork/src/client/dbManager.ts b/packages/portalnetwork/src/client/dbManager.ts
@@ -4,7 +4,7 @@ import { AbstractBatchOperation, AbstractLevel } from 'abstract-level'
 import { Debugger } from 'debug'
 import { MemoryLevel } from 'memory-level'
 import { fromHexString, serializedContentKeyToContentId } from '../index.js'
-import { ProtocolId } from '../subprotocols/index.js'
+import { ProtocolId } from '../index.js'
 
 export class DBManager {
   nodeId: string

diff --git a/packages/portalnetwork/src/client/provider.ts b/packages/portalnetwork/src/client/provider.ts
@@ -1,5 +1,6 @@
 import { ethers } from 'ethers'
-import { addRLPSerializedBlock, HistoryProtocol, ProtocolId } from '../subprotocols/index.js'
+import { addRLPSerializedBlock, HistoryProtocol } from '../subprotocols/index.js'
+import { ProtocolId } from '../types.js'
 import { toHexString } from '../util/discv5.js'
 import {
   ethJsBlockToEthersBlock,

diff --git a/packages/portalnetwork/src/index.ts b/packages/portalnetwork/src/index.ts
@@ -5,3 +5,4 @@ export * from './util/index.js'
 export * from './transports/index.js'
 export * from './subprotocols/index.js'
 export * from './client/index.js'
+export * from './types.js'
diff --git a/packages/portalnetwork/src/subprotocols/beacon/beacon.ts b/packages/portalnetwork/src/subprotocols/beacon/beacon.ts
@@ -1,6 +1,7 @@
 import { Debugger } from 'debug'
 import { BaseProtocol } from '../protocol.js'
-import { FoundContent, ProtocolId } from '../types.js'
+import { FoundContent } from '../types.js'
+import { ProtocolId } from '../../types.js'
 import { PortalNetwork } from '../../client/client.js'
 import debug from 'debug'
 import { Union } from '@chainsafe/ssz/lib/interface.js'
@@ -18,11 +19,9 @@ import {
   MessageCodes,
   PortalWireMessageType,
 } from '../../wire/types.js'
-import { ssz } from '@lodestar/types'
-import { getBeaconContentKey } from './util.js'
 import { bytesToInt } from '@ethereumjs/util'
 import { RequestCode } from '../../wire/index.js'
-
+import { ssz } from '@lodestar/types'
 export class BeaconLightClientNetwork extends BaseProtocol {
   protocolId: ProtocolId.BeaconLightClientNetwork
   beaconConfig: BeaconConfig
@@ -97,83 +96,88 @@ export class BeaconLightClientNetwork extends BaseProtocol {
             })
             break
           }
-        } /*
-        const contentHash = toHexString(key)
-        const forkhash = decoded.value.slice(0, 4) as Uint8Array
-        const forkname = this.beaconConfig.forkDigest2ForkName(forkhash) as any
-        switch (decoded.selector) {
-          case BeaconLightClientNetworkContentType.LightClientOptimisticUpdate:
-            try {
-              // TODO: Figure out how to use Forks type to limit selector in ssz[forkname] below and make typescript happy
-              ;(ssz as any)[forkname].LightClientOptimisticUpdate.deserialize(
-                (decoded.value as Uint8Array).slice(4),
-              )
-            } catch (err) {
-              this.logger(`received invalid content from ${shortId(dstId)}`)
-              break
-            }
-            this.logger(
-              `received ${
-                BeaconLightClientNetworkContentType[decoded.selector]
-              } content corresponding to ${contentHash}`,
-            )
-            await this.store(decoded.selector, contentHash, decoded.value as Uint8Array)
-            break
-          case BeaconLightClientNetworkContentType.LightClientFinalityUpdate:
-            try {
-              ;(ssz as any)[forkname].LightClientFinalityUpdate.deserialize(
-                (decoded.value as Uint8Array).slice(4),
-              )
-            } catch (err) {
-              this.logger(`received invalid content from ${shortId(dstId)}`)
-              break
-            }
-            this.logger(
-              `received ${
-                BeaconLightClientNetworkContentType[decoded.selector]
-              } content corresponding to ${contentHash}`,
-            )
-            await this.store(decoded.selector, contentHash, decoded.value as Uint8Array)
-            break
-          case BeaconLightClientNetworkContentType.LightClientBootstrap:
-            try {
-              ;(ssz as any)[forkname].LightClientBootstrap.deserialize(
-                (decoded.value as Uint8Array).slice(4),
-              )
-            } catch (err) {
-              this.logger(`received invalid content from ${shortId(dstId)}`)
-              break
-            }
-            this.logger(
-              `received ${
-                BeaconLightClientNetworkContentType[decoded.selector]
-              } content corresponding to ${contentHash}`,
-            )
-            await this.store(decoded.selector, contentHash, decoded.value as Uint8Array)
-            break
-          case BeaconLightClientNetworkContentType.LightClientUpdatesByRange:
-            try {
-              LightClientUpdatesByRange.deserialize((decoded.value as Uint8Array).slice(4))
-            } catch (err) {
-              this.logger(`received invalid content from ${shortId(dstId)}`)
-              break
+          case FoundContent.CONTENT: {
+            const contentKey = toHexString(key)
+            const forkhash = decoded.value.slice(0, 4) as Uint8Array
+            const forkname = this.beaconConfig.forkDigest2ForkName(forkhash) as any
+            switch (key[0]) {
+              case BeaconLightClientNetworkContentType.LightClientOptimisticUpdate:
+                try {
+                  // TODO: Figure out how to use Forks type to limit selector in ssz[forkname] below and make typescript happy
+                  ;(ssz as any)[forkname].LightClientOptimisticUpdate.deserialize(
+                    (decoded.value as Uint8Array).slice(4),
+                  )
+                } catch (err) {
+                  this.logger(`received invalid content from ${shortId(dstId)}`)
+                  break
+                }
+                this.logger(
+                  `received ${
+                    BeaconLightClientNetworkContentType[decoded.selector]
+                  } content corresponding to ${contentKey}`,
+                )
+                await this.store(key[0], contentKey, decoded.value as Uint8Array)
+                break
+              case BeaconLightClientNetworkContentType.LightClientFinalityUpdate:
+                try {
+                  ;(ssz as any)[forkname].LightClientFinalityUpdate.deserialize(
+                    (decoded.value as Uint8Array).slice(4),
+                  )
+                } catch (err) {
+                  this.logger(`received invalid content from ${shortId(dstId)}`)
+                  break
+                }
+                this.logger(
+                  `received ${
+                    BeaconLightClientNetworkContentType[decoded.selector]
+                  } content corresponding to ${contentKey}`,
+                )
+                await this.store(key[0], contentKey, decoded.value as Uint8Array)
+                break
+              case BeaconLightClientNetworkContentType.LightClientBootstrap:
+                try {
+                  ;(ssz as any)[forkname].LightClientBootstrap.deserialize(
+                    (decoded.value as Uint8Array).slice(4),
+                  )
+                } catch (err) {
+                  this.logger(`received invalid content from ${shortId(dstId)}`)
+                  break
+                }
+                this.logger(
+                  `received ${
+                    BeaconLightClientNetworkContentType[decoded.selector]
+                  } content corresponding to ${contentKey}`,
+                )
+                await this.store(key[0], contentKey, decoded.value as Uint8Array)
+                break
+              case BeaconLightClientNetworkContentType.LightClientUpdatesByRange:
+                try {
+                  LightClientUpdatesByRange.deserialize((decoded.value as Uint8Array).slice(4))
+                } catch (err) {
+                  this.logger(`received invalid content from ${shortId(dstId)}`)
+                  break
+                }
+                this.logger(
+                  `received ${
+                    BeaconLightClientNetworkContentType[decoded.selector]
+                  } content corresponding to ${contentKey}`,
+                )
+                await this.store(key[0], contentKey, decoded.value as Uint8Array)
+                break
+
+              default:
+                this.logger(
+                  `received ${
+                    BeaconLightClientNetworkContentType[decoded.selector]
+                  } content corresponding to ${contentKey}`,
+                )
+                break
             }
-            this.logger(
-              `received ${
-                BeaconLightClientNetworkContentType[decoded.selector]
-              } content corresponding to ${contentHash}`,
-            )
-            await this.store(decoded.selector, contentHash, decoded.value as Uint8Array)
-            break
-          default:
-            this.logger(
-              `received ${
-                BeaconLightClientNetworkContentType[decoded.selector]
-              } content corresponding to ${contentHash}`,
-            )
-        }*/
+          }
+        }
         return decoded
       }
+      // TODO Should we do anything other than ignore responses to FINDCONTENT messages that isn't a CONTENT response?
     } catch (err: any) {
       this.logger(`Error sending FINDCONTENT to ${shortId(dstId)} - ${err.message}`)
     }
@@ -188,6 +192,9 @@ export class BeaconLightClientNetwork extends BaseProtocol {
     if (contentType === BeaconLightClientNetworkContentType.LightClientUpdatesByRange) {
       throw new Error('special handling for update ranges not supported yet')
     }
+    this.logger(
+      `storing ${BeaconLightClientNetworkContentType[contentType]} content corresponding to ${contentKey}`,
+    )
     await this.put(this.protocolId, contentKey, toHexString(value))
   }
 }