Nimbus light client integration with status-go

.gitmodules

@@ -65,7 +65,7 @@
   branch = master
 [submodule "vendor/nim-confutils"]
   path = vendor/nim-confutils
-  url = https://github.com/status-im/nim-confutils.git
+	url = https://github.com/siphiuel/nim-confutils.git
   ignore = dirty
   branch = master
 [submodule "vendor/nim-blscurve"]

Makefile

@@ -70,6 +70,7 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 	nimbus \
 	fluffy \
 	nimbus_verified_proxy \
+	libverifproxy \
 	test \
 	test-reproducibility \
 	clean \
@@ -272,9 +273,31 @@ evmstate_test: | build deps evmstate
 txparse: | build deps
 	$(ENV_SCRIPT) nim c $(NIM_PARAMS) "tools/txparse/$@.nim"
 
+# Shared library for verified proxy
+OS = $(shell $(CC) -dumpmachine)
+ifneq (, $(findstring darwin, $(OS)))
+  SHAREDLIBEXT = dylib
+else
+ifneq (, $(findstring mingw, $(OS))$(findstring cygwin, $(OS))$(findstring msys, $(OS)))
+  SHAREDLIBEXT = dll
+else
+  SHAREDLIBEXT = so
+endif
+endif
+
+VERIF_PROXY_OUT_PATH ?= build/libverifproxy/
+
+libverifproxy: | build deps
+	+ echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim --version && \
+		$(ENV_SCRIPT) nim c --app:lib -d:"libp2p_pki_schemes=secp256k1" --gc:boehm --header:verifproxy.h --noMain:on --threads:on --nimcache:nimcache/libverifproxy -o:$(VERIF_PROXY_OUT_PATH)/$@.$(SHAREDLIBEXT) $(NIM_PARAMS) nimbus_verified_proxy/nimbus_verified_proxy.nim
+	cp nimcache/libverifproxy/verifproxy.h $(VERIF_PROXY_OUT_PATH)/
+	cp vendor/nimbus-build-system/vendor/Nim-csources-v1/c_code/nimbase.h $(VERIF_PROXY_OUT_PATH)/
+	echo -e $(BUILD_END_MSG) "build/$@"
+
 # usual cleaning
 clean: | clean-common
-	rm -rf build/{nimbus,fluffy,nimbus_verified_proxy,$(TOOLS_CSV),all_tests,test_kvstore_rocksdb,test_rpc,all_fluffy_tests,all_fluffy_portal_spec_tests,test_portal_testnet,portalcli,blockwalk,eth_data_exporter,utp_test_app,utp_test,*.dSYM}
+	rm -rf build/{nimbus,fluffy,libverifproxy,nimbus_verified_proxy,$(TOOLS_CSV),all_tests,test_kvstore_rocksdb,test_rpc,all_fluffy_tests,all_fluffy_portal_spec_tests,test_portal_testnet,portalcli,blockwalk,eth_data_exporter,utp_test_app,utp_test,*.dSYM}
 	rm -rf tools/t8n/{t8n,t8n_test}
 	rm -rf tools/evmstate/{evmstate,evmstate_test}
 ifneq ($(USE_LIBBACKTRACE), 0)

nimbus_verified_proxy/nimbus_verified_proxy.nim

@@ -8,7 +8,7 @@
 {.push raises: [].}
 
 import
-  std/[os, strutils],
+  std/[json, os, strutils],
   chronicles, chronicles/chronos_tools, chronos, confutils,
   eth/keys,
   json_rpc/rpcproxy,
@@ -40,14 +40,30 @@ func getConfiguredChainId(networkMetadata: Eth2NetworkMetadata): Quantity =
   else:
     return networkMetadata.cfg.DEPOSIT_CHAIN_ID.Quantity
 
+type OnHeaderCallback* = proc (s: cstring) {.cdecl, raises: [], gcsafe.}
+
+var optimisticHeaderCallback : OnHeaderCallback = nil
+var finalizedHeaderCallback : OnHeaderCallback = nil
+proc setOptimisticHeaderCallback*(cb: OnHeaderCallback) {.exportc, dynlib.} =
+  optimisticHeaderCallback = cb
+  echo "optimistic header callback set"
+
+proc setFinalizedHeaderCallback*(cb: OnHeaderCallback) {.exportc, dynlib.} =
+  finalizedHeaderCallback = cb
+  echo "finalized header callback set"
+
+
 proc run(config: VerifiedProxyConf) {.raises: [CatchableError].} =
   # Required as both Eth2Node and LightClient requires correct config type
   var lcConfig = config.asLightClientConf()
 
   setupLogging(config.logLevel, config.logStdout, none(OutFile))
 
-  notice "Launching Nimbus verified proxy",
-    version = fullVersionStr, cmdParams = commandLineParams(), config
+  try:
+    notice "Launching Nimbus verified proxy",
+      version = fullVersionStr, cmdParams = getCLIParams(), config
+  except Exception:
+    notice "getCLIParams() exception"
 
   let
     metadata = loadEth2Network(config.eth2Network)
@@ -163,6 +179,13 @@ proc run(config: VerifiedProxyConf) {.raises: [CatchableError].} =
       when lcDataFork > LightClientDataFork.None:
         info "New LC finalized header",
           finalized_header = shortLog(forkyHeader)
+        if finalizedHeaderCallback != nil:
+          notice "### Invoking finalizedHeaderCallback"
+          try:
+            finalizedHeaderCallback(Json.encode(finalizedHeader))
+          except SerializationError as e:
+            notice "finalizedHeaderCallback exception"
+
 
   proc onOptimisticHeader(
       lightClient: LightClient, optimisticHeader: ForkedLightClientHeader) =
@@ -171,6 +194,13 @@ proc run(config: VerifiedProxyConf) {.raises: [CatchableError].} =
         info "New LC optimistic header",
           optimistic_header = shortLog(forkyHeader)
         optimisticProcessor.setOptimisticHeader(forkyHeader.beacon)
+      if optimisticHeaderCallback != nil:
+        try:
+          notice "### Invoking optimisticHeaderCallback"
+          optimisticHeaderCallback(Json.encode(optimisticHeader))
+        except SerializationError as e:
+            notice "optimisticHeaderCallback exception"
+
 
   lightClient.onFinalizedHeader = onFinalizedHeader
   lightClient.onOptimisticHeader = onOptimisticHeader
@@ -249,10 +279,42 @@ proc run(config: VerifiedProxyConf) {.raises: [CatchableError].} =
   while true:
     poll()
 
-when isMainModule:
+proc quit*() {.exportc, dynlib.} = 
+  echo "Quitting"
+
+proc NimMain() {.importc.}
+proc startProxyViaJson*(configJson: cstring) {.exportc, dynlib.} =
+  NimMain()
+  let str = $configJson
+  try:
+    let jsonNode = parseJson(str)
+
+    let rpcAddr = jsonNode["RpcAddress"].getStr()
+    let config = VerifiedProxyConf(
+      rpcAddress: ValidIpAddress.init(rpcAddr), 
+      listenAddress: defaultListenAddress, 
+      eth2Network: some(jsonNode["Eth2Network"].getStr()), 
+      trustedBlockRoot: Eth2Digest.fromHex(jsonNode["TrustedBlockRoot"].getStr()),
+      web3Url: parseCmdArg(Web3Url, jsonNode["Web3Url"].getStr()),
+      rpcPort: Port(jsonNode["RpcPort"].getInt()),
+      logLevel: jsonNode["LogLevel"].getStr(),
+      maxPeers: 160,
+      nat: NatConfig(hasExtIp: false, nat: NatAny),
+      logStdout: StdoutLogKind.Auto,
+      dataDir: OutDir(defaultVerifiedProxyDataDir()),
+      tcpPort: Port(defaultEth2TcpPort),
+      udpPort: Port(defaultEth2TcpPort),
+      agentString: "nimbus",
+      discv5Enabled: true,
+    )
+
+    run(config)
+  except Exception as err:
+    echo "Exception when running ", getCurrentExceptionMsg(), err.getStackTrace() 
+
+when isMainModule and appType != "lib":
   {.pop.}
   var config = makeBannerAndConfig(
     "Nimbus verified proxy " & fullVersionStr, VerifiedProxyConf)
   {.push raises: [].}
-
   run(config)