Db store (#247)

* move files * stores * implement most of the methods * implement read all * link everything * fix tests * can't get upper bound working * remove useless code * add store test * fix * test with rocksdb and many issue fixes * avoid JamCoder overhead in most of the stores * fix codec * fix warnings * update tests
open-web3-stack · Dec 11, 2024 · 80e6ed0 · 80e6ed0
1 parent ccaedb9
commit 80e6ed0
Show file tree

Hide file tree

Showing 34 changed files with 1,232 additions and 185 deletions.
diff --git a/.gitignore b/.gitignore
@@ -26,3 +26,5 @@ settings.json
 c_cpp_properties.json
 
 Tools/openrpc.json
+
+tmp
diff --git a/Blockchain/Sources/Blockchain/BlockchainDataProvider/BlockchainDataProvider.swift b/Blockchain/Sources/Blockchain/BlockchainDataProvider/BlockchainDataProvider.swift
@@ -9,6 +9,11 @@ public struct HeadInfo: Sendable {
     public var number: UInt32
 }
 
+public enum BlockchainDataProviderError: Error, Equatable {
+    case noData(hash: Data32)
+    case uncanonical(hash: Data32)
+}
+
 public actor BlockchainDataProvider: Sendable {
     public private(set) var bestHead: HeadInfo
     public private(set) var finalizedHead: HeadInfo
@@ -18,21 +23,21 @@ public actor BlockchainDataProvider: Sendable {
         let heads = try await dataProvider.getHeads()
         var bestHead = HeadInfo(hash: dataProvider.genesisBlockHash, timeslot: 0, number: 0)
         for head in heads {
-            let header = try await dataProvider.getHeader(hash: head)
+            let header = try await dataProvider.getHeader(hash: head).unwrap()
             if header.value.timeslot > bestHead.timeslot {
-                let number = try await dataProvider.getBlockNumber(hash: head)
+                let number = try await dataProvider.getBlockNumber(hash: head).unwrap()
                 bestHead = HeadInfo(hash: head, timeslot: header.value.timeslot, number: number)
             }
         }
 
         self.bestHead = bestHead
 
-        let finalizedHeadHash = try await dataProvider.getFinalizedHead()
+        let finalizedHeadHash = try await dataProvider.getFinalizedHead().unwrap()
 
         finalizedHead = try await HeadInfo(
             hash: finalizedHeadHash,
-            timeslot: dataProvider.getHeader(hash: finalizedHeadHash).value.timeslot,
-            number: dataProvider.getBlockNumber(hash: finalizedHeadHash)
+            timeslot: dataProvider.getHeader(hash: finalizedHeadHash).unwrap().value.timeslot,
+            number: dataProvider.getBlockNumber(hash: finalizedHeadHash).unwrap()
         )
 
         self.dataProvider = dataProvider
@@ -44,7 +49,7 @@ public actor BlockchainDataProvider: Sendable {
         try await dataProvider.updateHead(hash: block.hash, parent: block.header.parentHash)
 
         if block.header.timeslot > bestHead.timeslot {
-            let number = try await dataProvider.getBlockNumber(hash: block.hash)
+            let number = try await getBlockNumber(hash: block.hash)
             bestHead = HeadInfo(hash: block.hash, timeslot: block.header.timeslot, number: number)
         }
 
@@ -66,19 +71,19 @@ extension BlockchainDataProvider {
     }
 
     public func getBlockNumber(hash: Data32) async throws -> UInt32 {
-        try await dataProvider.getBlockNumber(hash: hash)
+        try await dataProvider.getBlockNumber(hash: hash).unwrap(orError: BlockchainDataProviderError.noData(hash: hash))
     }
 
     public func getHeader(hash: Data32) async throws -> HeaderRef {
-        try await dataProvider.getHeader(hash: hash)
+        try await dataProvider.getHeader(hash: hash).unwrap(orError: BlockchainDataProviderError.noData(hash: hash))
     }
 
     public func getBlock(hash: Data32) async throws -> BlockRef {
-        try await dataProvider.getBlock(hash: hash)
+        try await dataProvider.getBlock(hash: hash).unwrap(orError: BlockchainDataProviderError.noData(hash: hash))
     }
 
     public func getState(hash: Data32) async throws -> StateRef {
-        try await dataProvider.getState(hash: hash)
+        try await dataProvider.getState(hash: hash).unwrap(orError: BlockchainDataProviderError.noData(hash: hash))
     }
 
     public func getHeads() async throws -> Set<Data32> {
@@ -122,7 +127,7 @@ extension BlockchainDataProvider {
         logger.debug("setting finalized head: \(hash)")
 
         let oldFinalizedHead = finalizedHead
-        let number = try await dataProvider.getBlockNumber(hash: hash)
+        let number = try await getBlockNumber(hash: hash)
 
         var hashToCheck = hash
         var hashToCheckNumber = number
@@ -132,11 +137,11 @@ extension BlockchainDataProvider {
                 logger.trace("purge block: \(hash)")
                 try await dataProvider.remove(hash: hash)
             }
-            hashToCheck = try await dataProvider.getHeader(hash: hashToCheck).value.parentHash
+            hashToCheck = try await getHeader(hash: hashToCheck).value.parentHash
             hashToCheckNumber -= 1
         }
 
-        let header = try await dataProvider.getHeader(hash: hash)
+        let header = try await getHeader(hash: hash)
         finalizedHead = HeadInfo(hash: hash, timeslot: header.value.timeslot, number: number)
         try await dataProvider.setFinalizedHead(hash: hash)
     }
@@ -152,6 +157,6 @@ extension BlockchainDataProvider {
     }
 
     public func getBestState() async throws -> StateRef {
-        try await dataProvider.getState(hash: bestHead.hash)
+        try await dataProvider.getState(hash: bestHead.hash).unwrap(orError: BlockchainDataProviderError.noData(hash: bestHead.hash))
     }
 }
diff --git a/Blockchain/Sources/Blockchain/BlockchainDataProvider/BlockchainDataProviderProtocol.swift b/Blockchain/Sources/Blockchain/BlockchainDataProvider/BlockchainDataProviderProtocol.swift
@@ -1,28 +1,19 @@
 import Utils
 
-public enum BlockchainDataProviderError: Error, Equatable {
-    case noData(hash: Data32)
-    case uncanonical(hash: Data32)
-}
-
 public protocol BlockchainDataProviderProtocol: Sendable {
     func hasBlock(hash: Data32) async throws -> Bool
     func hasState(hash: Data32) async throws -> Bool
     func isHead(hash: Data32) async throws -> Bool
 
-    func getBlockNumber(hash: Data32) async throws -> UInt32
+    func getBlockNumber(hash: Data32) async throws -> UInt32?
 
-    /// throw BlockchainDataProviderError.noData if not found
-    func getHeader(hash: Data32) async throws -> HeaderRef
+    func getHeader(hash: Data32) async throws -> HeaderRef?
 
-    /// throw BlockchainDataProviderError.noData if not found
-    func getBlock(hash: Data32) async throws -> BlockRef
+    func getBlock(hash: Data32) async throws -> BlockRef?
 
-    /// throw BlockchainDataProviderError.noData if not found
-    func getState(hash: Data32) async throws -> StateRef
+    func getState(hash: Data32) async throws -> StateRef?
 
-    /// throw BlockchainDataProviderError.noData if not found
-    func getFinalizedHead() async throws -> Data32
+    func getFinalizedHead() async throws -> Data32?
     func getHeads() async throws -> Set<Data32>
 
     /// return empty set if not found

diff --git a/Blockchain/Sources/Blockchain/BlockchainDataProvider/InMemoryDataProvider.swift b/Blockchain/Sources/Blockchain/BlockchainDataProvider/InMemoryDataProvider.swift
@@ -34,35 +34,23 @@ extension InMemoryDataProvider: BlockchainDataProviderProtocol {
         heads.contains(hash)
     }
 
-    public func getBlockNumber(hash: Data32) async throws -> UInt32 {
-        guard let number = numberByHash[hash] else {
-            throw BlockchainDataProviderError.noData(hash: hash)
-        }
-        return number
+    public func getBlockNumber(hash: Data32) async throws -> UInt32? {
+        numberByHash[hash]
     }
 
-    public func getHeader(hash: Data32) throws -> HeaderRef {
-        guard let header = blockByHash[hash]?.header.asRef() else {
-            throw BlockchainDataProviderError.noData(hash: hash)
-        }
-        return header
+    public func getHeader(hash: Data32) throws -> HeaderRef? {
+        blockByHash[hash]?.header.asRef()
     }
 
-    public func getBlock(hash: Data32) throws -> BlockRef {
-        guard let block = blockByHash[hash] else {
-            throw BlockchainDataProviderError.noData(hash: hash)
-        }
-        return block
+    public func getBlock(hash: Data32) throws -> BlockRef? {
+        blockByHash[hash]
     }
 
-    public func getState(hash: Data32) throws -> StateRef {
-        guard let state = stateByBlockHash[hash] else {
-            throw BlockchainDataProviderError.noData(hash: hash)
-        }
-        return state
+    public func getState(hash: Data32) throws -> StateRef? {
+        stateByBlockHash[hash]
     }
 
-    public func getFinalizedHead() -> Data32 {
+    public func getFinalizedHead() -> Data32? {
         finalizedHead
     }
 

diff --git a/Blockchain/Sources/Blockchain/RuntimeProtocols/Runtime.swift b/Blockchain/Sources/Blockchain/RuntimeProtocols/Runtime.swift
@@ -14,7 +14,7 @@ public final class Runtime {
         case invalidReportAuthorizer
         case encodeError(any Swift.Error)
         case invalidExtrinsicHash
-        case invalidParentHash
+        case invalidParentHash(state: Data32, header: Data32)
         case invalidHeaderStateRoot
         case invalidHeaderEpochMarker
         case invalidHeaderWinningTickets
@@ -53,7 +53,7 @@ public final class Runtime {
         let block = block.value
 
         guard block.header.parentHash == state.value.lastBlockHash else {
-            throw Error.invalidParentHash
+            throw Error.invalidParentHash(state: state.value.lastBlockHash, header: block.header.parentHash)
         }
 
         guard block.header.priorStateRoot == context.stateRoot else {
@@ -189,6 +189,8 @@ public final class Runtime {
 
             // after reports as it need old recent history
             try updateRecentHistory(block: block, state: &newState)
+
+            try await newState.save()
         } catch let error as Error {
             throw error
         } catch let error as SafroleError {

diff --git a/Blockchain/Sources/Blockchain/State/State.swift b/Blockchain/Sources/Blockchain/State/State.swift
@@ -220,6 +220,7 @@ public struct State: Sendable {
     // TODO: we don't really want to write to the underlying backend here
     // instead, it should be writting to a in memory layer
     // and when actually saving the state, save the in memory layer to the presistent store
+    @discardableResult
     public func save() async throws -> Data32 {
         try await backend.write(layer.toKV())
         return await backend.rootHash

diff --git a/Blockchain/Sources/Blockchain/State/StateBackend.swift b/Blockchain/Sources/Blockchain/State/StateBackend.swift
@@ -3,7 +3,7 @@ import Foundation
 import Utils
 
 public enum StateBackendError: Error {
-    case missingState
+    case missingState(key: Sendable)
     case invalidData
 }
 
@@ -35,7 +35,7 @@ public final class StateBackend: Sendable {
         if Key.optional {
             return nil
         }
-        throw StateBackendError.missingState
+        throw StateBackendError.missingState(key: key)
     }
 
     public func batchRead(_ keys: [any StateKey]) async throws -> [(key: any StateKey, value: (Codable & Sendable)?)] {
@@ -74,4 +74,8 @@ public final class StateBackend: Sendable {
             return nil
         }
     }
+
+    public func debugPrint() async throws {
+        try await trie.debugPrint()
+    }
 }
diff --git a/Blockchain/Sources/Blockchain/State/StateTrie.swift b/Blockchain/Sources/Blockchain/State/StateTrie.swift
@@ -24,7 +24,7 @@ private struct TrieNode {
         right = Data32(data.data.suffix(32))!
         self.isNew = isNew
         rawValue = nil
-        switch data.data[0] & 0b1100_0000 {
+        switch data.data.first! & 0b1100_0000 {
         case 0b1000_0000:
             type = .embeddedLeaf
         case 0b1100_0000:
@@ -66,7 +66,7 @@ private struct TrieNode {
         guard type == .embeddedLeaf else {
             return nil
         }
-        let len = left.data[0] & 0b0011_1111
+        let len = left.data.first! & 0b0011_1111
         return right.data[relative: 0 ..< Int(len)]
     }
 
@@ -85,7 +85,7 @@ private struct TrieNode {
 
     static func branch(left: Data32, right: Data32) -> TrieNode {
         var left = left.data
-        left[0] = left[0] & 0b0111_1111 // clear the highest bit
+        left[left.startIndex] = left[left.startIndex] & 0b0111_1111 // clear the highest bit
         return .init(left: Data32(left)!, right: right, type: .branch, isNew: true, rawValue: nil)
     }
 }
@@ -352,6 +352,7 @@ public actor StateTrie: Sendable {
             }
         }
 
+        logger.info("Root hash: \(rootHash.toHexString())")
         try await printNode(rootHash, depth: 0)
     }
 }

diff --git a/Boka/Sources/Boka.swift b/Boka/Sources/Boka.swift
@@ -116,9 +116,11 @@ struct Boka: AsyncParsableCommand {
             logger.info("Node name: \(name)")
         }
 
-        if let basePath {
-            logger.info("Base path: \(basePath)")
-        }
+        let database: Database = basePath.map {
+            var path = URL(fileURLWithPath: $0)
+            path.append(path: "db")
+            return .rocksDB(path: path)
+        } ?? .inMemory
 
         logger.info("Peers: \(peers)")
 
@@ -168,7 +170,8 @@ struct Boka: AsyncParsableCommand {
             network: networkConfig,
             peers: peers,
             local: local,
-            name: name
+            name: name,
+            database: database
         )
 
         let node: Node = if validator {

diff --git a/Codec/Sources/Codec/JamDecoder.swift b/Codec/Sources/Codec/JamDecoder.swift
@@ -40,7 +40,7 @@ public class JamDecoder {
     }
 }
 
-protocol ArrayWrapper: Collection where Element: Decodable {
+private protocol ArrayWrapper: Collection where Element: Decodable {
     static func from(array: [Element]) -> Self
 }
 
@@ -50,6 +50,16 @@ extension Array: ArrayWrapper where Element: Decodable {
     }
 }
 
+private protocol OptionalWrapper: Decodable {
+    static var wrappedType: Decodable.Type { get }
+}
+
+extension Optional: OptionalWrapper where Wrapped: Decodable {
+    static var wrappedType: Decodable.Type {
+        Wrapped.self
+    }
+}
+
 private class DecodeContext: Decoder {
     struct PushCodingPath: ~Copyable {
         let decoder: DecodeContext
@@ -160,8 +170,28 @@ private class DecodeContext: Decoder {
         }
     }
 
+    fileprivate func decodeOptional<T: Decodable>(_ type: T.Type, key: CodingKey?) throws -> T? {
+        let byte = try input.read()
+        switch byte {
+        case 0:
+            return nil
+        case 1:
+            return try decode(type, key: key)
+        default:
+            throw DecodingError.dataCorrupted(
+                DecodingError.Context(
+                    codingPath: codingPath,
+                    debugDescription: "Invalid boolean value: \(byte)"
+                )
+            )
+        }
+    }
+
     fileprivate func decode<T: Decodable>(_ type: T.Type, key: CodingKey?) throws -> T {
-        if type == Data.self {
+        // optional hanlding must be first to avoid type coercion
+        if let type = type as? any OptionalWrapper.Type {
+            try decodeOptional(type.wrappedType, key: key) as! T
+        } else if type == Data.self {
             try decodeData(codingPath: codingPath) as Data as! T
         } else if type == [UInt8].self {
             try decodeData(codingPath: codingPath) as [UInt8] as! T
Original file line number	Diff line number	Diff line change
Expand Up		@@ -26,3 +26,5 @@ settings.json
		c_cpp_properties.json

		Tools/openrpc.json

		tmp