Merge branch 'master' into dev-codec-coverage

* master:
  minor refactor for RocksDB (#244)

Database/Package.swift

@@ -16,6 +16,8 @@ let package = Package(
         ),
     ],
     dependencies: [
+        .package(path: "../Blockchain"),
+        .package(path: "../Utils"),
         .package(url: "https://github.com/apple/swift-testing.git", branch: "0.10.0"),
     ],
     targets: [
@@ -25,6 +27,8 @@ let package = Package(
             name: "Database",
             dependencies: [
                 "rocksdb",
+                "Blockchain",
+                "Utils",
             ],
             linkerSettings: [
                 .unsafeFlags(["-L../.lib", "-L/opt/homebrew/lib"]),

Database/Sources/Database/Options.swift

@@ -0,0 +1,48 @@
+import rocksdb
+import Utils
+
+public struct Options: ~Copyable, Sendable {
+    let ptr: SafePointer
+
+    var value: OpaquePointer { ptr.value }
+
+    public init() {
+        ptr = .init(ptr: rocksdb_options_create(), free: rocksdb_options_destroy)
+    }
+
+    public func increaseParallelism(cpus: Int) {
+        rocksdb_options_increase_parallelism(ptr.value, Int32(cpus))
+    }
+
+    public func optimizeLevelStyleCompaction(memtableMemoryBudget: UInt64) {
+        rocksdb_options_optimize_level_style_compaction(ptr.value, memtableMemoryBudget)
+    }
+
+    public func setCreateIfMissing(_ createIfMissing: Bool) {
+        rocksdb_options_set_create_if_missing(ptr.value, createIfMissing ? 1 : 0)
+    }
+
+    public func setLevelCompactionDynamicLevelBytes(levelCompactionDynamicLevelBytes: Bool) {
+        rocksdb_options_set_level_compaction_dynamic_level_bytes(ptr.value, levelCompactionDynamicLevelBytes ? 1 : 0)
+    }
+}
+
+public struct WriteOptions: ~Copyable, Sendable {
+    let ptr: SafePointer
+
+    var value: OpaquePointer { ptr.value }
+
+    public init() {
+        ptr = .init(ptr: rocksdb_writeoptions_create(), free: rocksdb_writeoptions_destroy)
+    }
+}
+
+public struct ReadOptions: ~Copyable, Sendable {
+    let ptr: SafePointer
+
+    var value: OpaquePointer { ptr.value }
+
+    public init() {
+        ptr = .init(ptr: rocksdb_readoptions_create(), free: rocksdb_readoptions_destroy)
+    }
+}

Database/Sources/Database/RocksDB.swift

@@ -1,7 +1,8 @@
 import Foundation
 import rocksdb
+import Utils
 
-public final class RocksDB {
+public final class RocksDB: Sendable {
     public enum BatchOperation {
         case delete(key: Data)
         case put(key: Data, value: Data)
@@ -16,42 +17,37 @@ public final class RocksDB {
         case noData
     }
 
-    private let dbOptions: OpaquePointer
-    private let writeOptions: OpaquePointer
-    private let readOptions: OpaquePointer
-    private let db: OpaquePointer
+    private let dbOptions: Options
+    private let writeOptions: WriteOptions
+    private let readOptions: ReadOptions
+    private let db: SendableOpaquePointer
 
     public init(path: URL) throws(Error) {
-        let dbOptions = rocksdb_options_create()
-        self.dbOptions = dbOptions!
-        let cpus = sysconf(Int32(_SC_NPROCESSORS_ONLN))
-
-        // Optimize rocksdb
-        rocksdb_options_increase_parallelism(dbOptions, Int32(cpus))
-        let memtable_memory_budget: UInt64 = 512 * 1024 * 1024 // 512 MB
-        rocksdb_options_optimize_level_style_compaction(dbOptions, memtable_memory_budget)
+        let dbOptions = Options()
 
-        // create the DB if it's not already present
-        rocksdb_options_set_create_if_missing(dbOptions, 1)
+        // TODO: starting from options here
+        // https://github.com/paritytech/parity-common/blob/e3787dc768b08e10809834c65419ad3c255b5cac/kvdb-rocksdb/src/lib.rs#L339
 
-        // create writeoptions
-        writeOptions = rocksdb_writeoptions_create()
-        // create readoptions
-        readOptions = rocksdb_readoptions_create()
+        let cpus = sysconf(Int32(_SC_NPROCESSORS_ONLN))
+        dbOptions.increaseParallelism(cpus: cpus)
+        dbOptions.optimizeLevelStyleCompaction(memtableMemoryBudget: 512 * 1024 * 1024) // 512 MB
+        dbOptions.setCreateIfMissing(true)
 
         // open DB
         db = try Self.call { err, _ in
-            rocksdb_open(dbOptions, path.path, &err)
+            rocksdb_open(dbOptions.value, path.path, &err).asSendable
         } onErr: { message throws(Error) in
             throw Error.openFailed(message: message)
         }
+
+        self.dbOptions = dbOptions
+
+        writeOptions = WriteOptions()
+        readOptions = ReadOptions()
     }
 
     deinit {
-        rocksdb_writeoptions_destroy(writeOptions)
-        rocksdb_readoptions_destroy(readOptions)
-        rocksdb_options_destroy(dbOptions)
-        rocksdb_close(db)
+        rocksdb_close(db.value)
     }
 }
 
@@ -61,8 +57,6 @@ extension RocksDB {
     private static func call<R>(
         _ data: [Data],
         fn: (inout UnsafeMutablePointer<Int8>?, [(ptr: UnsafeRawPointer, count: Int)]) -> R,
-        // need new swiftlint version https://github.com/realm/SwiftLint/issues/5631
-        // swiftlint:disable:next identifier_name
         onErr: (String) throws(Error) -> Void
     ) throws(Error) -> R {
         var err: UnsafeMutablePointer<Int8>?
@@ -127,7 +121,7 @@ extension RocksDB {
         try Self.call(key, value) { err, ptrs in
             let key = ptrs[0]
             let value = ptrs[1]
-            rocksdb_put(db, writeOptions, key.ptr, key.count, value.ptr, value.count, &err)
+            rocksdb_put(db.value, writeOptions.value, key.ptr, key.count, value.ptr, value.count, &err)
         } onErr: { message throws(Error) in
             throw Error.putFailed(message: message)
         }
@@ -138,22 +132,18 @@ extension RocksDB {
 
         let ret = try Self.call(key) { err, ptrs in
             let key = ptrs[0]
-            return rocksdb_get(db, readOptions, key.ptr, key.count, &len, &err)
+            return rocksdb_get(db.value, readOptions.value, key.ptr, key.count, &len, &err)
         } onErr: { message throws(Error) in
             throw Error.getFailed(message: message)
         }
 
-        defer {
-            free(ret)
-        }
-
-        return ret.map { Data(bytes: $0, count: len) }
+        return ret.map { Data(bytesNoCopy: $0, count: len, deallocator: .free) }
     }
 
     public func delete(key: Data) throws {
         try Self.call(key) { err, ptrs in
             let key = ptrs[0]
-            rocksdb_delete(db, writeOptions, key.ptr, key.count, &err)
+            rocksdb_delete(db.value, writeOptions.value, key.ptr, key.count, &err)
         } onErr: { message throws(Error) in
             throw Error.deleteFailed(message: message)
         }
@@ -182,7 +172,7 @@ extension RocksDB {
         }
 
         try Self.call { err, _ in
-            rocksdb_write(db, writeOptions, writeBatch, &err)
+            rocksdb_write(db.value, writeOptions.value, writeBatch, &err)
         } onErr: { message throws(Error) in
             throw Error.batchFailed(message: message)
         }

Database/Sources/Database/RocksDBBackend.swift

@@ -0,0 +1,27 @@
+import Blockchain
+import Foundation
+import Utils
+
+public final class RocksDBBackend: StateBackendProtocol {
+    public init() {}
+
+    public func read(key _: Data) async throws -> Data? {
+        fatalError("unimplemented")
+    }
+
+    public func readAll(prefix _: Data, startKey _: Data?, limit _: UInt32?) async throws -> [(key: Data, value: Data)] {
+        fatalError("unimplemented")
+    }
+
+    public func batchUpdate(_: [StateBackendOperation]) async throws {
+        fatalError("unimplemented")
+    }
+
+    public func readValue(hash _: Data32) async throws -> Data? {
+        fatalError("unimplemented")
+    }
+
+    public func gc(callback _: @Sendable (Data) -> Data32?) async throws {
+        fatalError("unimplemented")
+    }
+}

Utils/Sources/Utils/Crypto/BLS.swift

@@ -30,7 +30,7 @@ public enum BLS: KeyType {
             }
 
             // use SafePointer to ensure keypair is freed even `try PublicKey` throws
-            keyPairPtr = SafePointer(ptr: ptr.asSendable, free: keypair_free)
+            keyPairPtr = SafePointer(ptr: ptr, free: keypair_free)
             publicKey = try PublicKey(keyPair: keyPairPtr.ptr.value)
         }
 

Utils/Sources/Utils/Crypto/Bandersnatch.swift

@@ -29,7 +29,7 @@ public enum Bandersnatch: KeyType {
                 throw .createSecretFailed(err)
             }
 
-            self.ptr = SafePointer(ptr: ptr.asSendable, free: secret_free)
+            self.ptr = SafePointer(ptr: ptr, free: secret_free)
             publicKey = try PublicKey(secretKey: self.ptr.ptr.value)
         }
 

Utils/Sources/Utils/SafePointer.swift

@@ -1,5 +1,13 @@
 public struct SafePointer: ~Copyable, Sendable {
-    let ptr: SendableOpaquePointer
-    let free: @Sendable (_ ptr: OpaquePointer) -> Void
+    public let ptr: SendableOpaquePointer
+    private let free: @Sendable (_ ptr: OpaquePointer) -> Void
+
+    public var value: OpaquePointer { ptr.value }
+
+    public init(ptr: OpaquePointer, free: @Sendable @escaping (OpaquePointer) -> Void) {
+        self.ptr = ptr.asSendable
+        self.free = free
+    }
+
     deinit { free(ptr.value) }
 }