From 9b0dd84fc4a2555492812ebe05e86d33b9d2b3c8 Mon Sep 17 00:00:00 2001 From: buhe Date: Mon, 30 Oct 2023 20:01:54 +0800 Subject: [PATCH 1/4] "Add CoreDataCache class to the cache module." --- Sources/LangChain/cache/Cache.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Sources/LangChain/cache/Cache.swift b/Sources/LangChain/cache/Cache.swift index 7e37533..bcc3a79 100644 --- a/Sources/LangChain/cache/Cache.swift +++ b/Sources/LangChain/cache/Cache.swift @@ -35,3 +35,7 @@ public class InMemoryCache: BaseCache { memery = [:] } } + +public class CoreDataCache: BaseCache { + +} From 59d949d4a12a307352a742331c2d3d17dadb86f8 Mon Sep 17 00:00:00 2001 From: buhe Date: Mon, 30 Oct 2023 23:10:07 +0800 Subject: [PATCH 2/4] "Add SwiftFileStore package and update cache lookup and update methods to be asynchronous." --- Package.resolved | 9 ++++++ Package.swift | 4 ++- Sources/LangChain/cache/Cache.swift | 48 +++++++++++++++++++++++++---- Sources/LangChain/llms/LLM.swift | 4 +-- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/Package.resolved b/Package.resolved index 4f70bc1..740ff40 100644 --- a/Package.resolved +++ b/Package.resolved @@ -117,6 +117,15 @@ "version" : "1.0.4" } }, + { + "identity" : "swift-filestore", + "kind" : "remoteSourceControl", + "location" : "https://github.com/juyan/swift-filestore", + "state" : { + "revision" : "878a896cc2fef836bd70a46bbb873b495155e2d0", + "version" : "0.2.0" + } + }, { "identity" : "swift-log", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index 1c923f2..bbc5ddc 100644 --- a/Package.swift +++ b/Package.swift @@ -20,7 +20,8 @@ let package = Package( .package(url: "https://github.com/supabase-community/supabase-swift", .upToNextMajor(from: "0.2.1")), .package(url: "https://github.com/SwiftyJSON/SwiftyJSON", .upToNextMajor(from: "5.0.1")), .package(url: "https://github.com/drmohundro/SWXMLHash", .upToNextMajor(from: "7.0.2")), - .package(url: "https://github.com/scinfu/SwiftSoup", .upToNextMajor(from: "2.6.1")) + .package(url: "https://github.com/scinfu/SwiftSoup", .upToNextMajor(from: "2.6.1")), + .package(url: "https://github.com/juyan/swift-filestore", .upToNextMajor(from: "0.2.0")), ], targets: [ // Targets are the basic building blocks of a package, defining a module or a test suite. @@ -33,6 +34,7 @@ let package = Package( .product(name: "SwiftyJSON", package: "SwiftyJSON"), .product(name: "SWXMLHash", package: "SWXMLHash"), .product(name: "SwiftSoup", package: "SwiftSoup"), + .product(name: "SwiftFileStore", package: "swift-filestore"), ] ), diff --git a/Sources/LangChain/cache/Cache.swift b/Sources/LangChain/cache/Cache.swift index bcc3a79..c3f0fd7 100644 --- a/Sources/LangChain/cache/Cache.swift +++ b/Sources/LangChain/cache/Cache.swift @@ -6,12 +6,14 @@ // import Foundation +import SwiftFileStore + public class BaseCache { public init() {} - public func lookup(prompt: String) -> LLMResult? { + public func lookup(prompt: String) async -> LLMResult? { nil } - public func update(prompt: String, return_val: LLMResult) { + public func update(prompt: String, return_val: LLMResult) async { } //For test? @@ -23,11 +25,11 @@ public class BaseCache { public class InMemoryCache: BaseCache { var memery: [String: LLMResult] = [:] - public override func lookup(prompt: String) -> LLMResult? { + public override func lookup(prompt: String) async -> LLMResult? { print("🍰 Get \(prompt) from cache") return memery[prompt] } - public override func update(prompt: String, return_val: LLMResult) { + public override func update(prompt: String, return_val: LLMResult) async { print("🍰 Update \(prompt)") memery[prompt] = return_val } @@ -35,7 +37,41 @@ public class InMemoryCache: BaseCache { memery = [:] } } +struct LLMCache: Codable, JSONDataRepresentable { + let key: String + let value: String +} +public class FileCache: BaseCache { + let objectStore: FileObjectStore? -public class CoreDataCache: BaseCache { - + public override init() { + do { + self.objectStore = try FileObjectStore.create() + } catch { + self.objectStore = nil + } + } + public override func lookup(prompt: String) async -> LLMResult? { + print("🍰 Get \(prompt) from file") + do { + let cache = try await objectStore!.read(key: prompt, namespace: "llm_cache", objectType: LLMCache.self) + return LLMResult(llm_output: cache?.value) + } catch { + return nil + } + + + } + public override func update(prompt: String, return_val: LLMResult) async { + print("🍰 Update \(prompt) at file") + do { + let cache = LLMCache(key: prompt, value: return_val.llm_output!) + try await objectStore!.write(key: cache.key, namespace: "llm_cache", object: cache) + } catch { + + } + } + public override func clear() { + + } } diff --git a/Sources/LangChain/llms/LLM.swift b/Sources/LangChain/llms/LLM.swift index f65f238..861ddf1 100644 --- a/Sources/LangChain/llms/LLM.swift +++ b/Sources/LangChain/llms/LLM.swift @@ -29,14 +29,14 @@ public class LLM { callStart(prompt: text, reqId: reqId) do { if let cache = self.cache { - if let llmResult = cache.lookup(prompt: text) { + if let llmResult = await cache.lookup(prompt: text) { callEnd(output: llmResult.llm_output!, reqId: reqId, cost: 0) return llmResult } } let llmResult = try await _send(text: text, stops: stops) if let cache = self.cache { - cache.update(prompt: text, return_val: llmResult) + await cache.update(prompt: text, return_val: llmResult) } cost = Date.now.timeIntervalSince1970 - now if !llmResult.stream { From 2394e18c831803a139a02562e7d107c48df3fbbf Mon Sep 17 00:00:00 2001 From: buhe Date: Tue, 31 Oct 2023 09:27:12 +0800 Subject: [PATCH 3/4] "Add LLM Cache options to README.md" --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f4f6ad4..f906fab 100644 --- a/README.md +++ b/README.md @@ -374,7 +374,9 @@ Task { - [x] MultiPromptRouter - Callback - [x] StdOutCallbackHandler - +- LLM Cache + - [x] InMemery + - [x] File ## 👍 Got Ideas? Open an issue, and let's discuss! From 8af0763c5fad6a1fb17709ab950a6f0c889f6a1a Mon Sep 17 00:00:00 2001 From: buhe Date: Tue, 31 Oct 2023 10:29:32 +0800 Subject: [PATCH 4/4] "Refactor cache lookup and update methods to use SHA256 hashing for key generation." --- Sources/LangChain/LangChain.swift | 35 +++++++++++++++++++++++++++++ Sources/LangChain/cache/Cache.swift | 20 ++++++++++++----- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/Sources/LangChain/LangChain.swift b/Sources/LangChain/LangChain.swift index 20c7acb..48b8758 100644 --- a/Sources/LangChain/LangChain.swift +++ b/Sources/LangChain/LangChain.swift @@ -6,8 +6,43 @@ // import Foundation +import CommonCrypto enum LangChainError: Error { case LoaderError(String) } + +extension Data{ + public func sha256() -> String{ + return hexStringFromData(input: digest(input: self as NSData)) + } + + private func digest(input : NSData) -> NSData { + let digestLength = Int(CC_SHA256_DIGEST_LENGTH) + var hash = [UInt8](repeating: 0, count: digestLength) + CC_SHA256(input.bytes, UInt32(input.length), &hash) + return NSData(bytes: hash, length: digestLength) + } + + private func hexStringFromData(input: NSData) -> String { + var bytes = [UInt8](repeating: 0, count: input.length) + input.getBytes(&bytes, length: input.length) + + var hexString = "" + for byte in bytes { + hexString += String(format:"%02x", UInt8(byte)) + } + + return hexString + } +} + +public extension String { + func sha256() -> String{ + if let stringData = self.data(using: String.Encoding.utf8) { + return stringData.sha256() + } + return "" + } +} diff --git a/Sources/LangChain/cache/Cache.swift b/Sources/LangChain/cache/Cache.swift index c3f0fd7..0b0259a 100644 --- a/Sources/LangChain/cache/Cache.swift +++ b/Sources/LangChain/cache/Cache.swift @@ -54,8 +54,15 @@ public class FileCache: BaseCache { public override func lookup(prompt: String) async -> LLMResult? { print("🍰 Get \(prompt) from file") do { - let cache = try await objectStore!.read(key: prompt, namespace: "llm_cache", objectType: LLMCache.self) - return LLMResult(llm_output: cache?.value) + if let data = prompt.data(using: .utf8) { + let base64 = data.base64EncodedString() + + let cache = try await objectStore!.read(key: base64.sha256(), namespace: "llm_cache", objectType: LLMCache.self) + if let c = cache { + return LLMResult(llm_output: c.value) + } + } + return nil } catch { return nil } @@ -65,10 +72,13 @@ public class FileCache: BaseCache { public override func update(prompt: String, return_val: LLMResult) async { print("🍰 Update \(prompt) at file") do { - let cache = LLMCache(key: prompt, value: return_val.llm_output!) - try await objectStore!.write(key: cache.key, namespace: "llm_cache", object: cache) + if let data = prompt.data(using: .utf8) { + let base64 = data.base64EncodedString() + let cache = LLMCache(key: prompt, value: return_val.llm_output!) + try await objectStore!.write(key: base64.sha256(), namespace: "llm_cache", object: cache) + } } catch { - + print("FileCache set failed") } } public override func clear() {