Add multiple cache storages feature #156
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -16,22 +16,6 @@ struct FrameworkProducer { | |
| private let fileSystem: any FileSystem | ||
| private let toolchainEnvironment: [String: String]? | ||
|
|
||
| private var shouldGenerateVersionFile: Bool { | ||
| // cacheMode is not disabled | ||
| if case .disabled = cacheMode { | ||
|
|
@@ -104,20 +88,17 @@ struct FrameworkProducer { | |
| buildOptions: buildOptionsForProduct | ||
| ) | ||
| }) | ||
|
|
||
| let pinsStore = try descriptionPackage.workspace.pinsStore.load() | ||
| let cacheSystem = CacheSystem( | ||
| pinsStore: pinsStore, | ||
| outputDirectory: outputDir | ||
| ) | ||
|
|
||
| let cacheEnabledTargets = await restoreAllAvailableCachesIfNeeded( | ||
| availableTargets: Set(allTargets), | ||
| cacheSystem: cacheSystem | ||
| ) | ||
|
|
||
| let targetsToBuild = allTargets.subtracting(cacheEnabledTargets) | ||
|
|
||
|
|
@@ -129,9 +110,7 @@ struct FrameworkProducer { | |
| ) | ||
| } | ||
|
|
||
| await cacheFrameworksIfNeeded(Set(targetsToBuild), cacheSystem: cacheSystem) | ||
|
|
||
| if shouldGenerateVersionFile { | ||
| // Versionfiles should be generate for all targets | ||
|
|
@@ -141,22 +120,93 @@ struct FrameworkProducer { | |
| } | ||
| } | ||
|
|
||
| private func restoreAllAvailableCachesIfNeeded( | ||
| availableTargets: Set<CacheSystem.CacheTarget>, | ||
| cacheSystem: CacheSystem | ||
| ) async -> Set<CacheSystem.CacheTarget> { | ||
| let cacheStorages: [any CacheStorage] | ||
|
|
||
| switch cacheMode { | ||
| case .disabled: | ||
| return [] | ||
| case .project: | ||
ikesyo marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| // For `.project`, just checking whether the valid caches (already built frameworks under the project) | ||
| // exist or not (not restoring anything from external locations). | ||
| return await restoreCachesForTargets( | ||
| availableTargets, | ||
| cacheSystem: cacheSystem, | ||
| cacheStorage: nil | ||
| ) | ||
|
There was a problem hiding this comment. here: Scipio/Sources/ScipioKit/Producer/FrameworkProducer.swift Lines 245 to 259 in f6cbef1 |
||
| case .storage(let storage, let actors): | ||
| guard actors.contains(.consumer) else { return [] } | ||
| cacheStorages = [storage] | ||
| case .storages(let storages): | ||
| let storagesWithConsumer = storages.compactMap { storage, actors in | ||
| actors.contains(.consumer) ? storage : nil | ||
| } | ||
| guard !storagesWithConsumer.isEmpty else { return [] } | ||
| cacheStorages = storagesWithConsumer | ||
| } | ||
|
|
||
| var remainingTargets = availableTargets | ||
| var restored: Set<CacheSystem.CacheTarget> = [] | ||
|
|
||
| for index in cacheStorages.indices { | ||
| let storage = cacheStorages[index] | ||
|
|
||
| let logSuffix = "[\(index)] \(type(of: storage))" | ||
| if index == cacheStorages.startIndex { | ||
| logger.info( | ||
| "▶️ Starting restoration with cache storage: \(logSuffix)", | ||
| metadata: .color(.green) | ||
| ) | ||
| } else { | ||
| logger.info( | ||
| "⏭️ Falling back to next cache storage: \(logSuffix)", | ||
| metadata: .color(.green) | ||
| ) | ||
| } | ||
|
|
||
| let restoredPerStorage = await restoreCachesForTargets( | ||
| remainingTargets, | ||
| cacheSystem: cacheSystem, | ||
| cacheStorage: storage | ||
| ) | ||
| restored.formUnion(restoredPerStorage) | ||
|
|
||
| logger.info( | ||
| "⏸️ Restoration finished with cache storage: \(logSuffix)", | ||
| metadata: .color(.green) | ||
| ) | ||
|
|
||
| remainingTargets.subtract(restoredPerStorage) | ||
| if remainingTargets.isEmpty { | ||
ikesyo marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| break | ||
| } | ||
|
There was a problem hiding this comment. if all frameworks are successfully restored, we don't need to proceed to next cache storage. There was a problem hiding this comment. [not mandatory] it's kindly to comment this |
||
| } | ||
|
|
||
| logger.info("⏹️ Restoration finished", metadata: .color(.green)) | ||
| return restored | ||
| } | ||
|
|
||
| private func restoreCachesForTargets( | ||
| _ targets: Set<CacheSystem.CacheTarget>, | ||
| cacheSystem: CacheSystem, | ||
| cacheStorage: (any CacheStorage)? | ||
| ) async -> Set<CacheSystem.CacheTarget> { | ||
ikesyo marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| let chunked = targets.chunks(ofCount: cacheStorage?.parallelNumber ?? CacheSystem.defaultParalellNumber) | ||
|
|
||
| var restored: Set<CacheSystem.CacheTarget> = [] | ||
| for chunk in chunked { | ||
| let restorer = Restorer(outputDir: outputDir, fileSystem: fileSystem) | ||
| await withTaskGroup(of: CacheSystem.CacheTarget?.self) { group in | ||
| for target in chunk { | ||
| group.addTask { | ||
| do { | ||
| let restored = try await restorer.restore( | ||
| target: target, | ||
| cacheSystem: cacheSystem, | ||
| cacheStorage: cacheStorage | ||
| ) | ||
| return restored ? target : nil | ||
| } catch { | ||
|
|
@@ -175,25 +225,23 @@ struct FrameworkProducer { | |
| /// Sendable interface to provide restore caches | ||
| private struct Restorer: Sendable { | ||
| let outputDir: URL | ||
| let fileSystem: any FileSystem | ||
|
|
||
| // Return true if pre-built artifact is available (already existing or restored from cache) | ||
| func restore( | ||
| target: CacheSystem.CacheTarget, | ||
| cacheSystem: CacheSystem, | ||
| cacheStorage: (any CacheStorage)? | ||
| ) async throws -> Bool { | ||
| let product = target.buildProduct | ||
| let frameworkName = product.frameworkName | ||
| let outputPath = outputDir.appendingPathComponent(product.frameworkName) | ||
| let exists = fileSystem.exists(outputPath.absolutePath) | ||
|
|
||
|
There was a problem hiding this comment. This is checked within |
||
| let expectedCacheKey = try await cacheSystem.calculateCacheKey(of: target) | ||
| let isValidCache = await cacheSystem.existsValidCache(cacheKey: expectedCacheKey) | ||
| let expectedCacheKeyHash = try expectedCacheKey.calculateChecksum() | ||
|
|
||
| if isValidCache && exists { | ||
| logger.info( | ||
| "✅ Valid \(product.target.name).xcframework (\(expectedCacheKeyHash)) is exists. Skip building.", metadata: .color(.green) | ||
|
|
@@ -205,7 +253,12 @@ struct FrameworkProducer { | |
| logger.info("🗑️ Delete \(frameworkName)", metadata: .color(.red)) | ||
| try fileSystem.removeFileTree(outputPath.absolutePath) | ||
| } | ||
|
|
||
| guard let cacheStorage else { | ||
| return false | ||
| } | ||
|
|
||
| let restoreResult = await cacheSystem.restoreCacheIfPossible(target: target, storage: cacheStorage) | ||
| switch restoreResult { | ||
| case .succeeded: | ||
| logger.info("✅ Restore \(frameworkName) (\(expectedCacheKeyHash)) from cache storage.", metadata: .color(.green)) | ||
|
|
@@ -260,6 +313,29 @@ struct FrameworkProducer { | |
| return [] | ||
| } | ||
|
|
||
| private func cacheFrameworksIfNeeded(_ targets: Set<CacheSystem.CacheTarget>, cacheSystem: CacheSystem) async { | ||
| switch cacheMode { | ||
| case .disabled: | ||
| // no-op | ||
| break | ||
| case .project: | ||
| // For `.project` which is not tied to any (external) storages, we don't need to do anything. | ||
| // The built frameworks under the project themselves are treated as valid caches. | ||
| break | ||
| case .storage(let storage, let actors): | ||
| if actors.contains(.producer) { | ||
| await cacheSystem.cacheFrameworks(targets, storages: [storage]) | ||
| } | ||
| case .storages(let storages): | ||
| let storagesWithProducer = storages.compactMap { storage, actors in | ||
| actors.contains(.producer) ? storage : nil | ||
| } | ||
| if !storagesWithProducer.isEmpty { | ||
| await cacheSystem.cacheFrameworks(targets, storages: storagesWithProducer) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private func generateVersionFile(for target: CacheSystem.CacheTarget, using cacheSystem: CacheSystem) async { | ||
| do { | ||
| try await cacheSystem.generateVersionFile(for: target) | ||
|
|
@@ -268,14 +344,3 @@ struct FrameworkProducer { | |
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Due to the introduction of cache policies, storages may be different on when saving caches and when restoring caches. So removed the property and passes by arguments instead.