From 6d35c14da1cc6d2e119dabef386d0327ba3fd242 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Thu, 7 Nov 2024 18:19:57 +0900 Subject: [PATCH 01/31] Add multiple cache storages feature - `Runner.Options.CacheMode.storages` is added - When saving, cache frameworks into all storages with `CacheActorKind.producer` - When restoring, storages with `CacheActorKind.consumer` are used in order - If first storage failed to restore some caches, the failed ones are tried to be fetched from second storage --- .../Producer/Cache/CacheSystem.swift | 38 ++++--- .../Producer/FrameworkProducer.swift | 105 ++++++++++++++---- Sources/ScipioKit/Runner.swift | 1 + Tests/ScipioKitTests/CacheSystemTests.swift | 2 +- Tests/ScipioKitTests/RunnerTests.swift | 4 +- 5 files changed, 110 insertions(+), 40 deletions(-) diff --git a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift index 2bb62e1a..c159c6cd 100644 --- a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift +++ b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift @@ -108,7 +108,7 @@ struct CacheSystem: Sendable { static let defaultParalellNumber = 8 private let pinsStore: PinsStore private let outputDirectory: URL - private let storage: (any CacheStorage)? + private let writableStorages: [any CacheStorage] private let fileSystem: any FileSystem struct CacheTarget: Hashable, Sendable { @@ -139,17 +139,29 @@ struct CacheSystem: Sendable { init( pinsStore: PinsStore, outputDirectory: URL, - storage: (any CacheStorage)?, + writableStorages: [any CacheStorage], fileSystem: any FileSystem = localFileSystem ) { self.pinsStore = pinsStore self.outputDirectory = outputDirectory - self.storage = storage + self.writableStorages = writableStorages self.fileSystem = fileSystem } func cacheFrameworks(_ targets: Set) async { - let chunked = targets.chunks(ofCount: storage?.parallelNumber ?? CacheSystem.defaultParalellNumber) + guard !writableStorages.isEmpty else { + // About `CacheMode.project` which does not have any writableStorages, we don't need to do anything. + // The built frameworks under the project themselves are treated as valid caches. + return + } + + for storage in writableStorages { + await cacheFrameworks(targets, storage: storage) + } + } + + private func cacheFrameworks(_ targets: Set, storage: any CacheStorage) async { + let chunked = targets.chunks(ofCount: storage.parallelNumber ?? CacheSystem.defaultParalellNumber) for chunk in chunked { await withTaskGroup(of: Void.self) { group in @@ -159,10 +171,10 @@ struct CacheSystem: Sendable { let frameworkPath = outputDirectory.appendingPathComponent(frameworkName) do { logger.info( - "πŸš€ Cache \(frameworkName) to cache storage", + "πŸš€ Cache \(frameworkName) to cache storage: \(storage)", metadata: .color(.green) ) - try await cacheFramework(target, at: frameworkPath) + try await cacheFramework(target, at: frameworkPath, storage: storage) } catch { logger.warning("⚠️ Can't create caches for \(frameworkPath.path)") } @@ -173,10 +185,10 @@ struct CacheSystem: Sendable { } } - private func cacheFramework(_ target: CacheTarget, at frameworkPath: URL) async throws { + private func cacheFramework(_ target: CacheTarget, at frameworkPath: URL, storage: any CacheStorage) async throws { let cacheKey = try await calculateCacheKey(of: target) - try await storage?.cacheFramework(frameworkPath, for: cacheKey) + try await storage.cacheFramework(frameworkPath, for: cacheKey) } func generateVersionFile(for target: CacheTarget) async throws { @@ -210,8 +222,8 @@ struct CacheSystem: Sendable { case failed(LocalizedError?) case noCache } - func restoreCacheIfPossible(target: CacheTarget) async -> RestoreResult { - guard let storage = storage else { return .noCache } + + func restoreCacheIfPossible(target: CacheTarget, storage: any CacheStorage) async -> RestoreResult { do { let cacheKey = try await calculateCacheKey(of: target) if try await storage.existsValidCache(for: cacheKey) { @@ -225,12 +237,6 @@ struct CacheSystem: Sendable { } } - private func fetchArtifacts(target: CacheTarget, to destination: URL) async throws { - guard let storage = storage else { return } - let cacheKey = try await calculateCacheKey(of: target) - try await storage.fetchArtifacts(for: cacheKey, to: destination) - } - func calculateCacheKey(of target: CacheTarget) async throws -> SwiftPMCacheKey { let targetName = target.buildProduct.target.name let pin = try retrievePin(package: target.buildProduct.package) diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index cfb5be6f..f61676c7 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -16,19 +16,27 @@ struct FrameworkProducer { private let fileSystem: any FileSystem private let toolchainEnvironment: [String: String]? - private var cacheStorage: (any CacheStorage)? { + private func cacheStorages(for actor: Runner.Options.CacheMode.CacheActorKind) -> [any CacheStorage]? { switch cacheMode { - case .disabled, .project: return nil - case .storage(let storage, _): return storage + case .disabled, .project: + return nil + case .storage(let storage, let actors): + return actors.contains(actor) ? [storage] : nil + case .storages(let storages): + return storages.compactMap { $0.actors.contains(actor) ? $0.storage : nil } } } private var isProducingCacheEnabled: Bool { switch cacheMode { - case .disabled: return false - case .project: return true + case .disabled: + return false + case .project: + return true case .storage(_, let actors): return actors.contains(.producer) + case .storages(let storages): + return storages.contains { $0.actors.contains(.producer) } } } @@ -106,14 +114,17 @@ struct FrameworkProducer { }) let pinsStore = try descriptionPackage.workspace.pinsStore.load() - let cacheSystem = CacheSystem(pinsStore: pinsStore, - outputDirectory: outputDir, - storage: cacheStorage) + let cacheSystem = CacheSystem( + pinsStore: pinsStore, + outputDirectory: outputDir, + writableStorages: cacheStorages(for: .producer) ?? [] + ) let cacheEnabledTargets: Set if cacheMode.isConsumingCacheEnabled { cacheEnabledTargets = await restoreAllAvailableCaches( availableTargets: Set(allTargets), - cacheSystem: cacheSystem + cacheSystem: cacheSystem, + readableStorages: cacheStorages(for: .consumer) ) } else { cacheEnabledTargets = [] @@ -143,20 +154,65 @@ struct FrameworkProducer { private func restoreAllAvailableCaches( availableTargets: Set, - cacheSystem: CacheSystem + cacheSystem: CacheSystem, + readableStorages: [any CacheStorage]? + ) async -> Set { + guard let readableStorages, !readableStorages.isEmpty else { + // This is for `CacheMode.project`. + // + // In that case just checking whether the valid caches (already built frameworks under the project) + // exist or not (not restoring anything from external locations). + return await restoreCachesForTargets( + availableTargets, + cacheSystem: cacheSystem, + cacheStorage: nil + ) + } + + var remainingTargets = availableTargets + var restored: Set = [] + + for index in readableStorages.indices { + let storage = readableStorages[index] + + if index != readableStorages.startIndex { + logger.info("Falling back to \(storage)", metadata: .color(.green)) + } + + let restoredPerStorage = await restoreCachesForTargets( + remainingTargets, + cacheSystem: cacheSystem, + cacheStorage: storage + ) + restored.formUnion(restoredPerStorage) + + remainingTargets.subtract(restoredPerStorage) + if remainingTargets.isEmpty { + break + } + } + + return restored + } + + private func restoreCachesForTargets( + _ targets: Set, + cacheSystem: CacheSystem, + cacheStorage: (any CacheStorage)? ) async -> Set { - let chunked = availableTargets.chunks(ofCount: cacheStorage?.parallelNumber ?? CacheSystem.defaultParalellNumber) + let chunked = targets.chunks(ofCount: cacheStorage?.parallelNumber ?? CacheSystem.defaultParalellNumber) var restored: Set = [] for chunk in chunked { - let restorer = Restorer(outputDir: outputDir, cacheMode: cacheMode, fileSystem: fileSystem) + let restorer = Restorer(outputDir: outputDir, fileSystem: fileSystem) await withTaskGroup(of: CacheSystem.CacheTarget?.self) { group in for target in chunk { group.addTask { do { let restored = try await restorer.restore( target: target, - cacheSystem: cacheSystem + cacheSystem: cacheSystem, + cacheStorage: cacheStorage ) return restored ? target : nil } catch { @@ -175,25 +231,23 @@ struct FrameworkProducer { /// Sendable interface to provide restore caches private struct Restorer: Sendable { let outputDir: URL - let cacheMode: Runner.Options.CacheMode let fileSystem: any FileSystem // Return true if pre-built artifact is available (already existing or restored from cache) func restore( target: CacheSystem.CacheTarget, - cacheSystem: CacheSystem + cacheSystem: CacheSystem, + cacheStorage: (any CacheStorage)? ) async throws -> Bool { let product = target.buildProduct let frameworkName = product.frameworkName let outputPath = outputDir.appendingPathComponent(product.frameworkName) let exists = fileSystem.exists(outputPath.absolutePath) - guard cacheMode.isConsumingCacheEnabled else { - return false - } let expectedCacheKey = try await cacheSystem.calculateCacheKey(of: target) let isValidCache = await cacheSystem.existsValidCache(cacheKey: expectedCacheKey) let expectedCacheKeyHash = try expectedCacheKey.calculateChecksum() + if isValidCache && exists { logger.info( "βœ… Valid \(product.target.name).xcframework (\(expectedCacheKeyHash)) is exists. Skip building.", metadata: .color(.green) @@ -205,7 +259,12 @@ struct FrameworkProducer { logger.info("πŸ—‘οΈ Delete \(frameworkName)", metadata: .color(.red)) try fileSystem.removeFileTree(outputPath.absolutePath) } - let restoreResult = await cacheSystem.restoreCacheIfPossible(target: target) + + guard let cacheStorage else { + return false + } + + let restoreResult = await cacheSystem.restoreCacheIfPossible(target: target, storage: cacheStorage) switch restoreResult { case .succeeded: logger.info("βœ… Restore \(frameworkName) (\(expectedCacheKeyHash)) from cache storage.", metadata: .color(.green)) @@ -272,10 +331,14 @@ struct FrameworkProducer { extension Runner.Options.CacheMode { fileprivate var isConsumingCacheEnabled: Bool { switch self { - case .disabled: return false - case .project: return true + case .disabled: + return false + case .project: + return true case .storage(_, let actors): return actors.contains(.consumer) + case .storages(let storages): + return storages.contains { $0.actors.contains(.consumer) } } } } diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift index d3451403..f618c1eb 100644 --- a/Sources/ScipioKit/Runner.swift +++ b/Sources/ScipioKit/Runner.swift @@ -219,6 +219,7 @@ extension Runner { case disabled case project case storage(any CacheStorage, Set) + case storages([(storage: any CacheStorage, actors: Set)]) } public enum PlatformSpecifier: Equatable { diff --git a/Tests/ScipioKitTests/CacheSystemTests.swift b/Tests/ScipioKitTests/CacheSystemTests.swift index ace30d9a..f1965c33 100644 --- a/Tests/ScipioKitTests/CacheSystemTests.swift +++ b/Tests/ScipioKitTests/CacheSystemTests.swift @@ -96,7 +96,7 @@ final class CacheSystemTests: XCTestCase { let cacheSystem = CacheSystem( pinsStore: try descriptionPackage.workspace.pinsStore.load(), outputDirectory: FileManager.default.temporaryDirectory.appendingPathComponent("XCFrameworks"), - storage: nil + writableStorages: [] ) let testingPackage = descriptionPackage .graph diff --git a/Tests/ScipioKitTests/RunnerTests.swift b/Tests/ScipioKitTests/RunnerTests.swift index 6414d573..57765149 100644 --- a/Tests/ScipioKitTests/RunnerTests.swift +++ b/Tests/ScipioKitTests/RunnerTests.swift @@ -278,7 +278,7 @@ final class RunnerTests: XCTestCase { let cacheSystem = CacheSystem( pinsStore: pinsStore, outputDirectory: frameworkOutputDir, - storage: nil + writableStorages: [] ) let packages = descriptionPackage.graph.packages .filter { $0.manifest.displayName != descriptionPackage.manifest.displayName } @@ -437,7 +437,7 @@ final class RunnerTests: XCTestCase { let cacheSystem = CacheSystem( pinsStore: pinsStore, outputDirectory: frameworkOutputDir, - storage: nil + writableStorages: [] ) let packages = descriptionPackage.graph.packages .filter { $0.manifest.displayName != descriptionPackage.manifest.displayName } From 5953d6c0cfa454caa0dfbc0b01f343ab812d1b91 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Fri, 8 Nov 2024 11:09:23 +0900 Subject: [PATCH 02/31] Pass storages to `CacheSystem.cacheFrameworks` --- Sources/ScipioKit/Producer/Cache/CacheSystem.swift | 11 ++++------- Sources/ScipioKit/Producer/FrameworkProducer.swift | 8 +++++--- Tests/ScipioKitTests/CacheSystemTests.swift | 3 +-- Tests/ScipioKitTests/RunnerTests.swift | 6 ++---- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift index c159c6cd..9c8f8f91 100644 --- a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift +++ b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift @@ -108,7 +108,6 @@ struct CacheSystem: Sendable { static let defaultParalellNumber = 8 private let pinsStore: PinsStore private let outputDirectory: URL - private let writableStorages: [any CacheStorage] private let fileSystem: any FileSystem struct CacheTarget: Hashable, Sendable { @@ -139,23 +138,21 @@ struct CacheSystem: Sendable { init( pinsStore: PinsStore, outputDirectory: URL, - writableStorages: [any CacheStorage], fileSystem: any FileSystem = localFileSystem ) { self.pinsStore = pinsStore self.outputDirectory = outputDirectory - self.writableStorages = writableStorages self.fileSystem = fileSystem } - func cacheFrameworks(_ targets: Set) async { - guard !writableStorages.isEmpty else { - // About `CacheMode.project` which does not have any writableStorages, we don't need to do anything. + func cacheFrameworks(_ targets: Set, storages: [any CacheStorage]?) async { + guard let storages, !storages.isEmpty else { + // About `CacheMode.project` which is not tied to any (external) storages, we don't need to do anything. // The built frameworks under the project themselves are treated as valid caches. return } - for storage in writableStorages { + for storage in storages { await cacheFrameworks(targets, storage: storage) } } diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index f61676c7..40fe9226 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -116,8 +116,7 @@ struct FrameworkProducer { let cacheSystem = CacheSystem( pinsStore: pinsStore, - outputDirectory: outputDir, - writableStorages: cacheStorages(for: .producer) ?? [] + outputDirectory: outputDir ) let cacheEnabledTargets: Set if cacheMode.isConsumingCacheEnabled { @@ -141,7 +140,10 @@ struct FrameworkProducer { } if isProducingCacheEnabled { - await cacheSystem.cacheFrameworks(Set(targetsToBuild)) + await cacheSystem.cacheFrameworks( + Set(targetsToBuild), + storages: cacheStorages(for: .producer) + ) } if shouldGenerateVersionFile { diff --git a/Tests/ScipioKitTests/CacheSystemTests.swift b/Tests/ScipioKitTests/CacheSystemTests.swift index f1965c33..e1a0f11c 100644 --- a/Tests/ScipioKitTests/CacheSystemTests.swift +++ b/Tests/ScipioKitTests/CacheSystemTests.swift @@ -95,8 +95,7 @@ final class CacheSystemTests: XCTestCase { ) let cacheSystem = CacheSystem( pinsStore: try descriptionPackage.workspace.pinsStore.load(), - outputDirectory: FileManager.default.temporaryDirectory.appendingPathComponent("XCFrameworks"), - writableStorages: [] + outputDirectory: FileManager.default.temporaryDirectory.appendingPathComponent("XCFrameworks") ) let testingPackage = descriptionPackage .graph diff --git a/Tests/ScipioKitTests/RunnerTests.swift b/Tests/ScipioKitTests/RunnerTests.swift index 57765149..9eac9380 100644 --- a/Tests/ScipioKitTests/RunnerTests.swift +++ b/Tests/ScipioKitTests/RunnerTests.swift @@ -277,8 +277,7 @@ final class RunnerTests: XCTestCase { let pinsStore = try descriptionPackage.workspace.pinsStore.load() let cacheSystem = CacheSystem( pinsStore: pinsStore, - outputDirectory: frameworkOutputDir, - writableStorages: [] + outputDirectory: frameworkOutputDir ) let packages = descriptionPackage.graph.packages .filter { $0.manifest.displayName != descriptionPackage.manifest.displayName } @@ -436,8 +435,7 @@ final class RunnerTests: XCTestCase { let pinsStore = try descriptionPackage.workspace.pinsStore.load() let cacheSystem = CacheSystem( pinsStore: pinsStore, - outputDirectory: frameworkOutputDir, - writableStorages: [] + outputDirectory: frameworkOutputDir ) let packages = descriptionPackage.graph.packages .filter { $0.manifest.displayName != descriptionPackage.manifest.displayName } From a24f7c9eab34ed76a3f8fa9ec3c2fcb509254f8b Mon Sep 17 00:00:00 2001 From: ikesyo Date: Fri, 8 Nov 2024 15:30:10 +0900 Subject: [PATCH 03/31] Refactor to remove isProducingCacheEnabled --- .../Producer/Cache/CacheSystem.swift | 8 +--- .../Producer/FrameworkProducer.swift | 43 +++++++++++-------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift index 9c8f8f91..c17be5fe 100644 --- a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift +++ b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift @@ -145,13 +145,7 @@ struct CacheSystem: Sendable { self.fileSystem = fileSystem } - func cacheFrameworks(_ targets: Set, storages: [any CacheStorage]?) async { - guard let storages, !storages.isEmpty else { - // About `CacheMode.project` which is not tied to any (external) storages, we don't need to do anything. - // The built frameworks under the project themselves are treated as valid caches. - return - } - + func cacheFrameworks(_ targets: Set, storages: [any CacheStorage]) async { for storage in storages { await cacheFrameworks(targets, storage: storage) } diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index 40fe9226..08cf49f4 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -27,19 +27,6 @@ struct FrameworkProducer { } } - private var isProducingCacheEnabled: Bool { - switch cacheMode { - case .disabled: - return false - case .project: - return true - case .storage(_, let actors): - return actors.contains(.producer) - case .storages(let storages): - return storages.contains { $0.actors.contains(.producer) } - } - } - private var shouldGenerateVersionFile: Bool { // cacheMode is not disabled if case .disabled = cacheMode { @@ -139,12 +126,7 @@ struct FrameworkProducer { ) } - if isProducingCacheEnabled { - await cacheSystem.cacheFrameworks( - Set(targetsToBuild), - storages: cacheStorages(for: .producer) - ) - } + await cacheFrameworksIfNeeded(Set(targetsToBuild), cacheSystem: cacheSystem) if shouldGenerateVersionFile { // Versionfiles should be generate for all targets @@ -321,6 +303,29 @@ struct FrameworkProducer { return [] } + private func cacheFrameworksIfNeeded(_ targets: Set, cacheSystem: CacheSystem) async { + switch cacheMode { + case .disabled: + // no-op + break + case .project: + // For `.project` which is not tied to any (external) storages, we don't need to do anything. + // The built frameworks under the project themselves are treated as valid caches. + break + case .storage(let storage, let actors): + if actors.contains(.producer) { + await cacheSystem.cacheFrameworks(targets, storages: [storage]) + } + case .storages(let storages): + let storagesWithProducer = storages.compactMap { storage, actors in + actors.contains(.producer) ? storage : nil + } + if !storagesWithProducer.isEmpty { + await cacheSystem.cacheFrameworks(targets, storages: storagesWithProducer) + } + } + } + private func generateVersionFile(for target: CacheSystem.CacheTarget, using cacheSystem: CacheSystem) async { do { try await cacheSystem.generateVersionFile(for: target) From d7bd9ff247c62a82d00062fa81d2d02e4deefb82 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Fri, 8 Nov 2024 15:44:43 +0900 Subject: [PATCH 04/31] Refactor to remove isConsumingCacheEnabled --- .../Producer/FrameworkProducer.swift | 74 +++++++------------ 1 file changed, 27 insertions(+), 47 deletions(-) diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index 08cf49f4..161e6bce 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -16,17 +16,6 @@ struct FrameworkProducer { private let fileSystem: any FileSystem private let toolchainEnvironment: [String: String]? - private func cacheStorages(for actor: Runner.Options.CacheMode.CacheActorKind) -> [any CacheStorage]? { - switch cacheMode { - case .disabled, .project: - return nil - case .storage(let storage, let actors): - return actors.contains(actor) ? [storage] : nil - case .storages(let storages): - return storages.compactMap { $0.actors.contains(actor) ? $0.storage : nil } - } - } - private var shouldGenerateVersionFile: Bool { // cacheMode is not disabled if case .disabled = cacheMode { @@ -99,22 +88,17 @@ struct FrameworkProducer { buildOptions: buildOptionsForProduct ) }) - let pinsStore = try descriptionPackage.workspace.pinsStore.load() + let pinsStore = try descriptionPackage.workspace.pinsStore.load() let cacheSystem = CacheSystem( pinsStore: pinsStore, outputDirectory: outputDir ) - let cacheEnabledTargets: Set - if cacheMode.isConsumingCacheEnabled { - cacheEnabledTargets = await restoreAllAvailableCaches( - availableTargets: Set(allTargets), - cacheSystem: cacheSystem, - readableStorages: cacheStorages(for: .consumer) - ) - } else { - cacheEnabledTargets = [] - } + + let cacheEnabledTargets = await restoreAllAvailableCachesIfNeeded( + availableTargets: Set(allTargets), + cacheSystem: cacheSystem + ) let targetsToBuild = allTargets.subtracting(cacheEnabledTargets) @@ -136,30 +120,41 @@ struct FrameworkProducer { } } - private func restoreAllAvailableCaches( + private func restoreAllAvailableCachesIfNeeded( availableTargets: Set, - cacheSystem: CacheSystem, - readableStorages: [any CacheStorage]? + cacheSystem: CacheSystem ) async -> Set { - guard let readableStorages, !readableStorages.isEmpty else { - // This is for `CacheMode.project`. - // - // In that case just checking whether the valid caches (already built frameworks under the project) + let cacheStorages: [any CacheStorage] + + switch cacheMode { + case .disabled: + return [] + case .project: + // For `.project`, just checking whether the valid caches (already built frameworks under the project) // exist or not (not restoring anything from external locations). return await restoreCachesForTargets( availableTargets, cacheSystem: cacheSystem, cacheStorage: nil ) + case .storage(let storage, let actors): + guard actors.contains(.consumer) else { return [] } + cacheStorages = [storage] + case .storages(let storages): + let storagesWithConsumer = storages.compactMap { storage, actors in + actors.contains(.consumer) ? storage : nil + } + guard !storagesWithConsumer.isEmpty else { return [] } + cacheStorages = storagesWithConsumer } var remainingTargets = availableTargets var restored: Set = [] - for index in readableStorages.indices { - let storage = readableStorages[index] + for index in cacheStorages.indices { + let storage = cacheStorages[index] - if index != readableStorages.startIndex { + if index != cacheStorages.startIndex { logger.info("Falling back to \(storage)", metadata: .color(.green)) } @@ -334,18 +329,3 @@ struct FrameworkProducer { } } } - -extension Runner.Options.CacheMode { - fileprivate var isConsumingCacheEnabled: Bool { - switch self { - case .disabled: - return false - case .project: - return true - case .storage(_, let actors): - return actors.contains(.consumer) - case .storages(let storages): - return storages.contains { $0.actors.contains(.consumer) } - } - } -} From e7b20c5c14545f78842d22221065b560ba373910 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Fri, 8 Nov 2024 16:15:47 +0900 Subject: [PATCH 05/31] Improve logging --- .../Producer/Cache/CacheSystem.swift | 3 ++- .../Producer/FrameworkProducer.swift | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift index c17be5fe..ecc49edc 100644 --- a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift +++ b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift @@ -154,6 +154,7 @@ struct CacheSystem: Sendable { private func cacheFrameworks(_ targets: Set, storage: any CacheStorage) async { let chunked = targets.chunks(ofCount: storage.parallelNumber ?? CacheSystem.defaultParalellNumber) + let storageType = type(of: storage) for chunk in chunked { await withTaskGroup(of: Void.self) { group in for target in chunk { @@ -162,7 +163,7 @@ struct CacheSystem: Sendable { let frameworkPath = outputDirectory.appendingPathComponent(frameworkName) do { logger.info( - "πŸš€ Cache \(frameworkName) to cache storage: \(storage)", + "πŸš€ Cache \(frameworkName) to cache storage: \(storageType)", metadata: .color(.green) ) try await cacheFramework(target, at: frameworkPath, storage: storage) diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index 161e6bce..c91f5316 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -154,8 +154,17 @@ struct FrameworkProducer { for index in cacheStorages.indices { let storage = cacheStorages[index] - if index != cacheStorages.startIndex { - logger.info("Falling back to \(storage)", metadata: .color(.green)) + let logSuffix = "[\(index)] \(type(of: storage))" + if index == cacheStorages.startIndex { + logger.info( + "▢️ Starting restoration with cache storage: \(logSuffix)", + metadata: .color(.green) + ) + } else { + logger.info( + "⏭️ Falling back to next cache storage: \(logSuffix)", + metadata: .color(.green) + ) } let restoredPerStorage = await restoreCachesForTargets( @@ -165,12 +174,18 @@ struct FrameworkProducer { ) restored.formUnion(restoredPerStorage) + logger.info( + "⏸️ Restoration finished with cache storage: \(logSuffix)", + metadata: .color(.green) + ) + remainingTargets.subtract(restoredPerStorage) if remainingTargets.isEmpty { break } } + logger.info("⏹️ Restoration finished", metadata: .color(.green)) return restored } From 0f1c98aa794768d76a2291daec4aa258d659581e Mon Sep 17 00:00:00 2001 From: ikesyo Date: Fri, 8 Nov 2024 16:17:35 +0900 Subject: [PATCH 06/31] debugging --- Sources/ScipioKit/Producer/Cache/CacheSystem.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift index ecc49edc..19fbe9bf 100644 --- a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift +++ b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift @@ -245,7 +245,7 @@ struct CacheSystem: Sendable { buildOptions: buildOptions, clangVersion: clangVersion, xcodeVersion: xcodeVersion, - scipioVersion: currentScipioVersion + scipioVersion: "578ce98d236e79dad3e473cb11153e867be07174" // TODO: revert ) } From 87501535a6e51b3e90857a39af1b6c452a8a1e3d Mon Sep 17 00:00:00 2001 From: ikesyo Date: Mon, 11 Nov 2024 12:32:54 +0900 Subject: [PATCH 07/31] Add test case for storages cache mode --- Tests/ScipioKitTests/RunnerTests.swift | 68 ++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/Tests/ScipioKitTests/RunnerTests.swift b/Tests/ScipioKitTests/RunnerTests.swift index 9eac9380..ad030faf 100644 --- a/Tests/ScipioKitTests/RunnerTests.swift +++ b/Tests/ScipioKitTests/RunnerTests.swift @@ -346,10 +346,12 @@ final class RunnerTests: XCTestCase { XCTFail("Build should be succeeded. \(error.localizedDescription)") } - XCTAssertTrue(fileManager.fileExists(atPath: storageDir.appendingPathComponent("ScipioTesting").path)) + XCTAssertTrue( + fileManager.fileExists(atPath: storageDir.appendingPathComponent("ScipioTesting").path), + "The framework should be cached to the cache storage" + ) let outputFrameworkPath = frameworkOutputDir.appendingPathComponent("ScipioTesting.xcframework") - try self.fileManager.removeItem(atPath: outputFrameworkPath.path) // Fetch from local storage @@ -360,11 +362,71 @@ final class RunnerTests: XCTestCase { XCTFail("Build should be succeeded.") } - XCTAssertTrue(fileManager.fileExists(atPath: storageDir.appendingPathComponent("ScipioTesting").path)) + XCTAssertTrue( + fileManager.fileExists(atPath: outputFrameworkPath.path), + "The framework should be restored from the cache storage" + ) try fileManager.removeItem(at: storageDir) } + func testCacheModeMultipleStorages() async throws { + let storage1CacheDir = tempDir.appending(path: "storage1", directoryHint: .isDirectory) + let storage1 = LocalCacheStorage(cacheDirectory: .custom(storage1CacheDir)) + let storage1Dir = storage1CacheDir.appendingPathComponent("Scipio") + + let storage2CacheDir = tempDir.appending(path: "storage2", directoryHint: .isDirectory) + let storage2 = LocalCacheStorage(cacheDirectory: .custom(storage2CacheDir)) + let storage2Dir = storage2CacheDir.appendingPathComponent("Scipio") + + let runner = Runner( + mode: .prepareDependencies, + options: .init( + shouldOnlyUseVersionsFromResolvedFile: true, + cacheMode: .storages([ + (storage1, [.consumer, .producer] as Set), + (storage2, [.consumer, .producer] as Set), + ]) + ) + ) + do { + try await runner.run(packageDirectory: testPackagePath, + frameworkOutputDir: .custom(frameworkOutputDir)) + } catch { + XCTFail("Build should be succeeded. \(error.localizedDescription)") + } + + // The cache are stored into 2 storages + XCTAssertTrue( + fileManager.fileExists(atPath: storage1Dir.appendingPathComponent("ScipioTesting").path), + "The framework should be cached to the 1st cache storage" + ) + XCTAssertTrue( + fileManager.fileExists(atPath: storage2Dir.appendingPathComponent("ScipioTesting").path), + "The framework should be cached to the 2nd cache storage as well" + ) + + let outputFrameworkPath = frameworkOutputDir.appendingPathComponent("ScipioTesting.xcframework") + try self.fileManager.removeItem(atPath: outputFrameworkPath.path) + + // Remove the storage1's cache so storage2's cache should be used instead + do { + try fileManager.removeItem(at: storage1Dir.appendingPathComponent("ScipioTesting")) + try await runner.run(packageDirectory: testPackagePath, + frameworkOutputDir: .custom(frameworkOutputDir)) + } catch { + XCTFail("Build should be succeeded.") + } + + XCTAssertTrue( + fileManager.fileExists(atPath: outputFrameworkPath.path), + "The framework should be restored from the 2nd cache storage" + ) + + try fileManager.removeItem(at: storage1CacheDir) + try fileManager.removeItem(at: storage2CacheDir) + } + func testExtractBinary() async throws { let runner = Runner( mode: .createPackage, From 8c7c16ee1f347d773066c0bbb0f85276f00fa465 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Mon, 11 Nov 2024 15:03:29 +0900 Subject: [PATCH 08/31] Update `build-pipeline.md` for `.storages` cache mode --- Sources/scipio/scipio.docc/build-pipeline.md | 27 ++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Sources/scipio/scipio.docc/build-pipeline.md b/Sources/scipio/scipio.docc/build-pipeline.md index ce033233..133545e7 100644 --- a/Sources/scipio/scipio.docc/build-pipeline.md +++ b/Sources/scipio/scipio.docc/build-pipeline.md @@ -253,3 +253,30 @@ You can specify it by a second argument of `.storage` cache mode. `producer` is an actor who attempt to save cache to the cache storage. When build artifacts are built, then it try to save them. + +### Use multiple cache storages at the same time + +You can also use the `.storages` cache mode, which accepts multiple cache storages with different sets of cache actors: + +```swift +import ScipioS3Storage + +let s3Storage: some CacheStorage = ScipioS3Storage.S3Storage(config: ...) +let localCacheStorage: some CacheStorage = LocalCacheStorage() +let runner = Runner( + mode: .prepareDependencies, + options: .init( + baseBuildOptions: .init( + buildConfiguration: .release, + isSimulatorSupported: true + ), + cacheMode: .storages([ + (s3Storage, [.consumer] as Set), + (localCacheStorage, [.producer, .consumer] as Set), + ]) + ) +) +``` + +In the sample above, if some frameworks' caches are not found on `s3Storage`, those are tried to be fetched from the next `localCacheStorage` then. The frameworks not found on `localCacheStorage` will be built and cached into it (since the storage is tied to `.producer` actor as well). + From f6cbef149f247b1a09d9966ead372625b298ba59 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Mon, 11 Nov 2024 15:43:10 +0900 Subject: [PATCH 09/31] debugging --- Sources/ScipioKit/Producer/Cache/CacheSystem.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift index 19fbe9bf..094f7c97 100644 --- a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift +++ b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift @@ -245,7 +245,10 @@ struct CacheSystem: Sendable { buildOptions: buildOptions, clangVersion: clangVersion, xcodeVersion: xcodeVersion, - scipioVersion: "578ce98d236e79dad3e473cb11153e867be07174" // TODO: revert + // Making the cache key compatible with 0.24.0 temporarily for easier debugging. + // + // TODO: revert this before merging + scipioVersion: "578ce98d236e79dad3e473cb11153e867be07174" ) } From 413bb1276875911d45f7fed1e11b4fce1760000d Mon Sep 17 00:00:00 2001 From: ikesyo Date: Tue, 12 Nov 2024 12:49:21 +0900 Subject: [PATCH 10/31] Add StorageConfig struct --- .../Producer/FrameworkProducer.swift | 24 +++++++++---------- Sources/ScipioKit/Runner.swift | 14 +++++++++-- Sources/scipio/PrepareCommands.swift | 5 +++- Tests/ScipioKitTests/RunnerTests.swift | 9 ++++--- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index c91f5316..4f5a929e 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -137,12 +137,12 @@ struct FrameworkProducer { cacheSystem: cacheSystem, cacheStorage: nil ) - case .storage(let storage, let actors): - guard actors.contains(.consumer) else { return [] } - cacheStorages = [storage] - case .storages(let storages): - let storagesWithConsumer = storages.compactMap { storage, actors in - actors.contains(.consumer) ? storage : nil + case .storage(let config): + guard config.actors.contains(.consumer) else { return [] } + cacheStorages = [config.storage] + case .storages(let configs): + let storagesWithConsumer = configs.compactMap { cachePolicy in + cachePolicy.actors.contains(.consumer) ? cachePolicy.storage : nil } guard !storagesWithConsumer.isEmpty else { return [] } cacheStorages = storagesWithConsumer @@ -322,13 +322,13 @@ struct FrameworkProducer { // For `.project` which is not tied to any (external) storages, we don't need to do anything. // The built frameworks under the project themselves are treated as valid caches. break - case .storage(let storage, let actors): - if actors.contains(.producer) { - await cacheSystem.cacheFrameworks(targets, storages: [storage]) + case .storage(let config): + if config.actors.contains(.producer) { + await cacheSystem.cacheFrameworks(targets, storages: [config.storage]) } - case .storages(let storages): - let storagesWithProducer = storages.compactMap { storage, actors in - actors.contains(.producer) ? storage : nil + case .storages(let configs): + let storagesWithProducer = configs.compactMap { cachePolicy in + cachePolicy.actors.contains(.producer) ? cachePolicy.storage : nil } if !storagesWithProducer.isEmpty { await cacheSystem.cacheFrameworks(targets, storages: storagesWithProducer) diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift index f618c1eb..8b066cc1 100644 --- a/Sources/ScipioKit/Runner.swift +++ b/Sources/ScipioKit/Runner.swift @@ -209,6 +209,16 @@ extension Runner { } public enum CacheMode: Sendable { + public struct StorageConfig: Sendable { + public let storage: any CacheStorage + public let actors: Set + + public init(storage: any CacheStorage, actors: Set) { + self.storage = storage + self.actors = actors + } + } + public enum CacheActorKind: Sendable { // Save built product to cacheStorage case producer @@ -218,8 +228,8 @@ extension Runner { case disabled case project - case storage(any CacheStorage, Set) - case storages([(storage: any CacheStorage, actors: Set)]) + case storage(StorageConfig) + case storages([StorageConfig]) } public enum PlatformSpecifier: Equatable { diff --git a/Sources/scipio/PrepareCommands.swift b/Sources/scipio/PrepareCommands.swift index 1f64e848..e43f3a1f 100644 --- a/Sources/scipio/PrepareCommands.swift +++ b/Sources/scipio/PrepareCommands.swift @@ -38,7 +38,10 @@ extension Scipio { case .project: runnerCacheMode = .project case .local: - runnerCacheMode = .storage(LocalCacheStorage(), [.consumer, .producer]) + runnerCacheMode = .storage(.init( + storage: LocalCacheStorage(), + actors: [.consumer, .producer] + )) } let runner = Runner( diff --git a/Tests/ScipioKitTests/RunnerTests.swift b/Tests/ScipioKitTests/RunnerTests.swift index ad030faf..485f20f3 100644 --- a/Tests/ScipioKitTests/RunnerTests.swift +++ b/Tests/ScipioKitTests/RunnerTests.swift @@ -336,7 +336,10 @@ final class RunnerTests: XCTestCase { mode: .prepareDependencies, options: .init( shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .storage(storage, [.consumer, .producer]) + cacheMode: .storage(.init( + storage: storage, + actors: [.consumer, .producer] + )) ) ) do { @@ -384,8 +387,8 @@ final class RunnerTests: XCTestCase { options: .init( shouldOnlyUseVersionsFromResolvedFile: true, cacheMode: .storages([ - (storage1, [.consumer, .producer] as Set), - (storage2, [.consumer, .producer] as Set), + .init(storage: storage1, actors: [.consumer, .producer]), + .init(storage: storage2, actors: [.consumer, .producer]), ]) ) ) From c3d9efa90a6112bf2a8ff008f6a5ba440795aa81 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Tue, 12 Nov 2024 12:54:21 +0900 Subject: [PATCH 11/31] Address naming convention issues --- .../Producer/Cache/CacheSystem.swift | 10 +++---- .../Producer/FrameworkProducer.swift | 28 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift index 094f7c97..6d2da628 100644 --- a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift +++ b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift @@ -145,13 +145,13 @@ struct CacheSystem: Sendable { self.fileSystem = fileSystem } - func cacheFrameworks(_ targets: Set, storages: [any CacheStorage]) async { + func cacheFrameworks(_ targets: Set, to storages: [any CacheStorage]) async { for storage in storages { - await cacheFrameworks(targets, storage: storage) + await cacheFrameworks(targets, to: storage) } } - private func cacheFrameworks(_ targets: Set, storage: any CacheStorage) async { + private func cacheFrameworks(_ targets: Set, to storage: any CacheStorage) async { let chunked = targets.chunks(ofCount: storage.parallelNumber ?? CacheSystem.defaultParalellNumber) let storageType = type(of: storage) @@ -166,7 +166,7 @@ struct CacheSystem: Sendable { "πŸš€ Cache \(frameworkName) to cache storage: \(storageType)", metadata: .color(.green) ) - try await cacheFramework(target, at: frameworkPath, storage: storage) + try await cacheFramework(target, at: frameworkPath, to: storage) } catch { logger.warning("⚠️ Can't create caches for \(frameworkPath.path)") } @@ -177,7 +177,7 @@ struct CacheSystem: Sendable { } } - private func cacheFramework(_ target: CacheTarget, at frameworkPath: URL, storage: any CacheStorage) async throws { + private func cacheFramework(_ target: CacheTarget, at frameworkPath: URL, to storage: any CacheStorage) async throws { let cacheKey = try await calculateCacheKey(of: target) try await storage.cacheFramework(frameworkPath, for: cacheKey) diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index 4f5a929e..2523a18e 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -132,10 +132,10 @@ struct FrameworkProducer { case .project: // For `.project`, just checking whether the valid caches (already built frameworks under the project) // exist or not (not restoring anything from external locations). - return await restoreCachesForTargets( - availableTargets, - cacheSystem: cacheSystem, - cacheStorage: nil + return await restoreCaches( + for: availableTargets, + from: nil, + cacheSystem: cacheSystem ) case .storage(let config): guard config.actors.contains(.consumer) else { return [] } @@ -167,10 +167,10 @@ struct FrameworkProducer { ) } - let restoredPerStorage = await restoreCachesForTargets( - remainingTargets, - cacheSystem: cacheSystem, - cacheStorage: storage + let restoredPerStorage = await restoreCaches( + for: remainingTargets, + from: storage, + cacheSystem: cacheSystem ) restored.formUnion(restoredPerStorage) @@ -189,10 +189,10 @@ struct FrameworkProducer { return restored } - private func restoreCachesForTargets( - _ targets: Set, - cacheSystem: CacheSystem, - cacheStorage: (any CacheStorage)? + private func restoreCaches( + for targets: Set, + from cacheStorage: (any CacheStorage)?, + cacheSystem: CacheSystem ) async -> Set { let chunked = targets.chunks(ofCount: cacheStorage?.parallelNumber ?? CacheSystem.defaultParalellNumber) @@ -324,14 +324,14 @@ struct FrameworkProducer { break case .storage(let config): if config.actors.contains(.producer) { - await cacheSystem.cacheFrameworks(targets, storages: [config.storage]) + await cacheSystem.cacheFrameworks(targets, to: [config.storage]) } case .storages(let configs): let storagesWithProducer = configs.compactMap { cachePolicy in cachePolicy.actors.contains(.producer) ? cachePolicy.storage : nil } if !storagesWithProducer.isEmpty { - await cacheSystem.cacheFrameworks(targets, storages: storagesWithProducer) + await cacheSystem.cacheFrameworks(targets, to: storagesWithProducer) } } } From 25f5fcb36bd1eedd6be800691072f48ece9524a0 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Tue, 12 Nov 2024 12:59:36 +0900 Subject: [PATCH 12/31] Replace `case storage` with static func --- Sources/ScipioKit/Producer/FrameworkProducer.swift | 7 ------- Sources/ScipioKit/Runner.swift | 5 ++++- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index 2523a18e..5d666bfd 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -137,9 +137,6 @@ struct FrameworkProducer { from: nil, cacheSystem: cacheSystem ) - case .storage(let config): - guard config.actors.contains(.consumer) else { return [] } - cacheStorages = [config.storage] case .storages(let configs): let storagesWithConsumer = configs.compactMap { cachePolicy in cachePolicy.actors.contains(.consumer) ? cachePolicy.storage : nil @@ -322,10 +319,6 @@ struct FrameworkProducer { // For `.project` which is not tied to any (external) storages, we don't need to do anything. // The built frameworks under the project themselves are treated as valid caches. break - case .storage(let config): - if config.actors.contains(.producer) { - await cacheSystem.cacheFrameworks(targets, to: [config.storage]) - } case .storages(let configs): let storagesWithProducer = configs.compactMap { cachePolicy in cachePolicy.actors.contains(.producer) ? cachePolicy.storage : nil diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift index 8b066cc1..ef46656c 100644 --- a/Sources/ScipioKit/Runner.swift +++ b/Sources/ScipioKit/Runner.swift @@ -228,8 +228,11 @@ extension Runner { case disabled case project - case storage(StorageConfig) case storages([StorageConfig]) + + public static func storage(_ config: StorageConfig) -> Self { + .storages([config]) + } } public enum PlatformSpecifier: Equatable { From 82ec106a5b9df032fa67e057cd5a02623d37408a Mon Sep 17 00:00:00 2001 From: ikesyo Date: Tue, 12 Nov 2024 13:05:04 +0900 Subject: [PATCH 13/31] Replace `case disabled` with static func --- Sources/ScipioKit/Producer/FrameworkProducer.swift | 11 +++++------ Sources/ScipioKit/Runner.swift | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index 5d666bfd..b16a4199 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -18,7 +18,7 @@ struct FrameworkProducer { private var shouldGenerateVersionFile: Bool { // cacheMode is not disabled - if case .disabled = cacheMode { + if case .storages(let configs) = cacheMode, configs.isEmpty { return false } @@ -127,8 +127,6 @@ struct FrameworkProducer { let cacheStorages: [any CacheStorage] switch cacheMode { - case .disabled: - return [] case .project: // For `.project`, just checking whether the valid caches (already built frameworks under the project) // exist or not (not restoring anything from external locations). @@ -138,6 +136,8 @@ struct FrameworkProducer { cacheSystem: cacheSystem ) case .storages(let configs): + guard !configs.isEmpty else { return [] } + let storagesWithConsumer = configs.compactMap { cachePolicy in cachePolicy.actors.contains(.consumer) ? cachePolicy.storage : nil } @@ -312,14 +312,13 @@ struct FrameworkProducer { private func cacheFrameworksIfNeeded(_ targets: Set, cacheSystem: CacheSystem) async { switch cacheMode { - case .disabled: - // no-op - break case .project: // For `.project` which is not tied to any (external) storages, we don't need to do anything. // The built frameworks under the project themselves are treated as valid caches. break case .storages(let configs): + guard !configs.isEmpty else { return } + let storagesWithProducer = configs.compactMap { cachePolicy in cachePolicy.actors.contains(.producer) ? cachePolicy.storage : nil } diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift index ef46656c..6201d4aa 100644 --- a/Sources/ScipioKit/Runner.swift +++ b/Sources/ScipioKit/Runner.swift @@ -226,10 +226,10 @@ extension Runner { case consumer } - case disabled case project case storages([StorageConfig]) + public static let disabled: Self = .storages([]) public static func storage(_ config: StorageConfig) -> Self { .storages([config]) } From 7434f6b173c3f3003a4a72f8cf8eba1d967a1831 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Tue, 12 Nov 2024 16:44:00 +0900 Subject: [PATCH 14/31] Rename to LocalDiskCacheStorage --- ...ocalCacheStorage.swift => LocalDiskCacheStorage.swift} | 2 +- Sources/scipio/PrepareCommands.swift | 2 +- Tests/ScipioKitTests/RunnerTests.swift | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) rename Sources/ScipioKit/Producer/Cache/{LocalCacheStorage.swift => LocalDiskCacheStorage.swift} (98%) diff --git a/Sources/ScipioKit/Producer/Cache/LocalCacheStorage.swift b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift similarity index 98% rename from Sources/ScipioKit/Producer/Cache/LocalCacheStorage.swift rename to Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift index d7ebeba7..c640c1b5 100644 --- a/Sources/ScipioKit/Producer/Cache/LocalCacheStorage.swift +++ b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift @@ -3,7 +3,7 @@ import ScipioStorage import PackageGraph import TSCBasic -public struct LocalCacheStorage: CacheStorage { +public struct LocalDiskCacheStorage: CacheStorage { private let fileSystem: any FileSystem public var parallelNumber: Int? { nil } diff --git a/Sources/scipio/PrepareCommands.swift b/Sources/scipio/PrepareCommands.swift index e43f3a1f..f05d6291 100644 --- a/Sources/scipio/PrepareCommands.swift +++ b/Sources/scipio/PrepareCommands.swift @@ -39,7 +39,7 @@ extension Scipio { runnerCacheMode = .project case .local: runnerCacheMode = .storage(.init( - storage: LocalCacheStorage(), + storage: LocalDiskCacheStorage(), actors: [.consumer, .producer] )) } diff --git a/Tests/ScipioKitTests/RunnerTests.swift b/Tests/ScipioKitTests/RunnerTests.swift index 485f20f3..3753434f 100644 --- a/Tests/ScipioKitTests/RunnerTests.swift +++ b/Tests/ScipioKitTests/RunnerTests.swift @@ -328,8 +328,8 @@ final class RunnerTests: XCTestCase { } } - func testLocalStorage() async throws { - let storage = LocalCacheStorage(cacheDirectory: .custom(tempDir)) + func testLocalDiskCacheStorage() async throws { + let storage = LocalDiskCacheStorage(cacheDirectory: .custom(tempDir)) let storageDir = tempDir.appendingPathComponent("Scipio") let runner = Runner( @@ -375,11 +375,11 @@ final class RunnerTests: XCTestCase { func testCacheModeMultipleStorages() async throws { let storage1CacheDir = tempDir.appending(path: "storage1", directoryHint: .isDirectory) - let storage1 = LocalCacheStorage(cacheDirectory: .custom(storage1CacheDir)) + let storage1 = LocalDiskCacheStorage(cacheDirectory: .custom(storage1CacheDir)) let storage1Dir = storage1CacheDir.appendingPathComponent("Scipio") let storage2CacheDir = tempDir.appending(path: "storage2", directoryHint: .isDirectory) - let storage2 = LocalCacheStorage(cacheDirectory: .custom(storage2CacheDir)) + let storage2 = LocalDiskCacheStorage(cacheDirectory: .custom(storage2CacheDir)) let storage2Dir = storage2CacheDir.appendingPathComponent("Scipio") let runner = Runner( From a451614be4f96b98dc2f0cad0d0cc5b4415257ff Mon Sep 17 00:00:00 2001 From: ikesyo Date: Tue, 12 Nov 2024 16:57:05 +0900 Subject: [PATCH 15/31] Introduce ProjectCacheStorage --- .../ScipioKit/Producer/Cache/ProjectCacheStorage.swift | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 Sources/ScipioKit/Producer/Cache/ProjectCacheStorage.swift diff --git a/Sources/ScipioKit/Producer/Cache/ProjectCacheStorage.swift b/Sources/ScipioKit/Producer/Cache/ProjectCacheStorage.swift new file mode 100644 index 00000000..30a1b348 --- /dev/null +++ b/Sources/ScipioKit/Producer/Cache/ProjectCacheStorage.swift @@ -0,0 +1,10 @@ +import Foundation +import ScipioStorage + +/// The pseudo cache storage for "project cache policy", which treats built frameworks under the project's output directory (e.g. `XCFrameworks`) +/// as valid caches but does not saving / restoring anything. +public struct ProjectCacheStorage: CacheStorage { + public func existsValidCache(for cacheKey: some ScipioStorage.CacheKey) async throws -> Bool { false } + public func fetchArtifacts(for cacheKey: some ScipioStorage.CacheKey, to destinationDir: URL) async throws {} + public func cacheFramework(_ frameworkPath: URL, for cacheKey: some ScipioStorage.CacheKey) async throws {} +} From c22ed6fea76bfa54ad6e178b7f5bcc8d0c0fd3c3 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Tue, 12 Nov 2024 16:57:56 +0900 Subject: [PATCH 16/31] Replace `case project` with static func (using ProjectCacheStorage) --- Sources/ScipioKit/Producer/FrameworkProducer.swift | 14 -------------- Sources/ScipioKit/Runner.swift | 6 +++++- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index c4b2b0c9..c94a5943 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -97,13 +97,6 @@ struct FrameworkProducer { let targetsToBuild: OrderedSet switch cacheMode { - case .project: - let valid = await validateExistingFrameworks( - availableTargets: Set(allTargets), - cacheSystem: cacheSystem - ) - targetsToBuild = allTargets.subtracting(valid) - case .storages(let configs): if configs.isEmpty { // no-op because cache is disabled @@ -208,9 +201,6 @@ struct FrameworkProducer { let cacheStorages: [any CacheStorage] switch cacheMode { - case .project: - // For `.project`, there is nothing to restore from external locations. - return [] case .storages(let configs): guard !configs.isEmpty else { return [] } @@ -368,10 +358,6 @@ struct FrameworkProducer { private func cacheFrameworksIfNeeded(_ targets: Set, cacheSystem: CacheSystem) async { switch cacheMode { - case .project: - // For `.project` which is not tied to any (external) storages, we don't need to do anything. - // The built frameworks under the project themselves are treated as valid caches. - break case .storages(let configs): guard !configs.isEmpty else { return } diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift index 6201d4aa..027be6ab 100644 --- a/Sources/ScipioKit/Runner.swift +++ b/Sources/ScipioKit/Runner.swift @@ -226,10 +226,14 @@ extension Runner { case consumer } - case project case storages([StorageConfig]) public static let disabled: Self = .storages([]) + + public static let project: Self = .storages([ + .init(storage: ProjectCacheStorage(), actors: [.producer]), + ]) + public static func storage(_ config: StorageConfig) -> Self { .storages([config]) } From 0006fbbf81972da79565e74300358a3d46af4e75 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Tue, 12 Nov 2024 17:20:17 +0900 Subject: [PATCH 17/31] Introduce CachePolicy and replace CacheMode with an array of CachePolicy --- .../Producer/FrameworkProducer.swift | 70 ++++++++----------- Sources/ScipioKit/Runner.swift | 42 +++++------ Sources/scipio/CommandType.swift | 20 +++--- Sources/scipio/PrepareCommands.swift | 13 ++-- Tests/ScipioKitTests/IntegrationTests.swift | 2 +- Tests/ScipioKitTests/RunnerTests.swift | 27 ++++--- 6 files changed, 76 insertions(+), 98 deletions(-) diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index c94a5943..17516a65 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -10,15 +10,15 @@ struct FrameworkProducer { private let descriptionPackage: DescriptionPackage private let baseBuildOptions: BuildOptions private let buildOptionsMatrix: [String: BuildOptions] - private let cacheMode: Runner.Options.CacheMode + private let cachePolicies: [Runner.Options.CachePolicy] private let overwrite: Bool private let outputDir: URL private let fileSystem: any FileSystem private let toolchainEnvironment: [String: String]? private var shouldGenerateVersionFile: Bool { - // cacheMode is not disabled - if case .storages(let configs) = cacheMode, configs.isEmpty { + // cache is not disabled + guard !cachePolicies.isEmpty else { return false } @@ -33,7 +33,7 @@ struct FrameworkProducer { descriptionPackage: DescriptionPackage, buildOptions: BuildOptions, buildOptionsMatrix: [String: BuildOptions], - cacheMode: Runner.Options.CacheMode, + cachePolicies: [Runner.Options.CachePolicy], overwrite: Bool, outputDir: URL, toolchainEnvironment: [String: String]? = nil, @@ -42,7 +42,7 @@ struct FrameworkProducer { self.descriptionPackage = descriptionPackage self.baseBuildOptions = buildOptions self.buildOptionsMatrix = buildOptionsMatrix - self.cacheMode = cacheMode + self.cachePolicies = cachePolicies self.overwrite = overwrite self.outputDir = outputDir self.toolchainEnvironment = toolchainEnvironment @@ -96,14 +96,10 @@ struct FrameworkProducer { ) let targetsToBuild: OrderedSet - switch cacheMode { - case .storages(let configs): - if configs.isEmpty { - // no-op because cache is disabled - targetsToBuild = allTargets - break - } - + if cachePolicies.isEmpty { + // no-op because cache is disabled + targetsToBuild = allTargets + } else { let targets = Set(allTargets) // Validate the existing frameworks in `outputDir` before restoration @@ -112,15 +108,14 @@ struct FrameworkProducer { cacheSystem: cacheSystem ) - let storagesWithConsumer = configs.compactMap { config in - config.actors.contains(.consumer) ? config.storage : nil - } + let storagesWithConsumer = cachePolicies.storages(for: .consumer) if storagesWithConsumer.isEmpty { // no-op targetsToBuild = allTargets.subtracting(valid) } else { let restored = await restoreAllAvailableCachesIfNeeded( availableTargets: targets.subtracting(valid), + to: storagesWithConsumer, cacheSystem: cacheSystem ) targetsToBuild = allTargets @@ -196,29 +191,17 @@ struct FrameworkProducer { private func restoreAllAvailableCachesIfNeeded( availableTargets: Set, + to storages: [any CacheStorage], cacheSystem: CacheSystem ) async -> Set { - let cacheStorages: [any CacheStorage] - - switch cacheMode { - case .storages(let configs): - guard !configs.isEmpty else { return [] } - - let storagesWithConsumer = configs.compactMap { config in - config.actors.contains(.consumer) ? config.storage : nil - } - guard !storagesWithConsumer.isEmpty else { return [] } - cacheStorages = storagesWithConsumer - } - var remainingTargets = availableTargets var restored: Set = [] - for index in cacheStorages.indices { - let storage = cacheStorages[index] + for index in storages.indices { + let storage = storages[index] let logSuffix = "[\(index)] \(type(of: storage))" - if index == cacheStorages.startIndex { + if index == storages.startIndex { logger.info( "▢️ Starting restoration with cache storage: \(logSuffix)", metadata: .color(.green) @@ -357,16 +340,9 @@ struct FrameworkProducer { } private func cacheFrameworksIfNeeded(_ targets: Set, cacheSystem: CacheSystem) async { - switch cacheMode { - case .storages(let configs): - guard !configs.isEmpty else { return } - - let storagesWithProducer = configs.compactMap { config in - config.actors.contains(.producer) ? config.storage : nil - } - if !storagesWithProducer.isEmpty { - await cacheSystem.cacheFrameworks(targets, to: storagesWithProducer) - } + let storagesWithProducer = cachePolicies.storages(for: .producer) + if !storagesWithProducer.isEmpty { + await cacheSystem.cacheFrameworks(targets, to: storagesWithProducer) } } @@ -378,3 +354,13 @@ struct FrameworkProducer { } } } + +extension [Runner.Options.CachePolicy] { + fileprivate func storages(for actor: Runner.Options.CachePolicy.CacheActorKind) -> [any CacheStorage] { + reduce(into: []) { result, cachePolicy in + if cachePolicy.actors.contains(actor) { + result.append(cachePolicy.storage) + } + } + } +} diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift index 027be6ab..4b99ce92 100644 --- a/Sources/ScipioKit/Runner.swift +++ b/Sources/ScipioKit/Runner.swift @@ -97,7 +97,7 @@ public struct Runner { descriptionPackage: descriptionPackage, buildOptions: buildOptions, buildOptionsMatrix: buildOptionsMatrix, - cacheMode: options.cacheMode, + cachePolicies: options.cachePolicies, overwrite: options.overwrite, outputDir: outputDir ) @@ -208,17 +208,7 @@ extension Runner { public var buildOptionsMatrix: [String: TargetBuildOptions] } - public enum CacheMode: Sendable { - public struct StorageConfig: Sendable { - public let storage: any CacheStorage - public let actors: Set - - public init(storage: any CacheStorage, actors: Set) { - self.storage = storage - self.actors = actors - } - } - + public struct CachePolicy: Sendable { public enum CacheActorKind: Sendable { // Save built product to cacheStorage case producer @@ -226,17 +216,23 @@ extension Runner { case consumer } - case storages([StorageConfig]) + public let storage: any CacheStorage + public let actors: Set - public static let disabled: Self = .storages([]) + public init(storage: any CacheStorage, actors: Set) { + self.storage = storage + self.actors = actors + } - public static let project: Self = .storages([ - .init(storage: ProjectCacheStorage(), actors: [.producer]), - ]) + public static let project = Self( + storage: ProjectCacheStorage(), + actors: [.producer] + ) - public static func storage(_ config: StorageConfig) -> Self { - .storages([config]) - } + public static let localDisk = Self( + storage: LocalDiskCacheStorage(), + actors: [.producer, .consumer] + ) } public enum PlatformSpecifier: Equatable { @@ -255,7 +251,7 @@ extension Runner { public var buildOptionsContainer: BuildOptionsContainer public var shouldOnlyUseVersionsFromResolvedFile: Bool - public var cacheMode: CacheMode + public var cachePolicies: [CachePolicy] public var overwrite: Bool public var verbose: Bool public var toolchainEnvironment: [String: String]? @@ -264,7 +260,7 @@ extension Runner { baseBuildOptions: BuildOptions = .init(), buildOptionsMatrix: [String: TargetBuildOptions] = [:], shouldOnlyUseVersionsFromResolvedFile: Bool = false, - cacheMode: CacheMode = .project, + cachePolicies: [CachePolicy] = [.project], overwrite: Bool = false, verbose: Bool = false, toolchainEnvironment: [String: String]? = nil @@ -274,7 +270,7 @@ extension Runner { buildOptionsMatrix: buildOptionsMatrix ) self.shouldOnlyUseVersionsFromResolvedFile = shouldOnlyUseVersionsFromResolvedFile - self.cacheMode = cacheMode + self.cachePolicies = cachePolicies self.overwrite = overwrite self.verbose = verbose self.toolchainEnvironment = toolchainEnvironment diff --git a/Sources/scipio/CommandType.swift b/Sources/scipio/CommandType.swift index 3c31d9c2..1869a927 100644 --- a/Sources/scipio/CommandType.swift +++ b/Sources/scipio/CommandType.swift @@ -3,7 +3,7 @@ import ScipioKit enum CommandType { case create(platformSpecifier: Runner.Options.PlatformSpecifier) - case prepare(cacheMode: Runner.Options.CacheMode) + case prepare(cachePolicies: [Runner.Options.CachePolicy]) var mode: Runner.Mode { switch self { @@ -23,12 +23,12 @@ enum CommandType { } } - var cacheMode: Runner.Options.CacheMode { + var cachePolicies: [Runner.Options.CachePolicy] { switch self { case .create: - return .disabled - case .prepare(let cacheMode): - return cacheMode + return [] + case .prepare(let cachePolicies): + return cachePolicies } } } @@ -51,19 +51,19 @@ extension Runner { let runnerOptions = Runner.Options( baseBuildOptions: baseBuildOptions, shouldOnlyUseVersionsFromResolvedFile: buildOptions.shouldOnlyUseVersionsFromResolvedFile, - cacheMode: Self.cacheMode(from: commandType), + cachePolicies: Self.cachePolicies(from: commandType), overwrite: buildOptions.overwrite, verbose: globalOptions.verbose ) self.init(mode: commandType.mode, options: runnerOptions) } - private static func cacheMode(from commandType: CommandType) -> Runner.Options.CacheMode { + private static func cachePolicies(from commandType: CommandType) -> [Runner.Options.CachePolicy] { switch commandType { case .create: - return .disabled - case .prepare(let cacheMode): - return cacheMode + return [] + case .prepare(let cachePolicies): + return cachePolicies } } diff --git a/Sources/scipio/PrepareCommands.swift b/Sources/scipio/PrepareCommands.swift index f05d6291..199cff2b 100644 --- a/Sources/scipio/PrepareCommands.swift +++ b/Sources/scipio/PrepareCommands.swift @@ -31,21 +31,18 @@ extension Scipio { let logLevel: Logger.Level = globalOptions.verbose ? .trace : .info LoggingSystem.bootstrap(logLevel: logLevel) - let runnerCacheMode: Runner.Options.CacheMode + let runnerCachePolicies: [Runner.Options.CachePolicy] switch cachePolicy { case .disabled: - runnerCacheMode = .disabled + runnerCachePolicies = [] case .project: - runnerCacheMode = .project + runnerCachePolicies = [.project] case .local: - runnerCacheMode = .storage(.init( - storage: LocalDiskCacheStorage(), - actors: [.consumer, .producer] - )) + runnerCachePolicies = [.localDisk] } let runner = Runner( - commandType: .prepare(cacheMode: runnerCacheMode), + commandType: .prepare(cachePolicies: runnerCachePolicies), buildOptions: buildOptions, globalOptions: globalOptions ) diff --git a/Tests/ScipioKitTests/IntegrationTests.swift b/Tests/ScipioKitTests/IntegrationTests.swift index 0be120a0..a8f8022e 100644 --- a/Tests/ScipioKitTests/IntegrationTests.swift +++ b/Tests/ScipioKitTests/IntegrationTests.swift @@ -108,7 +108,7 @@ final class IntegrationTests: XCTestCase { ), buildOptionsMatrix: buildOptionsMatrix, shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .disabled, + cachePolicies: [], overwrite: true, verbose: false ) diff --git a/Tests/ScipioKitTests/RunnerTests.swift b/Tests/ScipioKitTests/RunnerTests.swift index 3753434f..93d41b81 100644 --- a/Tests/ScipioKitTests/RunnerTests.swift +++ b/Tests/ScipioKitTests/RunnerTests.swift @@ -306,7 +306,7 @@ final class RunnerTests: XCTestCase { options: .init( baseBuildOptions: .init(enableLibraryEvolution: true), shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .project + cachePolicies: [.project] ) ) do { @@ -336,10 +336,9 @@ final class RunnerTests: XCTestCase { mode: .prepareDependencies, options: .init( shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .storage(.init( - storage: storage, - actors: [.consumer, .producer] - )) + cachePolicies: [ + .init(storage: storage, actors: [.consumer, .producer]), + ] ) ) do { @@ -373,7 +372,7 @@ final class RunnerTests: XCTestCase { try fileManager.removeItem(at: storageDir) } - func testCacheModeMultipleStorages() async throws { + func testMultipleCachePolicies() async throws { let storage1CacheDir = tempDir.appending(path: "storage1", directoryHint: .isDirectory) let storage1 = LocalDiskCacheStorage(cacheDirectory: .custom(storage1CacheDir)) let storage1Dir = storage1CacheDir.appendingPathComponent("Scipio") @@ -386,10 +385,10 @@ final class RunnerTests: XCTestCase { mode: .prepareDependencies, options: .init( shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .storages([ + cachePolicies: [ .init(storage: storage1, actors: [.consumer, .producer]), .init(storage: storage2, actors: [.consumer, .producer]), - ]) + ] ) ) do { @@ -441,7 +440,7 @@ final class RunnerTests: XCTestCase { frameworkType: .dynamic ), shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .project, + cachePolicies: [.project], overwrite: false, verbose: false) ) @@ -468,7 +467,7 @@ final class RunnerTests: XCTestCase { frameworkType: .dynamic ), shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .project, + cachePolicies: [.project], overwrite: false, verbose: false) ) @@ -539,7 +538,7 @@ final class RunnerTests: XCTestCase { enableLibraryEvolution: true ), shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .project, + cachePolicies: [.project], overwrite: false, verbose: false) ) @@ -575,7 +574,7 @@ final class RunnerTests: XCTestCase { ), ], shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .project, + cachePolicies: [.project], overwrite: false, verbose: false) ) @@ -613,7 +612,7 @@ final class RunnerTests: XCTestCase { isSimulatorSupported: true ), shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .disabled + cachePolicies: [] ) ) @@ -656,7 +655,7 @@ final class RunnerTests: XCTestCase { frameworkType: .mergeable ), shouldOnlyUseVersionsFromResolvedFile: true, - cacheMode: .disabled + cachePolicies: [] ) ) From 4d8ababee2400067dbd298ea6ef444d971cf5453 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Tue, 12 Nov 2024 18:05:38 +0900 Subject: [PATCH 18/31] Update documentation --- Sources/scipio/scipio.docc/build-pipeline.md | 20 ++++++++++--------- .../scipio/scipio.docc/using-s3-storage.md | 4 +++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Sources/scipio/scipio.docc/build-pipeline.md b/Sources/scipio/scipio.docc/build-pipeline.md index 133545e7..f59e5547 100644 --- a/Sources/scipio/scipio.docc/build-pipeline.md +++ b/Sources/scipio/scipio.docc/build-pipeline.md @@ -153,7 +153,7 @@ let runner = Runner( ], enableLibraryEvolution: false ), - cacheMode: .project, + cachePolicies: [.project], overwrite: true, verbose: true ) @@ -230,7 +230,9 @@ let runner = Runner( buildConfiguration: .release, isSimulatorSupported: true ), - cacheMode: .storage(s3Storage, [.consumer]) + cachePolicies: [ + .init(storage: s3Storage, actors: [.consumer]), + ] ) ) ``` @@ -246,7 +248,7 @@ You can also implement your custom cache storage by implementing `CacheStorage` There are two cache actors `consumer` and `producer`. -You can specify it by a second argument of `.storage` cache mode. +You can specify it by `Runner.Options.CachePolicy.actors`. `consumer` is an actor who can fetch cache from the cache storage. @@ -254,9 +256,9 @@ You can specify it by a second argument of `.storage` cache mode. When build artifacts are built, then it try to save them. -### Use multiple cache storages at the same time +### Use multiple cache policies at the same time -You can also use the `.storages` cache mode, which accepts multiple cache storages with different sets of cache actors: +You can also use multiple cache policies, which accepts multiple cache storages with different sets of cache actors: ```swift import ScipioS3Storage @@ -270,10 +272,10 @@ let runner = Runner( buildConfiguration: .release, isSimulatorSupported: true ), - cacheMode: .storages([ - (s3Storage, [.consumer] as Set), - (localCacheStorage, [.producer, .consumer] as Set), - ]) + cachePolicies: [ + .init(storage: s3Storage, actors: [.consumer]), + .init(storage: localCacheStorage, actors: [.producer, .consumer]), + ] ) ) ``` diff --git a/Sources/scipio/scipio.docc/using-s3-storage.md b/Sources/scipio/scipio.docc/using-s3-storage.md index a72266ab..40cd6faa 100644 --- a/Sources/scipio/scipio.docc/using-s3-storage.md +++ b/Sources/scipio/scipio.docc/using-s3-storage.md @@ -39,7 +39,9 @@ let runner = Runner( buildConfiguration: .release, isSimulatorSupported: true ), - cacheMode: .storage(s3Storage, [.consumer, .producer]) + cachePolicies: [ + .init(storage: s3Storage, actors: [.consumer, .producer]), + ] ) ) ``` From 76ae5af1afe8dcc43fcdf623f57eadf9af2727ff Mon Sep 17 00:00:00 2001 From: ikesyo Date: Tue, 12 Nov 2024 18:19:59 +0900 Subject: [PATCH 19/31] Avoid direct usages of `type(of: storage)` --- Sources/ScipioKit/Producer/Cache/CacheSystem.swift | 4 ++-- .../ScipioKit/Producer/CacheStorage+DisplayName.swift | 9 +++++++++ Sources/ScipioKit/Producer/FrameworkProducer.swift | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 Sources/ScipioKit/Producer/CacheStorage+DisplayName.swift diff --git a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift index 6d2da628..66f7c123 100644 --- a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift +++ b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift @@ -154,7 +154,7 @@ struct CacheSystem: Sendable { private func cacheFrameworks(_ targets: Set, to storage: any CacheStorage) async { let chunked = targets.chunks(ofCount: storage.parallelNumber ?? CacheSystem.defaultParalellNumber) - let storageType = type(of: storage) + let storageName = storage.displayName for chunk in chunked { await withTaskGroup(of: Void.self) { group in for target in chunk { @@ -163,7 +163,7 @@ struct CacheSystem: Sendable { let frameworkPath = outputDirectory.appendingPathComponent(frameworkName) do { logger.info( - "πŸš€ Cache \(frameworkName) to cache storage: \(storageType)", + "πŸš€ Cache \(frameworkName) to cache storage: \(storageName)", metadata: .color(.green) ) try await cacheFramework(target, at: frameworkPath, to: storage) diff --git a/Sources/ScipioKit/Producer/CacheStorage+DisplayName.swift b/Sources/ScipioKit/Producer/CacheStorage+DisplayName.swift new file mode 100644 index 00000000..2f520647 --- /dev/null +++ b/Sources/ScipioKit/Producer/CacheStorage+DisplayName.swift @@ -0,0 +1,9 @@ +import ScipioStorage + +extension CacheStorage { + /// The display name of the cache storage used for logging purpose + var displayName: String { + // TODO: Define the property as CacheStorage's requirement in scipio-cache-storage + "\(type(of: self))" + } +} diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index 17516a65..8120f35d 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -200,7 +200,7 @@ struct FrameworkProducer { for index in storages.indices { let storage = storages[index] - let logSuffix = "[\(index)] \(type(of: storage))" + let logSuffix = "[\(index)] \(storage.displayName)" if index == storages.startIndex { logger.info( "▢️ Starting restoration with cache storage: \(logSuffix)", From ff1d483154f5dfcdbdc084f05875695f0e57c75c Mon Sep 17 00:00:00 2001 From: ikesyo Date: Thu, 14 Nov 2024 10:48:15 +0900 Subject: [PATCH 20/31] Make LocalDiskCacheStorage and ProjectCacheStorage internal --- .../Cache/LocalDiskCacheStorage.swift | 32 ++++++++----------- .../Producer/Cache/ProjectCacheStorage.swift | 8 ++--- Sources/ScipioKit/Runner.swift | 17 ++++++++-- Sources/scipio/scipio.docc/build-pipeline.md | 5 ++- Tests/ScipioKitTests/RunnerTests.swift | 6 ++-- 5 files changed, 37 insertions(+), 31 deletions(-) diff --git a/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift index c640c1b5..98606d00 100644 --- a/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift +++ b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift @@ -3,37 +3,33 @@ import ScipioStorage import PackageGraph import TSCBasic -public struct LocalDiskCacheStorage: CacheStorage { +struct LocalDiskCacheStorage: CacheStorage { private let fileSystem: any FileSystem - public var parallelNumber: Int? { nil } + var parallelNumber: Int? { nil } enum Error: Swift.Error { case cacheDirectoryIsNotFound } - public enum CacheDirectory: Sendable { - case system - case custom(URL) - } - - private let cacheDirectroy: CacheDirectory - - public init(cacheDirectory: CacheDirectory = .system, fileSystem: FileSystem = localFileSystem) { - self.cacheDirectroy = cacheDirectory + private let baseURL: URL? + + /// - Parameters: + /// - baseURL: The base url for the local disk cache. When it is nil, the system cache directory (`~/Library/Caches`) will be used. + init(baseURL: URL? = nil, fileSystem: FileSystem = localFileSystem) { + self.baseURL = baseURL self.fileSystem = fileSystem } private func buildBaseDirectoryPath() throws -> URL { let cacheDir: URL - switch cacheDirectroy { - case .system: + if let baseURL { + cacheDir = baseURL + } else { guard let systemCacheDir = fileSystem.cachesDirectory else { throw Error.cacheDirectoryIsNotFound } cacheDir = systemCacheDir.asURL - case .custom(let customPath): - cacheDir = customPath } return cacheDir.appendingPathComponent("Scipio") } @@ -51,7 +47,7 @@ public struct LocalDiskCacheStorage: CacheStorage { .appendingPathComponent(xcFrameworkFileName(for: cacheKey)) } - public func existsValidCache(for cacheKey: some CacheKey) async -> Bool { + func existsValidCache(for cacheKey: some CacheKey) async -> Bool { do { let xcFrameworkPath = try cacheFrameworkPath(for: cacheKey) return fileSystem.exists(xcFrameworkPath.absolutePath) @@ -60,7 +56,7 @@ public struct LocalDiskCacheStorage: CacheStorage { } } - public func cacheFramework(_ frameworkPath: URL, for cacheKey: some CacheKey) async { + func cacheFramework(_ frameworkPath: URL, for cacheKey: some CacheKey) async { do { let destination = try cacheFrameworkPath(for: cacheKey) let directoryPath = destination.deletingLastPathComponent() @@ -72,7 +68,7 @@ public struct LocalDiskCacheStorage: CacheStorage { } } - public func fetchArtifacts(for cacheKey: some CacheKey, to destinationDir: URL) async throws { + func fetchArtifacts(for cacheKey: some CacheKey, to destinationDir: URL) async throws { let source = try cacheFrameworkPath(for: cacheKey) let destination = destinationDir.appendingPathComponent(xcFrameworkFileName(for: cacheKey)) try fileSystem.copy(from: source.absolutePath, to: destination.absolutePath) diff --git a/Sources/ScipioKit/Producer/Cache/ProjectCacheStorage.swift b/Sources/ScipioKit/Producer/Cache/ProjectCacheStorage.swift index 30a1b348..b2179ac6 100644 --- a/Sources/ScipioKit/Producer/Cache/ProjectCacheStorage.swift +++ b/Sources/ScipioKit/Producer/Cache/ProjectCacheStorage.swift @@ -3,8 +3,8 @@ import ScipioStorage /// The pseudo cache storage for "project cache policy", which treats built frameworks under the project's output directory (e.g. `XCFrameworks`) /// as valid caches but does not saving / restoring anything. -public struct ProjectCacheStorage: CacheStorage { - public func existsValidCache(for cacheKey: some ScipioStorage.CacheKey) async throws -> Bool { false } - public func fetchArtifacts(for cacheKey: some ScipioStorage.CacheKey, to destinationDir: URL) async throws {} - public func cacheFramework(_ frameworkPath: URL, for cacheKey: some ScipioStorage.CacheKey) async throws {} +struct ProjectCacheStorage: CacheStorage { + func existsValidCache(for cacheKey: some ScipioStorage.CacheKey) async throws -> Bool { false } + func fetchArtifacts(for cacheKey: some ScipioStorage.CacheKey, to destinationDir: URL) async throws {} + func cacheFramework(_ frameworkPath: URL, for cacheKey: some ScipioStorage.CacheKey) async throws {} } diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift index 4b99ce92..20c05127 100644 --- a/Sources/ScipioKit/Runner.swift +++ b/Sources/ScipioKit/Runner.swift @@ -224,15 +224,26 @@ extension Runner { self.actors = actors } - public static let project = Self( + /// The cache policy which treats built frameworks under the project's output directory (e.g. `XCFrameworks`) + /// as valid caches, but does not saving to / restoring from any external locations. + public static let project: Self = Self( storage: ProjectCacheStorage(), actors: [.producer] ) - public static let localDisk = Self( - storage: LocalDiskCacheStorage(), + /// The cache policy for saving to and restoring from the system cache directory `~/Library/Caches/Scipio`. + public static let localDisk: Self = Self( + storage: LocalDiskCacheStorage(baseURL: nil), actors: [.producer, .consumer] ) + + /// The cache policy for saving to and restoring from the custom cache directory `baseURL.appendingPath("Scipio")`. + public static func localDisk(baseURL: URL) -> Self { + Self( + storage: LocalDiskCacheStorage(baseURL: baseURL), + actors: [.producer, .consumer] + ) + } } public enum PlatformSpecifier: Equatable { diff --git a/Sources/scipio/scipio.docc/build-pipeline.md b/Sources/scipio/scipio.docc/build-pipeline.md index f59e5547..bba9b950 100644 --- a/Sources/scipio/scipio.docc/build-pipeline.md +++ b/Sources/scipio/scipio.docc/build-pipeline.md @@ -264,7 +264,6 @@ You can also use multiple cache policies, which accepts multiple cache storages import ScipioS3Storage let s3Storage: some CacheStorage = ScipioS3Storage.S3Storage(config: ...) -let localCacheStorage: some CacheStorage = LocalCacheStorage() let runner = Runner( mode: .prepareDependencies, options: .init( @@ -274,11 +273,11 @@ let runner = Runner( ), cachePolicies: [ .init(storage: s3Storage, actors: [.consumer]), - .init(storage: localCacheStorage, actors: [.producer, .consumer]), + .localDisk, ] ) ) ``` -In the sample above, if some frameworks' caches are not found on `s3Storage`, those are tried to be fetched from the next `localCacheStorage` then. The frameworks not found on `localCacheStorage` will be built and cached into it (since the storage is tied to `.producer` actor as well). +In the sample above, if some frameworks' caches are not found on `s3Storage`, those are tried to be fetched from the next `.localDisk` cache policie's storage then. The frameworks not found on the storage of `.localDisk` cache policy will be built and cached into it (since the storage is tied to `.producer` actor). diff --git a/Tests/ScipioKitTests/RunnerTests.swift b/Tests/ScipioKitTests/RunnerTests.swift index 93d41b81..bfd689aa 100644 --- a/Tests/ScipioKitTests/RunnerTests.swift +++ b/Tests/ScipioKitTests/RunnerTests.swift @@ -329,7 +329,7 @@ final class RunnerTests: XCTestCase { } func testLocalDiskCacheStorage() async throws { - let storage = LocalDiskCacheStorage(cacheDirectory: .custom(tempDir)) + let storage = LocalDiskCacheStorage(baseURL: tempDir) let storageDir = tempDir.appendingPathComponent("Scipio") let runner = Runner( @@ -374,11 +374,11 @@ final class RunnerTests: XCTestCase { func testMultipleCachePolicies() async throws { let storage1CacheDir = tempDir.appending(path: "storage1", directoryHint: .isDirectory) - let storage1 = LocalDiskCacheStorage(cacheDirectory: .custom(storage1CacheDir)) + let storage1 = LocalDiskCacheStorage(baseURL: storage1CacheDir) let storage1Dir = storage1CacheDir.appendingPathComponent("Scipio") let storage2CacheDir = tempDir.appending(path: "storage2", directoryHint: .isDirectory) - let storage2 = LocalDiskCacheStorage(cacheDirectory: .custom(storage2CacheDir)) + let storage2 = LocalDiskCacheStorage(baseURL: storage2CacheDir) let storage2Dir = storage2CacheDir.appendingPathComponent("Scipio") let runner = Runner( From 4712f89bfd15af0f35fd276d48f699396359ebc9 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Thu, 14 Nov 2024 11:26:09 +0900 Subject: [PATCH 21/31] Use some over any for arguments --- Sources/ScipioKit/Runner.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift index 20c05127..c0b7e953 100644 --- a/Sources/ScipioKit/Runner.swift +++ b/Sources/ScipioKit/Runner.swift @@ -219,7 +219,7 @@ extension Runner { public let storage: any CacheStorage public let actors: Set - public init(storage: any CacheStorage, actors: Set) { + public init(storage: some CacheStorage, actors: Set) { self.storage = storage self.actors = actors } From 3d333e6b68687f12053e0bd42c91f0fa9960d919 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Thu, 14 Nov 2024 11:31:44 +0900 Subject: [PATCH 22/31] [CI] macos-15 image is required to use Xcode 16 --- .github/workflows/docc.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/tests.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docc.yml b/.github/workflows/docc.yml index ada030ae..359d3ed6 100644 --- a/.github/workflows/docc.yml +++ b/.github/workflows/docc.yml @@ -8,7 +8,7 @@ env: DEVELOPER_DIR: "/Applications/Xcode_16.0.app/Contents/Developer" jobs: DocC: - runs-on: macos-14 + runs-on: macos-15 steps: - uses: actions/checkout@v4 - name: Build DocC diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index fb10205f..3d19536c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,7 +14,7 @@ jobs: xcode_version: ["16.0"] env: DEVELOPER_DIR: "/Applications/Xcode_${{ matrix.xcode_version }}.app/Contents/Developer" - runs-on: macos-14 + runs-on: macos-15 steps: - uses: actions/checkout@v4 - name: Install SwiftLint diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3b49b94b..431da157 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ env: jobs: release: name: Build and Upload Artifact Bundle - runs-on: macos-14 + runs-on: macos-15 steps: - uses: actions/checkout@v4 - name: Resolve Dependencies diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 36202247..8a4f7577 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: - "16.0" # 6.0 env: DEVELOPER_DIR: "/Applications/Xcode_${{ matrix.xcode_version }}.app/Contents/Developer" - runs-on: macos-14 + runs-on: macos-15 steps: - name: Get swift version run: swift --version From 0c81a2ea1d46f032bfcf86eb263e6909f6657bb7 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Thu, 14 Nov 2024 11:36:57 +0900 Subject: [PATCH 23/31] Add info log when a cache is not found when restoring as well as `.succeeded` and `.failed` cases --- Sources/ScipioKit/Producer/FrameworkProducer.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index 8120f35d..83396157 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -297,6 +297,7 @@ struct FrameworkProducer { } return false case .noCache: + logger.info("ℹ️ Cache not found for \(frameworkName) (\(expectedCacheKeyHash)) from cache storage.", metadata: .color(.green)) return false } } From d71053bccc48bbf09317256e7c82f3ed523dddeb Mon Sep 17 00:00:00 2001 From: ikesyo Date: Thu, 14 Nov 2024 11:44:08 +0900 Subject: [PATCH 24/31] Fix format --- Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift index 98606d00..0bca5e47 100644 --- a/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift +++ b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift @@ -13,7 +13,7 @@ struct LocalDiskCacheStorage: CacheStorage { } private let baseURL: URL? - + /// - Parameters: /// - baseURL: The base url for the local disk cache. When it is nil, the system cache directory (`~/Library/Caches`) will be used. init(baseURL: URL? = nil, fileSystem: FileSystem = localFileSystem) { From ed52625d5d524a035ecfc822c417e9c4618caeb8 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Thu, 14 Nov 2024 15:43:15 +0900 Subject: [PATCH 25/31] Use some over any for arguments --- Sources/ScipioKit/Producer/Cache/CacheSystem.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift index 66f7c123..aa4df34b 100644 --- a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift +++ b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift @@ -151,7 +151,7 @@ struct CacheSystem: Sendable { } } - private func cacheFrameworks(_ targets: Set, to storage: any CacheStorage) async { + private func cacheFrameworks(_ targets: Set, to storage: some CacheStorage) async { let chunked = targets.chunks(ofCount: storage.parallelNumber ?? CacheSystem.defaultParalellNumber) let storageName = storage.displayName @@ -215,7 +215,7 @@ struct CacheSystem: Sendable { case noCache } - func restoreCacheIfPossible(target: CacheTarget, storage: any CacheStorage) async -> RestoreResult { + func restoreCacheIfPossible(target: CacheTarget, storage: some CacheStorage) async -> RestoreResult { do { let cacheKey = try await calculateCacheKey(of: target) if try await storage.existsValidCache(for: cacheKey) { From 4de26b54e534e8a470a6f54f094311eb584fbc03 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Thu, 14 Nov 2024 15:43:37 +0900 Subject: [PATCH 26/31] Add inline comment --- Sources/ScipioKit/Producer/FrameworkProducer.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift index 83396157..a4b50ae1 100644 --- a/Sources/ScipioKit/Producer/FrameworkProducer.swift +++ b/Sources/ScipioKit/Producer/FrameworkProducer.swift @@ -226,6 +226,7 @@ struct FrameworkProducer { ) remainingTargets.subtract(restoredPerStorage) + // If all frameworks are successfully restored, we don't need to proceed to next cache storage. if remainingTargets.isEmpty { break } From bced0e2fd89be39129f7a1a1e6bedd02602654d1 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Thu, 14 Nov 2024 15:59:15 +0900 Subject: [PATCH 27/31] Reduce duplication --- .../ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift | 2 +- Sources/ScipioKit/Runner.swift | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift index 0bca5e47..4a14f40e 100644 --- a/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift +++ b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift @@ -16,7 +16,7 @@ struct LocalDiskCacheStorage: CacheStorage { /// - Parameters: /// - baseURL: The base url for the local disk cache. When it is nil, the system cache directory (`~/Library/Caches`) will be used. - init(baseURL: URL? = nil, fileSystem: FileSystem = localFileSystem) { + init(baseURL: URL?, fileSystem: FileSystem = localFileSystem) { self.baseURL = baseURL self.fileSystem = fileSystem } diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift index c0b7e953..d4c4529f 100644 --- a/Sources/ScipioKit/Runner.swift +++ b/Sources/ScipioKit/Runner.swift @@ -232,13 +232,14 @@ extension Runner { ) /// The cache policy for saving to and restoring from the system cache directory `~/Library/Caches/Scipio`. - public static let localDisk: Self = Self( - storage: LocalDiskCacheStorage(baseURL: nil), - actors: [.producer, .consumer] - ) + public static let localDisk: Self = _localDisk(baseURL: nil) /// The cache policy for saving to and restoring from the custom cache directory `baseURL.appendingPath("Scipio")`. public static func localDisk(baseURL: URL) -> Self { + _localDisk(baseURL: baseURL) + } + + private static func _localDisk(baseURL: URL?) -> Self { Self( storage: LocalDiskCacheStorage(baseURL: baseURL), actors: [.producer, .consumer] From dfe443c70bd84f8d728ee33e1a5ddb76812a9a3d Mon Sep 17 00:00:00 2001 From: ikesyo Date: Thu, 14 Nov 2024 16:14:23 +0900 Subject: [PATCH 28/31] Revert "Reduce duplication" This reverts commit bced0e2fd89be39129f7a1a1e6bedd02602654d1. --- .../ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift | 2 +- Sources/ScipioKit/Runner.swift | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift index 4a14f40e..0bca5e47 100644 --- a/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift +++ b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift @@ -16,7 +16,7 @@ struct LocalDiskCacheStorage: CacheStorage { /// - Parameters: /// - baseURL: The base url for the local disk cache. When it is nil, the system cache directory (`~/Library/Caches`) will be used. - init(baseURL: URL?, fileSystem: FileSystem = localFileSystem) { + init(baseURL: URL? = nil, fileSystem: FileSystem = localFileSystem) { self.baseURL = baseURL self.fileSystem = fileSystem } diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift index d4c4529f..c0b7e953 100644 --- a/Sources/ScipioKit/Runner.swift +++ b/Sources/ScipioKit/Runner.swift @@ -232,14 +232,13 @@ extension Runner { ) /// The cache policy for saving to and restoring from the system cache directory `~/Library/Caches/Scipio`. - public static let localDisk: Self = _localDisk(baseURL: nil) + public static let localDisk: Self = Self( + storage: LocalDiskCacheStorage(baseURL: nil), + actors: [.producer, .consumer] + ) /// The cache policy for saving to and restoring from the custom cache directory `baseURL.appendingPath("Scipio")`. public static func localDisk(baseURL: URL) -> Self { - _localDisk(baseURL: baseURL) - } - - private static func _localDisk(baseURL: URL?) -> Self { Self( storage: LocalDiskCacheStorage(baseURL: baseURL), actors: [.producer, .consumer] From acdd25136186506bbdb0a8ab955e99187b33c3e7 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Thu, 14 Nov 2024 16:26:55 +0900 Subject: [PATCH 29/31] Reduce duplication --- .../Producer/Cache/LocalDiskCacheStorage.swift | 2 +- Sources/ScipioKit/Runner.swift | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift index 0bca5e47..4a14f40e 100644 --- a/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift +++ b/Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift @@ -16,7 +16,7 @@ struct LocalDiskCacheStorage: CacheStorage { /// - Parameters: /// - baseURL: The base url for the local disk cache. When it is nil, the system cache directory (`~/Library/Caches`) will be used. - init(baseURL: URL? = nil, fileSystem: FileSystem = localFileSystem) { + init(baseURL: URL?, fileSystem: FileSystem = localFileSystem) { self.baseURL = baseURL self.fileSystem = fileSystem } diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift index c0b7e953..6d11b87d 100644 --- a/Sources/ScipioKit/Runner.swift +++ b/Sources/ScipioKit/Runner.swift @@ -224,6 +224,10 @@ extension Runner { self.actors = actors } + private init(_ storage: LocalDiskCacheStorage) { + self.init(storage: storage, actors: [.producer, .consumer]) + } + /// The cache policy which treats built frameworks under the project's output directory (e.g. `XCFrameworks`) /// as valid caches, but does not saving to / restoring from any external locations. public static let project: Self = Self( @@ -232,17 +236,11 @@ extension Runner { ) /// The cache policy for saving to and restoring from the system cache directory `~/Library/Caches/Scipio`. - public static let localDisk: Self = Self( - storage: LocalDiskCacheStorage(baseURL: nil), - actors: [.producer, .consumer] - ) + public static let localDisk: Self = Self(LocalDiskCacheStorage(baseURL: nil)) /// The cache policy for saving to and restoring from the custom cache directory `baseURL.appendingPath("Scipio")`. public static func localDisk(baseURL: URL) -> Self { - Self( - storage: LocalDiskCacheStorage(baseURL: baseURL), - actors: [.producer, .consumer] - ) + Self(LocalDiskCacheStorage(baseURL: baseURL)) } } From afafa8dbf3f8590ad0c4af57dad3ae66e1da6e11 Mon Sep 17 00:00:00 2001 From: ikesyo Date: Thu, 14 Nov 2024 17:39:03 +0900 Subject: [PATCH 30/31] Revert "debugging" This reverts commit 0f1c98aa794768d76a2291daec4aa258d659581e and f6cbef149f247b1a09d9966ead372625b298ba59. --- Sources/ScipioKit/Producer/Cache/CacheSystem.swift | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift index aa4df34b..27230098 100644 --- a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift +++ b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift @@ -245,10 +245,7 @@ struct CacheSystem: Sendable { buildOptions: buildOptions, clangVersion: clangVersion, xcodeVersion: xcodeVersion, - // Making the cache key compatible with 0.24.0 temporarily for easier debugging. - // - // TODO: revert this before merging - scipioVersion: "578ce98d236e79dad3e473cb11153e867be07174" + scipioVersion: currentScipioVersion ) } From 2720cde7a2c1f7e1d3e8d8afb310fc783cd30d5f Mon Sep 17 00:00:00 2001 From: ikesyo Date: Fri, 15 Nov 2024 17:07:50 +0900 Subject: [PATCH 31/31] Add `[Runner.Options.CachePolicy].disabled` --- Sources/ScipioKit/Runner.swift | 4 ++++ Sources/scipio/CommandType.swift | 4 ++-- Sources/scipio/PrepareCommands.swift | 2 +- Tests/ScipioKitTests/IntegrationTests.swift | 2 +- Tests/ScipioKitTests/RunnerTests.swift | 4 ++-- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift index 6d11b87d..e8636272 100644 --- a/Sources/ScipioKit/Runner.swift +++ b/Sources/ScipioKit/Runner.swift @@ -397,3 +397,7 @@ extension Runner.Options.BuildOptionsContainer { } } } + +extension [Runner.Options.CachePolicy] { + public static let disabled: Self = [] +} diff --git a/Sources/scipio/CommandType.swift b/Sources/scipio/CommandType.swift index 1869a927..7d176532 100644 --- a/Sources/scipio/CommandType.swift +++ b/Sources/scipio/CommandType.swift @@ -26,7 +26,7 @@ enum CommandType { var cachePolicies: [Runner.Options.CachePolicy] { switch self { case .create: - return [] + return .disabled case .prepare(let cachePolicies): return cachePolicies } @@ -61,7 +61,7 @@ extension Runner { private static func cachePolicies(from commandType: CommandType) -> [Runner.Options.CachePolicy] { switch commandType { case .create: - return [] + return .disabled case .prepare(let cachePolicies): return cachePolicies } diff --git a/Sources/scipio/PrepareCommands.swift b/Sources/scipio/PrepareCommands.swift index 199cff2b..3f307bf6 100644 --- a/Sources/scipio/PrepareCommands.swift +++ b/Sources/scipio/PrepareCommands.swift @@ -34,7 +34,7 @@ extension Scipio { let runnerCachePolicies: [Runner.Options.CachePolicy] switch cachePolicy { case .disabled: - runnerCachePolicies = [] + runnerCachePolicies = .disabled case .project: runnerCachePolicies = [.project] case .local: diff --git a/Tests/ScipioKitTests/IntegrationTests.swift b/Tests/ScipioKitTests/IntegrationTests.swift index a8f8022e..81dece9d 100644 --- a/Tests/ScipioKitTests/IntegrationTests.swift +++ b/Tests/ScipioKitTests/IntegrationTests.swift @@ -108,7 +108,7 @@ final class IntegrationTests: XCTestCase { ), buildOptionsMatrix: buildOptionsMatrix, shouldOnlyUseVersionsFromResolvedFile: true, - cachePolicies: [], + cachePolicies: .disabled, overwrite: true, verbose: false ) diff --git a/Tests/ScipioKitTests/RunnerTests.swift b/Tests/ScipioKitTests/RunnerTests.swift index bfd689aa..80cd084e 100644 --- a/Tests/ScipioKitTests/RunnerTests.swift +++ b/Tests/ScipioKitTests/RunnerTests.swift @@ -612,7 +612,7 @@ final class RunnerTests: XCTestCase { isSimulatorSupported: true ), shouldOnlyUseVersionsFromResolvedFile: true, - cachePolicies: [] + cachePolicies: .disabled ) ) @@ -655,7 +655,7 @@ final class RunnerTests: XCTestCase { frameworkType: .mergeable ), shouldOnlyUseVersionsFromResolvedFile: true, - cachePolicies: [] + cachePolicies: .disabled ) )