From 7868d4025d6911526984daf157cafe29a95e5681 Mon Sep 17 00:00:00 2001 From: zsolt-marta-bitrise Date: Tue, 17 Sep 2024 16:38:26 +0200 Subject: [PATCH 1/3] feat: ACI-2805 Add flag to skip SPM --- cmd/saveXcodeDerivedDataFiles.go | 5 ++++ cmd/saveXcodeDerivedDataFiles_test.go | 2 +- internal/xcode/file_group_info.go | 42 ++++++++++++++++++++++----- internal/xcode/metadata.go | 4 +++ 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/cmd/saveXcodeDerivedDataFiles.go b/cmd/saveXcodeDerivedDataFiles.go index ab2679c..58da808 100644 --- a/cmd/saveXcodeDerivedDataFiles.go +++ b/cmd/saveXcodeDerivedDataFiles.go @@ -39,6 +39,7 @@ var saveXcodeDerivedDataFilesCmd = &cobra.Command{ ddPath, _ := cmd.Flags().GetString("deriveddata-path") xcodeCachePath, _ := cmd.Flags().GetString("xcodecache-path") followSymlinks, _ := cmd.Flags().GetBool("follow-symlinks") + skipSPM, _ := cmd.Flags().GetBool("skip-spm") tracker := xcode.NewDefaultStepTracker("save-xcode-build-cache", os.Getenv, logger) defer tracker.Wait() @@ -58,6 +59,7 @@ var saveXcodeDerivedDataFilesCmd = &cobra.Command{ ddPath, xcodeCachePath, followSymlinks, + skipSPM, logger, tracker, startT, @@ -99,6 +101,7 @@ func init() { } saveXcodeDerivedDataFilesCmd.Flags().String("xcodecache-path", "", "Path to the Xcode cache directory folder to be saved. If not set, it will not be uploaded.") saveXcodeDerivedDataFilesCmd.Flags().Bool("follow-symlinks", false, "Follow symlinks when calculating metadata and save referenced files to the cache (default: false)") + saveXcodeDerivedDataFilesCmd.Flags().Bool("skip-spm", false, "Skip saving files under \"DerivedData/*/SourcePackages\", i.e. skip SPM dependencies. Consider enabling this flag if using SPM cache steps. Default: false") } func saveXcodeDerivedDataFilesCmdFn(ctx context.Context, @@ -109,6 +112,7 @@ func saveXcodeDerivedDataFilesCmdFn(ctx context.Context, derivedDataPath, xcodeCachePath string, followSymlinks bool, + skipSPM bool, logger log.Logger, tracker xcode.StepAnalyticsTracker, startT time.Time, @@ -150,6 +154,7 @@ func saveXcodeDerivedDataFilesCmdFn(ctx context.Context, XcodeCacheDirPath: xcodeCachePath, CacheKey: cacheKey, FollowSymlinks: followSymlinks, + SkipSPM: skipSPM, }, envProvider, logger) if err != nil { return op, fmt.Errorf("create metadata: %w", err) diff --git a/cmd/saveXcodeDerivedDataFiles_test.go b/cmd/saveXcodeDerivedDataFiles_test.go index 365a13a..7093e00 100644 --- a/cmd/saveXcodeDerivedDataFiles_test.go +++ b/cmd/saveXcodeDerivedDataFiles_test.go @@ -36,7 +36,7 @@ func Test_saveXcodeDerivedDataFilesCmdFn(t *testing.T) { envVars := createEnvProvider(map[string]string{ "BITRISEIO_BITRISE_SERVICES_ACCESS_TOKEN": "ServiceAccessTokenValue", }) - _, err := saveXcodeDerivedDataFilesCmdFn(context.Background(), common.CacheAuthConfig{}, "", "", "", "", "", false, mockLogger, mockTracker, time.Now(), envVars) + _, err := saveXcodeDerivedDataFilesCmdFn(context.Background(), common.CacheAuthConfig{}, "", "", "", "", "", false, false, mockLogger, mockTracker, time.Now(), envVars) // then require.EqualError(t, err, "get cache key: cache key is required if BITRISE_GIT_BRANCH env var is not set") diff --git a/internal/xcode/file_group_info.go b/internal/xcode/file_group_info.go index e8f6551..a16e9f9 100644 --- a/internal/xcode/file_group_info.go +++ b/internal/xcode/file_group_info.go @@ -16,6 +16,8 @@ import ( "io" "syscall" + "strings" + "github.com/bitrise-io/go-utils/v2/log" ) @@ -86,7 +88,11 @@ func (mc *fileGroupInfoCollector) isSeen(path string) bool { return mc.seen[path] } -func collectFileGroupInfo(cacheDirPath string, collectAttributes, followSymlinks bool, logger log.Logger) (FileGroupInfo, error) { +func collectFileGroupInfo(cacheDirPath string, + collectAttributes, + followSymlinks bool, + skipSPM bool, + logger log.Logger) (FileGroupInfo, error) { var dd FileGroupInfo fgi := fileGroupInfoCollector{ @@ -120,7 +126,7 @@ func collectFileGroupInfo(cacheDirPath string, collectAttributes, followSymlinks if !filepath.IsAbs(path) { path = filepath.Join(cacheDirPath, path) } - if err := collectFileMetadata(path, inf, inf.IsDir(), &fgi, collectAttributes, followSymlinks, logger); err != nil { + if err := collectFileMetadata(cacheDirPath, path, inf, inf.IsDir(), &fgi, collectAttributes, followSymlinks, skipSPM, logger); err != nil { logger.Errorf("Failed to collect metadata: %s", err) } }(d) @@ -145,7 +151,11 @@ func collectFileGroupInfo(cacheDirPath string, collectAttributes, followSymlinks } // nolint:wrapcheck -func followSymlink(path string, target string, fgi *fileGroupInfoCollector, followSymlinks bool, logger log.Logger) error { +func followSymlink(rootPath, path, target string, + fgi *fileGroupInfoCollector, + followSymlinks, + skipSPM bool, + logger log.Logger) error { if !followSymlinks { logger.Debugf("Skipping symbolic link: %s", path) @@ -172,7 +182,7 @@ func followSymlink(path string, target string, fgi *fileGroupInfoCollector, foll }) if !stat.IsDir() { - return collectFileMetadata(target, stat, false, fgi, false, followSymlinks, logger) + return collectFileMetadata(rootPath, target, stat, false, fgi, false, followSymlinks, skipSPM, logger) } logger.Debugf("Symlink target is a directory, walking it: %s", target) @@ -191,11 +201,17 @@ func followSymlink(path string, target string, fgi *fileGroupInfoCollector, foll path = filepath.Join(target, path) } - return collectFileMetadata(path, inf, inf.IsDir(), fgi, false, followSymlinks, logger) + return collectFileMetadata(target, path, inf, inf.IsDir(), fgi, false, followSymlinks, skipSPM, logger) }) } -func collectFileMetadata(path string, fileInfo fs.FileInfo, isDirectory bool, fgi *fileGroupInfoCollector, collectAttributes, followSymlinks bool, logger log.Logger) error { +func collectFileMetadata( + rootPath, path string, + fileInfo fs.FileInfo, + isDirectory bool, + fgi *fileGroupInfoCollector, + collectAttributes, followSymlinks, skipSPM bool, + logger log.Logger) error { if fgi.isSeen(path) { logger.Debugf("Skipping path %s, already seen", path) @@ -211,6 +227,18 @@ func collectFileMetadata(path string, fileInfo fs.FileInfo, isDirectory bool, fg return nil } + if skipSPM { + relPath, _ := filepath.Rel(rootPath, path) + parts := strings.Split(filepath.ToSlash(relPath), "/") + + // Checking for */SourcePackages/* under the DeriveData directory + if len(parts) > 2 && parts[1] == "SourcePackages" { + logger.Debugf("Skipping SPM package: %s", path) + + return nil + } + } + isSymlink := fileInfo.Mode()&os.ModeSymlink != 0 if isSymlink { @@ -223,7 +251,7 @@ func collectFileMetadata(path string, fileInfo fs.FileInfo, isDirectory bool, fg target = filepath.Join(filepath.Dir(path), target) } - return followSymlink(path, target, fgi, followSymlinks, logger) + return followSymlink(rootPath, path, target, fgi, followSymlinks, skipSPM, logger) } file, err := os.Open(path) diff --git a/internal/xcode/metadata.go b/internal/xcode/metadata.go index 3dfa2e3..f208950 100644 --- a/internal/xcode/metadata.go +++ b/internal/xcode/metadata.go @@ -33,6 +33,7 @@ type CreateMetadataParams struct { XcodeCacheDirPath string CacheKey string FollowSymlinks bool + SkipSPM bool } func CreateMetadata(params CreateMetadataParams, envProvider func(string) string, logger log.Logger) (*Metadata, error) { @@ -43,6 +44,7 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string projectFiles, err := collectFileGroupInfo(params.ProjectRootDirPath, true, params.FollowSymlinks, + params.SkipSPM, logger) if err != nil { return nil, fmt.Errorf("calculate project files info: %w", err) @@ -53,6 +55,7 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string derivedData, err = collectFileGroupInfo(params.DerivedDataPath, false, params.FollowSymlinks, + params.SkipSPM, logger) if err != nil { return nil, fmt.Errorf("calculate derived data info: %w", err) @@ -64,6 +67,7 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string xcodeCacheDir, err = collectFileGroupInfo(params.XcodeCacheDirPath, false, params.FollowSymlinks, + params.SkipSPM, logger) if err != nil { return nil, fmt.Errorf("calculate xcode cache dir info: %w", err) From 1550bf6cca8572d071e726f2ce57e4d56fd6cd7f Mon Sep 17 00:00:00 2001 From: zsolt-marta-bitrise Date: Tue, 17 Sep 2024 18:26:25 +0200 Subject: [PATCH 2/3] feat: ACI-2805 Only skip in DD dir, skip if SourcePackage is the root --- internal/xcode/file_group_info.go | 5 +++-- internal/xcode/metadata.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/xcode/file_group_info.go b/internal/xcode/file_group_info.go index a16e9f9..11d0f1f 100644 --- a/internal/xcode/file_group_info.go +++ b/internal/xcode/file_group_info.go @@ -231,8 +231,9 @@ func collectFileMetadata( relPath, _ := filepath.Rel(rootPath, path) parts := strings.Split(filepath.ToSlash(relPath), "/") - // Checking for */SourcePackages/* under the DeriveData directory - if len(parts) > 2 && parts[1] == "SourcePackages" { + // Checking for SourcePackages/* or */SourcePackages/* under the directory + if len(parts) >= 1 && parts[0] == "SourcePackages" || + len(parts) >= 2 && parts[1] == "SourcePackages" { logger.Debugf("Skipping SPM package: %s", path) return nil diff --git a/internal/xcode/metadata.go b/internal/xcode/metadata.go index f208950..be0e584 100644 --- a/internal/xcode/metadata.go +++ b/internal/xcode/metadata.go @@ -44,7 +44,7 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string projectFiles, err := collectFileGroupInfo(params.ProjectRootDirPath, true, params.FollowSymlinks, - params.SkipSPM, + false, logger) if err != nil { return nil, fmt.Errorf("calculate project files info: %w", err) From b91639437d24d1c4ef891112811c24775e2f1362 Mon Sep 17 00:00:00 2001 From: zsolt-marta-bitrise Date: Tue, 17 Sep 2024 18:42:59 +0200 Subject: [PATCH 3/3] fix: ACI-2805 Remove debug log, too verbose --- internal/xcode/file_group_info.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/xcode/file_group_info.go b/internal/xcode/file_group_info.go index 11d0f1f..27ecbf3 100644 --- a/internal/xcode/file_group_info.go +++ b/internal/xcode/file_group_info.go @@ -234,8 +234,6 @@ func collectFileMetadata( // Checking for SourcePackages/* or */SourcePackages/* under the directory if len(parts) >= 1 && parts[0] == "SourcePackages" || len(parts) >= 2 && parts[1] == "SourcePackages" { - logger.Debugf("Skipping SPM package: %s", path) - return nil } }