From 471d7b224d3fcfebfdf010c76745d0b44e45dcf9 Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Thu, 2 May 2024 16:03:29 +0100 Subject: [PATCH] delete old stale plugin from cache folder (#209) --- main.go | 12 +- runner/plugin.go | 17 +++ sysutil/filesystem.go | 72 +++++++++++ sysutil/filesystem_test.go | 256 ++++++++++++++++++++++++++++++------- 4 files changed, 301 insertions(+), 56 deletions(-) diff --git a/main.go b/main.go index c453762f..23273644 100644 --- a/main.go +++ b/main.go @@ -436,17 +436,15 @@ preferences: {} // since /tmp will be mounted on PV we need to do manual clean up // on restart. appData will be excluded from this clean up func cleanupTmpDir() { - fileDescriptors, err := os.ReadDir(os.TempDir()) + err := sysutil.RemoveDirContentsIf( + os.TempDir(), + func(path string, fi os.FileInfo) (bool, error) { + return fi.Name() != appData, nil + }) if err != nil { fmt.Printf("unable to cleanup %s Error: %v\n", os.TempDir(), err) return } - - for _, fd := range fileDescriptors { - if fd.Name() != appData { - sysutil.RemoveAll(path.Join(os.TempDir(), fd.Name())) - } - } } func applyGitDefaults(c *cli.Context, mirrorConf mirror.RepoPoolConfig) mirror.RepoPoolConfig { diff --git a/runner/plugin.go b/runner/plugin.go index 98a610c9..665a8df4 100644 --- a/runner/plugin.go +++ b/runner/plugin.go @@ -6,11 +6,13 @@ import ( "os" "path" "sync" + "time" "github.com/utilitywarehouse/terraform-applier/sysutil" ) var pluginCacheMain = "plugin-cache-main" +var stalePluginTimeout = 7 * 24 * time.Hour // The plugin cache directory is not guaranteed to be concurrency safe. // https://github.com/hashicorp/terraform/issues/31964 @@ -29,10 +31,25 @@ type pluginCache struct { func newPluginCache(log *slog.Logger, root string) (*pluginCache, error) { main := path.Join(root, pluginCacheMain) + // crate main plugin cache dir if not exit if err := os.MkdirAll(main, defaultDirMode); err != nil { return nil, fmt.Errorf("unable to create main cache dir err:%w", err) } + // clean up plugin cache dir in case its an old one + err := sysutil.RemoveDirContentsRecursiveIf(main, + func(path string, fi os.FileInfo) (bool, error) { + // delete all plugin/dir older then stalePluginTimeout + if time.Since(fi.ModTime()) > stalePluginTimeout { + log.Info("clearing stale plugin path", "path", path) + return true, nil + } + return false, nil + }) + if err != nil { + log.Error("unable to clean up main plugin cache dir", "err", err) + } + return &pluginCache{ &sync.RWMutex{}, log, diff --git a/sysutil/filesystem.go b/sysutil/filesystem.go index 52fb55f2..4defb77d 100644 --- a/sysutil/filesystem.go +++ b/sysutil/filesystem.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "sync" ) @@ -114,3 +115,74 @@ func CopyDir(src string, dst string, withReplace bool) error { return nil } + +func IsDirEmpty(path string) (bool, error) { + dirents, err := os.ReadDir(path) + if err != nil { + return false, err + } + return len(dirents) == 0, nil +} + +func RemoveDirContentsRecursiveIf(dir string, fn func(path string, fi os.FileInfo) (bool, error)) error { + var errs []error + + // check if any file/dir needs to be removed from current dir + if err := RemoveDirContentsIf(dir, fn); err != nil { + errs = append(errs, err) + } + + // read current dir and check sub directories + dirEnts, err := os.ReadDir(dir) + if err != nil { + return err + } + + for _, fi := range dirEnts { + if !fi.IsDir() { + continue + } + p := filepath.Join(dir, fi.Name()) + if err := RemoveDirContentsRecursiveIf(p, fn); err != nil { + errs = append(errs, err) + } + } + + if len(errs) != 0 { + return fmt.Errorf("%s", errs) + } + + return nil +} + +// RemoveDirContentsIf iterated the specified dir and removes entries +// if given function returns true for the given entry +func RemoveDirContentsIf(dir string, fn func(path string, fi os.FileInfo) (bool, error)) error { + dirEnts, err := os.ReadDir(dir) + if err != nil { + return err + } + + // Save errors until the end. + var errs []error + for _, fi := range dirEnts { + p := filepath.Join(dir, fi.Name()) + stat, err := os.Stat(p) + if err != nil { + return err + } + if shouldDelete, err := fn(p, stat); err != nil { + return err + } else if !shouldDelete { + continue + } + if err := RemoveAll(p); err != nil { + errs = append(errs, err) + } + } + + if len(errs) != 0 { + return fmt.Errorf("%s", errs) + } + return nil +} diff --git a/sysutil/filesystem_test.go b/sysutil/filesystem_test.go index b684e60f..ff42571f 100644 --- a/sysutil/filesystem_test.go +++ b/sysutil/filesystem_test.go @@ -3,23 +3,20 @@ package sysutil import ( "os" "path" + "path/filepath" "testing" ) func TestCopyDirWithReplace(t *testing.T) { // Create a temporary directory for testing. - testTmp, err := os.MkdirTemp("", "tf-applier-copy-dir-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(testTmp) + testTmp := t.TempDir() srcDir := path.Join(testTmp, "src") dstDir := path.Join(testTmp, "dst") // Create some test files. writeFile := func(name string, content []byte) { - err = os.WriteFile(name, content, 0644) + err := os.WriteFile(name, content, 0644) if err != nil { t.Fatalf("error creating file: %v", err) } @@ -50,54 +47,35 @@ func TestCopyDirWithReplace(t *testing.T) { } // Copy the directory. - err = CopyDir(srcDir, dstDir, true) + err := CopyDir(srcDir, dstDir, true) if err != nil { t.Fatalf("error copying directory: %v", err) } - - verify := func(name string, want []byte) { - _, err := os.Stat(name) - if err != nil { - t.Fatalf("error verifying %s: %v", name, err) - } - got, err := os.ReadFile(name) - if err != nil { - t.Fatalf("error reading %s: %v", name, err) - } - if string(got) != string(want) { - t.Fatalf("file content mismatch name:%s got:%s want:%s", name, got, want) - } - } - // verify dst file structure for _, dir := range dirs { for _, nDir := range nestedDirs { for _, file := range files { name := path.Join(dstDir, dir, nDir, file) - verify(name, []byte("src file contents")) + assertFileContent(t, name, []byte("src file contents")) } } for _, file := range files { name := path.Join(dstDir, dir, file) - verify(name, []byte("src file contents")) + assertFileContent(t, name, []byte("src file contents")) } } } func TestCopyDirWithoutReplace(t *testing.T) { // Create a temporary directory for testing. - testTmp, err := os.MkdirTemp("", "tf-applier-copy-dir-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(testTmp) + testTmp := t.TempDir() srcDir := path.Join(testTmp, "src") dstDir := path.Join(testTmp, "dst") // Create some test files. writeFile := func(name string, content []byte) { - err = os.WriteFile(name, content, 0644) + err := os.WriteFile(name, content, 0644) if err != nil { t.Fatalf("error creating file: %v", err) } @@ -144,44 +122,224 @@ func TestCopyDirWithoutReplace(t *testing.T) { } // Copy the directory. - err = CopyDir(srcDir, dstDir, false) + err := CopyDir(srcDir, dstDir, false) if err != nil { t.Fatalf("error copying directory: %v", err) } - verify := func(name string, want []byte) { - _, err := os.Stat(name) - if err != nil { - t.Fatalf("error verifying %s: %v", name, err) - } - got, err := os.ReadFile(name) - if err != nil { - t.Fatalf("error reading %s: %v", name, err) - } - if string(got) != string(want) { - t.Fatalf("file content mismatch name:%s got:%s want:%s", name, got, want) - } - } - // verify dst file structure for _, dir := range dirs { for _, nDir := range nestedDirs { for i, file := range files { name := path.Join(dstDir, dir, nDir, file) if i == 0 { - verify(name, []byte("dst file contents")) + assertFileContent(t, name, []byte("dst file contents")) } else { - verify(name, []byte("src file contents")) + assertFileContent(t, name, []byte("src file contents")) } } } for i, file := range files { name := path.Join(dstDir, dir, file) if i == 0 { - verify(name, []byte("dst file contents")) + assertFileContent(t, name, []byte("dst file contents")) } else { - verify(name, []byte("src file contents")) + assertFileContent(t, name, []byte("src file contents")) + } + } + } +} + +func TestRemoveDirContentsRecursiveIf(t *testing.T) { + // Create a temporary directory for testing. + testTmp := t.TempDir() + + target := path.Join(testTmp, "src") + + // Create some test files. + writeFile := func(name string, content []byte) { + err := os.WriteFile(name, content, 0644) + if err != nil { + t.Fatalf("error creating file: %v", err) + } + } + + files := []string{"file1", "file2"} + dirs := []string{"subDir1", "subDir2"} + nestedDirs := []string{"nestedDir1", "nestedDir2"} + + // create source file structure + for _, dir := range dirs { + for _, nDir := range nestedDirs { + p := path.Join(target, dir, nDir) + if err := os.MkdirAll(p, 0700); err != nil { + t.Fatal(err) + } + + for _, file := range files { + name := path.Join(target, dir, nDir, file) + writeFile(name, []byte("src file contents")) } } + + for _, file := range files { + name := path.Join(target, dir, file) + writeFile(name, []byte("src file contents")) + } + } + + // delete all file2 + err := RemoveDirContentsRecursiveIf(target, + func(path string, fi os.FileInfo) (bool, error) { return fi.Name() == "file2", nil }) + if err != nil { + t.Fatalf("error removing contents: %v", err) + } + + // verify file1 exits and file2 deleted + for _, dir := range dirs { + for _, nDir := range nestedDirs { + assertFileContent(t, path.Join(target, dir, nDir, "file1"), []byte("src file contents")) + assertMissing(t, path.Join(target, dir, nDir, "file2")) + } + assertFileContent(t, path.Join(target, dir, "file1"), []byte("src file contents")) + assertMissing(t, path.Join(target, dir, "file2")) + } + + // delete all + err = RemoveDirContentsRecursiveIf(target, + func(path string, fi os.FileInfo) (bool, error) { return true, nil }) + if err != nil { + t.Fatalf("error removing contents: %v", err) + } + + // target dir should be empty + if empty, err := IsDirEmpty(target); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if !empty { + t.Errorf("expected %q to be deemed not-empty", target) + } +} + +func Test_removeDirContentsIf(t *testing.T) { + tempRoot := t.TempDir() + + // create target folder with some files + target := filepath.Join(tempRoot, "target") + if err := os.Mkdir(target, 0755); err != nil { + t.Fatalf("failed to make a temp subdir: %v", err) + } + for _, file := range []string{"a", "b", "c"} { + path := filepath.Join(target, file) + if err := os.WriteFile(path, []byte{}, 0755); err != nil { + t.Fatalf("failed to write a file: %v", err) + } + } + + // should delete everything form the target dir + err := RemoveDirContentsIf(target, func(path string, fi os.FileInfo) (bool, error) { + return true, nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // target dir should be empty + if empty, err := IsDirEmpty(target); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if !empty { + t.Errorf("expected %q to be deemed not-empty", target) + } + + // add more files and dir + for _, file := range []string{"a1", "b2", "c2", "d2"} { + path := filepath.Join(target, file) + if err := os.WriteFile(path, []byte{}, 0755); err != nil { + t.Fatalf("failed to write a file: %v", err) + } + } + if err := os.Mkdir(filepath.Join(target, "Dirs"), 0755); err != nil { + t.Fatalf("failed to make a subdir: %v", err) + } + if err := os.Mkdir(filepath.Join(target, ".git"), 0755); err != nil { + t.Fatalf("failed to make a subdir: %v", err) + } + + // should delete everything except file b2 + if err := RemoveDirContentsIf( + target, + func(path string, fi os.FileInfo) (bool, error) { return fi.Name() != "b2", nil }, + ); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if _, err := os.Stat(filepath.Join(target, "b2")); err != nil { + t.Fatalf("failed to read %q : %v", filepath.Join(target, "b2"), err) + } + + // folder test + dirTarget := filepath.Join(tempRoot, "target2") + // create folder F1/F1.1 + // create folder F2/F2.1 + if err := os.MkdirAll(path.Join(dirTarget, "F1", "F1.1"), 0700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(path.Join(dirTarget, "F2", "F2.1"), 0700); err != nil { + t.Fatal(err) + } + + // should delete everything except file F2 + if err := RemoveDirContentsIf( + dirTarget, + func(path string, fi os.FileInfo) (bool, error) { return fi.Name() != "F2", nil }, + ); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if _, err := os.Stat(filepath.Join(dirTarget, "F2")); err != nil { + t.Fatalf("failed to read %q : %v", filepath.Join(dirTarget, "F2"), err) + } + + // should delete everything form the target dir + err = RemoveDirContentsIf(dirTarget, func(path string, fi os.FileInfo) (bool, error) { + return true, nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // target dir should be empty + if empty, err := IsDirEmpty(dirTarget); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if !empty { + t.Errorf("expected %q to be deemed not-empty", dirTarget) + } +} + +func assertFileContent(t *testing.T, path string, want []byte) { + t.Helper() + + _, err := os.Stat(path) + if err != nil { + t.Fatalf("error verifying %s: %v", path, err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("error reading %s: %v", path, err) + } + if string(got) != string(want) { + t.Fatalf("file content mismatch name:%s got:%s want:%s", path, got, want) + } +} + +func assertMissing(t *testing.T, path string) { + t.Helper() + + _, err := os.Stat(path) + if err != nil && !os.IsNotExist(err) { + t.Fatalf("unable to read existing file error: %v", err) + } else if os.IsNotExist(err) { + return + } else { + t.Errorf("file (%s) exits but it should not", path) } }