From 61b255c8e8803245ab03b1039b2d93a5ccebc24c Mon Sep 17 00:00:00 2001 From: james pickett Date: Fri, 5 Jan 2024 12:01:34 -0800 Subject: [PATCH 1/8] add func to exclude launcher db from time machine --- .../timemachine/timemachine_darwin.go | 36 +++++++++++++++++++ .../internal/timemachine/timemachine_other.go | 16 +++++++++ cmd/launcher/launcher.go | 2 ++ 3 files changed, 54 insertions(+) create mode 100644 cmd/launcher/internal/timemachine/timemachine_darwin.go create mode 100644 cmd/launcher/internal/timemachine/timemachine_other.go diff --git a/cmd/launcher/internal/timemachine/timemachine_darwin.go b/cmd/launcher/internal/timemachine/timemachine_darwin.go new file mode 100644 index 000000000..38fa6f53f --- /dev/null +++ b/cmd/launcher/internal/timemachine/timemachine_darwin.go @@ -0,0 +1,36 @@ +//go:build darwin +// +build darwin + +package timemachine + +import ( + "context" + "log/slog" + + "github.com/kolide/launcher/ee/agent/types" + "github.com/kolide/launcher/ee/allowedcmd" +) + +// ExcludeLauncherDB adds the launcher db to the time machine exclusions for +// darwin and is noop for other oses +func ExcludeLauncherDB(ctx context.Context, k types.Knapsack) { + dbPath := k.BboltDB().Path() + cmd, err := allowedcmd.Tmutil(ctx, "addexclusion", dbPath) + if err != nil { + k.Slogger().Log(ctx, slog.LevelError, + "failed to add launcher db to time machine exclusions", + "err", err, + "path", dbPath, + ) + return + } + + if err := cmd.Run(); err != nil { + k.Slogger().Log(ctx, slog.LevelError, + "failed to add launcher db to time machine exclusions", + "err", err, + "path", dbPath, + ) + return + } +} diff --git a/cmd/launcher/internal/timemachine/timemachine_other.go b/cmd/launcher/internal/timemachine/timemachine_other.go new file mode 100644 index 000000000..bb7b3e8da --- /dev/null +++ b/cmd/launcher/internal/timemachine/timemachine_other.go @@ -0,0 +1,16 @@ +//go:build !darwin +// +build !darwin + +package timemachine + +import ( + "context" + + "github.com/kolide/launcher/ee/agent/types" +) + +// ExcludeLauncherDB adds the launcher db to the time machine exclusions for +// darwin and is noop for other oses +func ExcludeLauncherDB(_ context.Context, _ types.Knapsack) { + // do nothing, no time machine on non darwin +} diff --git a/cmd/launcher/launcher.go b/cmd/launcher/launcher.go index c66b6e560..e383a7c98 100644 --- a/cmd/launcher/launcher.go +++ b/cmd/launcher/launcher.go @@ -25,6 +25,7 @@ import ( "github.com/kolide/kit/ulid" "github.com/kolide/kit/version" "github.com/kolide/launcher/cmd/launcher/internal" + "github.com/kolide/launcher/cmd/launcher/internal/timemachine" "github.com/kolide/launcher/cmd/launcher/internal/updater" "github.com/kolide/launcher/ee/agent" "github.com/kolide/launcher/ee/agent/flags" @@ -171,6 +172,7 @@ func runLauncher(ctx context.Context, cancel func(), slogger, systemSlogger *mul k := knapsack.New(stores, flagController, db, slogger, systemSlogger) go runOsqueryVersionCheck(ctx, logger, k.LatestOsquerydPath(ctx)) + go timemachine.ExcludeLauncherDB(ctx, k) if k.Debug() { // If we're in debug mode, then we assume we want to echo _all_ logs to stderr. From 96fd2e8c5a470e7c7dca78f6a391c7fd54848934 Mon Sep 17 00:00:00 2001 From: james pickett Date: Fri, 5 Jan 2024 12:39:37 -0800 Subject: [PATCH 2/8] move timemachine to ee/agent --- cmd/launcher/launcher.go | 2 +- .../internal => ee/agent}/timemachine/timemachine_darwin.go | 0 .../internal => ee/agent}/timemachine/timemachine_other.go | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename {cmd/launcher/internal => ee/agent}/timemachine/timemachine_darwin.go (100%) rename {cmd/launcher/internal => ee/agent}/timemachine/timemachine_other.go (100%) diff --git a/cmd/launcher/launcher.go b/cmd/launcher/launcher.go index ec3b98c5f..42baa5dcb 100644 --- a/cmd/launcher/launcher.go +++ b/cmd/launcher/launcher.go @@ -25,7 +25,6 @@ import ( "github.com/kolide/kit/ulid" "github.com/kolide/kit/version" "github.com/kolide/launcher/cmd/launcher/internal" - "github.com/kolide/launcher/cmd/launcher/internal/timemachine" "github.com/kolide/launcher/cmd/launcher/internal/updater" "github.com/kolide/launcher/ee/agent" "github.com/kolide/launcher/ee/agent/flags" @@ -33,6 +32,7 @@ import ( "github.com/kolide/launcher/ee/agent/startupsettings" "github.com/kolide/launcher/ee/agent/storage" agentbbolt "github.com/kolide/launcher/ee/agent/storage/bbolt" + "github.com/kolide/launcher/ee/agent/timemachine" "github.com/kolide/launcher/ee/control/actionqueue" "github.com/kolide/launcher/ee/control/consumers/acceleratecontrolconsumer" "github.com/kolide/launcher/ee/control/consumers/flareconsumer" diff --git a/cmd/launcher/internal/timemachine/timemachine_darwin.go b/ee/agent/timemachine/timemachine_darwin.go similarity index 100% rename from cmd/launcher/internal/timemachine/timemachine_darwin.go rename to ee/agent/timemachine/timemachine_darwin.go diff --git a/cmd/launcher/internal/timemachine/timemachine_other.go b/ee/agent/timemachine/timemachine_other.go similarity index 100% rename from cmd/launcher/internal/timemachine/timemachine_other.go rename to ee/agent/timemachine/timemachine_other.go From 7b5be2d6484123896bb4924c7502f90b19a5cd90 Mon Sep 17 00:00:00 2001 From: james pickett Date: Fri, 5 Jan 2024 14:18:13 -0800 Subject: [PATCH 3/8] better error message wording --- ee/agent/timemachine/timemachine_darwin.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ee/agent/timemachine/timemachine_darwin.go b/ee/agent/timemachine/timemachine_darwin.go index 38fa6f53f..2d8e2f699 100644 --- a/ee/agent/timemachine/timemachine_darwin.go +++ b/ee/agent/timemachine/timemachine_darwin.go @@ -18,7 +18,7 @@ func ExcludeLauncherDB(ctx context.Context, k types.Knapsack) { cmd, err := allowedcmd.Tmutil(ctx, "addexclusion", dbPath) if err != nil { k.Slogger().Log(ctx, slog.LevelError, - "failed to add launcher db to time machine exclusions", + "could not create tmutil command to add launcher db to time machine exclusions", "err", err, "path", dbPath, ) @@ -27,7 +27,7 @@ func ExcludeLauncherDB(ctx context.Context, k types.Knapsack) { if err := cmd.Run(); err != nil { k.Slogger().Log(ctx, slog.LevelError, - "failed to add launcher db to time machine exclusions", + "running command to add launcher db to time machine exclusions", "err", err, "path", dbPath, ) From 9aae794476447045bb919be4d5a8487325781692 Mon Sep 17 00:00:00 2001 From: james pickett Date: Wed, 10 Jan 2024 12:58:13 -0800 Subject: [PATCH 4/8] pass slogger instead of logger --- cmd/launcher/launcher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/launcher/launcher.go b/cmd/launcher/launcher.go index 0168caf1d..6038aad4c 100644 --- a/cmd/launcher/launcher.go +++ b/cmd/launcher/launcher.go @@ -178,8 +178,8 @@ func runLauncher(ctx context.Context, cancel func(), multiSlogger, systemMultiSl k := knapsack.New(stores, flagController, db, multiSlogger, systemMultiSlogger) // reassign slogger to knapsack slogger to get launcher run id added to slogger slogger = k.Slogger() - - go runOsqueryVersionCheck(ctx, logger, k.LatestOsquerydPath(ctx)) + + go runOsqueryVersionCheck(ctx, slogger, k.LatestOsquerydPath(ctx)) go timemachine.ExcludeLauncherDB(ctx, k) if k.Debug() { From e35d272b3720cf9f910c60b7bf2e7aa8d7564beb Mon Sep 17 00:00:00 2001 From: james pickett Date: Wed, 10 Jan 2024 16:08:32 -0800 Subject: [PATCH 5/8] glob for files paths, add test --- cmd/launcher/launcher.go | 2 +- ee/agent/timemachine/timemachine_darwin.go | 65 +++++++++++-- .../timemachine/timemachine_darwin_test.go | 93 +++++++++++++++++++ ee/agent/timemachine/timemachine_other.go | 4 +- 4 files changed, 152 insertions(+), 12 deletions(-) create mode 100644 ee/agent/timemachine/timemachine_darwin_test.go diff --git a/cmd/launcher/launcher.go b/cmd/launcher/launcher.go index 6038aad4c..6d1bf52c1 100644 --- a/cmd/launcher/launcher.go +++ b/cmd/launcher/launcher.go @@ -180,7 +180,7 @@ func runLauncher(ctx context.Context, cancel func(), multiSlogger, systemMultiSl slogger = k.Slogger() go runOsqueryVersionCheck(ctx, slogger, k.LatestOsquerydPath(ctx)) - go timemachine.ExcludeLauncherDB(ctx, k) + go timemachine.AddExclusions(ctx, k) if k.Debug() { // If we're in debug mode, then we assume we want to echo _all_ logs to stderr. diff --git a/ee/agent/timemachine/timemachine_darwin.go b/ee/agent/timemachine/timemachine_darwin.go index 2d8e2f699..fe431235f 100644 --- a/ee/agent/timemachine/timemachine_darwin.go +++ b/ee/agent/timemachine/timemachine_darwin.go @@ -6,31 +6,78 @@ package timemachine import ( "context" "log/slog" + "path/filepath" "github.com/kolide/launcher/ee/agent/types" "github.com/kolide/launcher/ee/allowedcmd" ) -// ExcludeLauncherDB adds the launcher db to the time machine exclusions for +// AddExclusions adds specific launcher time machine exclusions for // darwin and is noop for other oses -func ExcludeLauncherDB(ctx context.Context, k types.Knapsack) { - dbPath := k.BboltDB().Path() - cmd, err := allowedcmd.Tmutil(ctx, "addexclusion", dbPath) +func AddExclusions(ctx context.Context, k types.Knapsack) { + /* + example of root dir: + augeas-lenses/ menu.json + debug-2024-01-09T21-15-14.055.json.gz menu_template.json + debug-2024-01-09T21-20-54.621.json.gz metadata.json + debug-2024-01-09T21-23-58.097.json.gz metadata.plist + debug-2024-01-10T18-49-33.610.json.gz osquery.autoload + debug.json osquery.db/ + desktop_501/ osquery.pid + desktop_503/ osquery.sock + kolide.png osquery.sock.51807 + kv.sqlite osquery.sock.63071 + launcher-tuf/ osqueryd-tuf/ + launcher-version-1.4.1-4-gdb7106f tuf/ + launcher.db updates/ + launcher.pid + */ + + exclusionPatterns := []string{ + "*.json", + "*.json.gz", + "*.db", + "*.sqlite", + "desktop_*", + "*.pid", + "augeas-lenses", + "*.plist", + "osquery*", + } + + var exclusionPaths []string + for _, pattern := range exclusionPatterns { + matches, err := filepath.Glob(filepath.Join(k.RootDirectory(), pattern)) + + if err != nil { + k.Slogger().Log(ctx, slog.LevelError, + "could not create glob for launcher machine exclusions", + "err", err, + ) + + continue + } + + exclusionPaths = append(exclusionPaths, matches...) + } + + cmd, err := allowedcmd.Tmutil(ctx, append([]string{"addexclusion"}, exclusionPaths...)...) if err != nil { k.Slogger().Log(ctx, slog.LevelError, - "could not create tmutil command to add launcher db to time machine exclusions", + "could not create tmutil command to add launcher machine exclusions", "err", err, - "path", dbPath, ) return } - if err := cmd.Run(); err != nil { + out, err := cmd.CombinedOutput() + if err != nil { k.Slogger().Log(ctx, slog.LevelError, - "running command to add launcher db to time machine exclusions", + "running command to add launcher machine exclusions", "err", err, - "path", dbPath, + "output", string(out), ) + return } } diff --git a/ee/agent/timemachine/timemachine_darwin_test.go b/ee/agent/timemachine/timemachine_darwin_test.go new file mode 100644 index 000000000..ab2e08a02 --- /dev/null +++ b/ee/agent/timemachine/timemachine_darwin_test.go @@ -0,0 +1,93 @@ +//go:build darwin +// +build darwin + +package timemachine + +import ( + "context" + "os" + "os/user" + "path/filepath" + "strings" + "testing" + + "github.com/kolide/launcher/ee/agent/types/mocks" + "github.com/kolide/launcher/ee/allowedcmd" + "github.com/stretchr/testify/require" +) + +func TestAddExclusions(t *testing.T) { + u, err := user.Current() + require.NoError(t, err) + + // time machine will not include temp dirs, so use home dir to test + // and clean up when done + testDir := filepath.Join(u.HomeDir, "launcher_tmutil_test_delete_me") + require.NoError(t, os.RemoveAll(testDir)) + + require.NoError(t, os.MkdirAll(testDir, 0755)) + defer os.RemoveAll(testDir) + + shouldBeExcluded := map[string]bool{ + // these should be excluded + "augeas-lenses/": true, + "debug-2024-01-09T21-15-14.055.json.gz": true, + "debug.json": true, + "desktop_501/": true, + "desktop_503/": true, + "kv.sqlite": true, + "launcher.db": true, + "launcher.pid": true, + "menu.json": true, + "menu_template.json": true, + "metadata.json": true, + "metadata.plist": true, + "osquery.autoload": true, + "osquery.db/": true, + "osquery.pid": true, + "osquery.sock": true, + "osquery.sock.51807": true, + "osquery.sock.63071": true, + "osqueryd-tuf/": true, + + // these should NOT be excluded + "launcher-tuf/": false, + "tuf/": false, + "updates/": false, + "kolide.png": false, + "launcher-version-1.4.1-4-gdb7106f": false, + } + + for k := range shouldBeExcluded { + path := filepath.Join(testDir, k) + + if strings.HasSuffix(k, "/") { + require.NoError(t, os.MkdirAll(path, 0755)) + } else { + f, err := os.Create(path) + require.NoError(t, err) + f.Close() + } + } + + k := mocks.NewKnapsack(t) + k.On("RootDirectory").Return(testDir) + + AddExclusions(context.TODO(), k) + + for fileName, shouldBeExcluded := range shouldBeExcluded { + cmd, err := allowedcmd.Tmutil(context.TODO(), "isexcluded", filepath.Join(testDir, fileName)) + require.NoError(t, err) + + out, err := cmd.CombinedOutput() + require.NoError(t, err) + + if shouldBeExcluded { + require.Contains(t, string(out), "[Excluded]") + continue + } + + // should be included + require.Contains(t, string(out), "[Included]") + } +} diff --git a/ee/agent/timemachine/timemachine_other.go b/ee/agent/timemachine/timemachine_other.go index bb7b3e8da..0ac978ebc 100644 --- a/ee/agent/timemachine/timemachine_other.go +++ b/ee/agent/timemachine/timemachine_other.go @@ -9,8 +9,8 @@ import ( "github.com/kolide/launcher/ee/agent/types" ) -// ExcludeLauncherDB adds the launcher db to the time machine exclusions for +// AddExclusions adds specific launcher time machine exclusions for // darwin and is noop for other oses -func ExcludeLauncherDB(_ context.Context, _ types.Knapsack) { +func AddExclusions(_ context.Context, _ types.Knapsack) { // do nothing, no time machine on non darwin } From 615031875f0aa0cdf3d8d428e54fbe901f6984ae Mon Sep 17 00:00:00 2001 From: james pickett Date: Wed, 10 Jan 2024 16:33:22 -0800 Subject: [PATCH 6/8] lint --- ee/agent/timemachine/timemachine_darwin_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ee/agent/timemachine/timemachine_darwin_test.go b/ee/agent/timemachine/timemachine_darwin_test.go index ab2e08a02..93a9003d6 100644 --- a/ee/agent/timemachine/timemachine_darwin_test.go +++ b/ee/agent/timemachine/timemachine_darwin_test.go @@ -17,6 +17,8 @@ import ( ) func TestAddExclusions(t *testing.T) { + t.Parallel() + u, err := user.Current() require.NoError(t, err) @@ -58,6 +60,7 @@ func TestAddExclusions(t *testing.T) { "launcher-version-1.4.1-4-gdb7106f": false, } + // create files and dirs for k := range shouldBeExcluded { path := filepath.Join(testDir, k) @@ -75,6 +78,7 @@ func TestAddExclusions(t *testing.T) { AddExclusions(context.TODO(), k) + // ensure the files are included / excluded as expected for fileName, shouldBeExcluded := range shouldBeExcluded { cmd, err := allowedcmd.Tmutil(context.TODO(), "isexcluded", filepath.Join(testDir, fileName)) require.NoError(t, err) From 89cb5e3bec6e4d7d2ac8efb149779f315805ce46 Mon Sep 17 00:00:00 2001 From: james pickett Date: Wed, 10 Jan 2024 16:44:28 -0800 Subject: [PATCH 7/8] update var names --- ee/agent/timemachine/timemachine_darwin_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ee/agent/timemachine/timemachine_darwin_test.go b/ee/agent/timemachine/timemachine_darwin_test.go index 93a9003d6..80defc162 100644 --- a/ee/agent/timemachine/timemachine_darwin_test.go +++ b/ee/agent/timemachine/timemachine_darwin_test.go @@ -61,10 +61,10 @@ func TestAddExclusions(t *testing.T) { } // create files and dirs - for k := range shouldBeExcluded { - path := filepath.Join(testDir, k) + for filename := range shouldBeExcluded { + path := filepath.Join(testDir, filename) - if strings.HasSuffix(k, "/") { + if strings.HasSuffix(filename, "/") { require.NoError(t, os.MkdirAll(path, 0755)) } else { f, err := os.Create(path) @@ -73,10 +73,10 @@ func TestAddExclusions(t *testing.T) { } } - k := mocks.NewKnapsack(t) - k.On("RootDirectory").Return(testDir) + knapsack := mocks.NewKnapsack(t) + knapsack.On("RootDirectory").Return(testDir) - AddExclusions(context.TODO(), k) + AddExclusions(context.TODO(), knapsack) // ensure the files are included / excluded as expected for fileName, shouldBeExcluded := range shouldBeExcluded { From 7c19fba5dde4e59ec1788eac5cd64b0b01dfee50 Mon Sep 17 00:00:00 2001 From: james pickett Date: Thu, 11 Jan 2024 08:00:58 -0800 Subject: [PATCH 8/8] remove else statement --- ee/agent/timemachine/timemachine_darwin_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ee/agent/timemachine/timemachine_darwin_test.go b/ee/agent/timemachine/timemachine_darwin_test.go index 80defc162..90913559f 100644 --- a/ee/agent/timemachine/timemachine_darwin_test.go +++ b/ee/agent/timemachine/timemachine_darwin_test.go @@ -64,13 +64,16 @@ func TestAddExclusions(t *testing.T) { for filename := range shouldBeExcluded { path := filepath.Join(testDir, filename) + // create dir if strings.HasSuffix(filename, "/") { require.NoError(t, os.MkdirAll(path, 0755)) - } else { - f, err := os.Create(path) - require.NoError(t, err) - f.Close() + continue } + + // create file + f, err := os.Create(path) + require.NoError(t, err) + f.Close() } knapsack := mocks.NewKnapsack(t)