Skip to content

Commit

Permalink
Move name inside config and make config a list of tables instead of a…
Browse files Browse the repository at this point in the history 
… map
  • Loading branch information
@RebeccaMahany
RebeccaMahany committed Jul 3, 2024
1 parent 5fa2de9 commit 03777f0
Show file tree
Hide file tree
Showing 4 changed files with 115 additions and 47 deletions.
31 changes: 26 additions & 5 deletions ee/katc/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ func (r *rowTransformStep) UnmarshalJSON(data []byte) error {
// katcTableConfig is the configuration for a specific KATC table. The control server
// sends down these configurations.
type katcTableConfig struct {
Name string `json:"name"`
SourceType katcSourceType `json:"source_type"`
SourcePaths []string `json:"source_paths"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported
Filter string `json:"filter"`
Expand All @@ -102,12 +103,32 @@ type katcTableConfig struct {
// and returns the constructed tables.
func ConstructKATCTables(config map[string]string, slogger *slog.Logger) []osquery.OsqueryPlugin {
plugins := make([]osquery.OsqueryPlugin, 0)
for tableName, tableConfigStr := range config {

tableConfigs, tableConfigsExist := config["tables"]
if !tableConfigsExist {
slogger.Log(context.TODO(), slog.LevelWarn,
"missing top-level tables key in KATC config, cannot construct tables",
)

return plugins
}

// We want to unmarshal each table config separately, so that we don't fail to configure all tables
// if only some are malformed.
var rawTableConfigs []json.RawMessage
if err := json.Unmarshal([]byte(tableConfigs), &rawTableConfigs); err != nil {
slogger.Log(context.TODO(), slog.LevelWarn,
"could not unmarshal tables in KATC config",
"err", err,
)
return plugins
}

for _, rawTableConfig := range rawTableConfigs {
var cfg katcTableConfig
if err := json.Unmarshal([]byte(tableConfigStr), &cfg); err != nil {
if err := json.Unmarshal(rawTableConfig, &cfg); err != nil {
slogger.Log(context.TODO(), slog.LevelWarn,
"unable to unmarshal config for Kolide ATC table, skipping",
"table_name", tableName,
"err", err,
)
continue
Expand All @@ -118,8 +139,8 @@ func ConstructKATCTables(config map[string]string, slogger *slog.Logger) []osque
continue
}

t, columns := newKatcTable(tableName, cfg, slogger)
plugins = append(plugins, table.NewPlugin(tableName, columns, t.generate))
t, columns := newKatcTable(cfg, slogger)
plugins = append(plugins, table.NewPlugin(cfg.Name, columns, t.generate))
}

return plugins
Expand Down
124 changes: 85 additions & 39 deletions ee/katc/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,70 +21,116 @@ func TestConstructKATCTables(t *testing.T) {
{
testCaseName: "snappy_sqlite",
katcConfig: map[string]string{
"kolide_snappy_sqlite_test": fmt.Sprintf(`{
"source_type": "sqlite",
"filter": "%s",
"columns": ["data"],
"source_paths": ["/some/path/to/db.sqlite"],
"source_query": "SELECT data FROM object_data JOIN object_store ON (object_data.object_store_id = object_store.id) WHERE object_store.name=\"testtable\";",
"row_transform_steps": ["snappy"]
}`, runtime.GOOS),
"tables": fmt.Sprintf(`[
{
"name": "kolide_snappy_sqlite_test",
"source_type": "sqlite",
"filter": "%s",
"columns": ["data"],
"source_paths": ["/some/path/to/db.sqlite"],
"source_query": "SELECT data FROM object_data JOIN object_store ON (object_data.object_store_id = object_store.id) WHERE object_store.name=\"testtable\";",
"row_transform_steps": ["snappy"]
}
]`, runtime.GOOS),
},
expectedPluginCount: 1,
},
{
testCaseName: "multiple plugins",
katcConfig: map[string]string{
"test_1": fmt.Sprintf(`{
"source_type": "sqlite",
"filter": "%s",
"columns": ["data"],
"source_paths": ["/some/path/to/db.sqlite"],
"source_query": "SELECT data FROM object_data;",
"row_transform_steps": ["snappy"]
}`, runtime.GOOS),
"test_2": fmt.Sprintf(`{
"source_type": "sqlite",
"filter": "%s",
"columns": ["col1", "col2"],
"source_paths": ["/some/path/to/a/different/db.sqlite"],
"source_query": "SELECT col1, col2 FROM some_table;",
"row_transform_steps": ["camel_to_snake"]
}`, runtime.GOOS),
"tables": fmt.Sprintf(`[
{
"name": "test_1",
"source_type": "sqlite",
"filter": "%s",
"columns": ["data"],
"source_paths": ["/some/path/to/db.sqlite"],
"source_query": "SELECT data FROM object_data;",
"row_transform_steps": ["snappy"]
},
{
"name": "test_2",
"source_type": "sqlite",
"filter": "%s",
"columns": ["col1", "col2"],
"source_paths": ["/some/path/to/a/different/db.sqlite"],
"source_query": "SELECT col1, col2 FROM some_table;",
"row_transform_steps": ["camel_to_snake"]
}
]`, runtime.GOOS, runtime.GOOS),
},
expectedPluginCount: 2,
},
{
testCaseName: "skips invalid tables and returns valid tables",
katcConfig: map[string]string{
"tables": fmt.Sprintf(`[
{
"name": "not_a_valid_table",
"source_type": "not a real type",
"filter": "%s",
"columns": ["col1", "col2"],
"source_paths": ["/some/path/to/a/different/db.sqlite"],
"source_query": "SELECT col1, col2 FROM some_table;",
"row_transform_steps": ["not a real row transform step"]
},
{
"name": "valid_table",
"source_type": "sqlite",
"filter": "%s",
"columns": ["data"],
"source_paths": ["/some/path/to/db.sqlite"],
"source_query": "SELECT data FROM object_data;",
"row_transform_steps": ["snappy"]
}
]`, runtime.GOOS, runtime.GOOS),
},
expectedPluginCount: 1,
},
{
testCaseName: "malformed config",
katcConfig: map[string]string{
"malformed_test": "this is not a config",
},
expectedPluginCount: 0,
},
{
testCaseName: "malformed table",
katcConfig: map[string]string{
"tables": "this is not a config",
},
expectedPluginCount: 0,
},
{
testCaseName: "invalid table source",
katcConfig: map[string]string{
"kolide_snappy_test": fmt.Sprintf(`{
"source_type": "unknown_source",
"filter": "%s",
"columns": ["data"],
"source_paths": ["/some/path/to/db.sqlite"],
"source_query": "SELECT data FROM object_data;"
}`, runtime.GOOS),
"tables": fmt.Sprintf(`[
{
"name": "kolide_snappy_test",
"source_type": "unknown_source",
"filter": "%s",
"columns": ["data"],
"source_paths": ["/some/path/to/db.sqlite"],
"source_query": "SELECT data FROM object_data;"
}
]`, runtime.GOOS),
},
expectedPluginCount: 0,
},
{
testCaseName: "invalid data processing step type",
katcConfig: map[string]string{
"kolide_snappy_test": fmt.Sprintf(`{
"source_type": "sqlite",
"filter": "%s",
"columns": ["data"],
"source_paths": ["/some/path/to/db.sqlite"],
"source_query": "SELECT data FROM object_data;",
"row_transform_steps": ["unknown_step"]
}`, runtime.GOOS),
"tables": fmt.Sprintf(`[
{
"name": "kolide_snappy_test",
"source_type": "sqlite",
"filter": "%s",
"columns": ["data"],
"source_paths": ["/some/path/to/db.sqlite"],
"source_query": "SELECT data FROM object_data;",
"row_transform_steps": ["unknown_step"]
}
]`, runtime.GOOS),
},
expectedPluginCount: 0,
},
Expand Down
4 changes: 2 additions & 2 deletions ee/katc/table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ type katcTable struct {
}

// newKatcTable returns a new table with the given `cfg`, as well as the osquery columns for that table.
func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) (*katcTable, []table.ColumnDefinition) {
func newKatcTable(cfg katcTableConfig, slogger *slog.Logger) (*katcTable, []table.ColumnDefinition) {
columns := []table.ColumnDefinition{
{
Name: pathColumnName,
Expand All @@ -43,7 +43,7 @@ func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) (
cfg: cfg,
columnLookup: columnLookup,
slogger: slogger.With(
"table_name", tableName,
"table_name", cfg.Name,
"table_type", cfg.SourceType,
"table_source_paths", cfg.SourcePaths,
),
Expand Down
3 changes: 2 additions & 1 deletion ee/katc/table_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) {
// At long last, our source is adequately configured.
// Move on to constructing our KATC table.
cfg := katcTableConfig{
Name: "test_katc_table",
SourceType: katcSourceType{
name: sqliteSourceType,
dataFunc: sqliteData,
Expand All @@ -101,7 +102,7 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) {
},
},
}
testTable, _ := newKatcTable("test_katc_table", cfg, multislogger.NewNopLogger())
testTable, _ := newKatcTable(cfg, multislogger.NewNopLogger())

// Make a query context restricting the source to our exact source sqlite database
queryContext := table.QueryContext{
Expand Down

0 comments on commit 03777f0

Please sign in to comment.