diff --git a/cmd/launcher/launcher.go b/cmd/launcher/launcher.go index 960717043..e9f0f7fa1 100644 --- a/cmd/launcher/launcher.go +++ b/cmd/launcher/launcher.go @@ -416,7 +416,7 @@ func runLauncher(ctx context.Context, cancel func(), multiSlogger, systemMultiSl // agentFlagConsumer handles agent flags pushed from the control server controlService.RegisterConsumer(agentFlagsSubsystemName, keyvalueconsumer.New(flagController)) // katcConfigConsumer handles updates to Kolide's custom ATC tables - controlService.RegisterConsumer(katcSubsystemName, keyvalueconsumer.New(k.KatcConfigStore())) + controlService.RegisterConsumer(katcSubsystemName, keyvalueconsumer.NewConfigConsumer(k.KatcConfigStore())) controlService.RegisterSubscriber(katcSubsystemName, osqueryRunner) controlService.RegisterSubscriber(katcSubsystemName, startupSettingsWriter) diff --git a/ee/control/consumers/keyvalueconsumer/config_consumer.go b/ee/control/consumers/keyvalueconsumer/config_consumer.go new file mode 100644 index 000000000..21733ad7f --- /dev/null +++ b/ee/control/consumers/keyvalueconsumer/config_consumer.go @@ -0,0 +1,47 @@ +package keyvalueconsumer + +import ( + "encoding/json" + "errors" + "fmt" + "io" + + "github.com/kolide/launcher/ee/agent/types" +) + +type ConfigConsumer struct { + updater types.Updater +} + +func NewConfigConsumer(updater types.Updater) *ConfigConsumer { + c := &ConfigConsumer{ + updater: updater, + } + + return c +} + +func (c *ConfigConsumer) Update(data io.Reader) error { + if c == nil { + return errors.New("key value consumer is nil") + } + + var kvPairs map[string]any + if err := json.NewDecoder(data).Decode(&kvPairs); err != nil { + return fmt.Errorf("failed to decode key-value json: %w", err) + } + + kvStringPairs := make(map[string]string) + for k, v := range kvPairs { + b, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("unable to marshal value for `%s`: %w", k, err) + } + kvStringPairs[k] = string(b) + } + + // Turn the map into a slice of key, value, ... and send it to the thing storing this data + _, err := c.updater.Update(kvStringPairs) + + return err +} diff --git a/ee/katc/config.go b/ee/katc/config.go index 62fc3eb58..fa45770d4 100644 --- a/ee/katc/config.go +++ b/ee/katc/config.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "log/slog" - "runtime" "github.com/kolide/launcher/ee/indexeddb" "github.com/osquery/osquery-go" @@ -16,8 +15,9 @@ import ( // identifier parsed from the JSON KATC config, and the `dataFunc` is the function // that performs the query against the source. type katcSourceType struct { - name string - dataFunc func(ctx context.Context, slogger *slog.Logger, sourcePattern string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) + name string + // queryContext contains the constraints from the WHERE clause of the query against the KATC table. + dataFunc func(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, queryContext table.QueryContext) ([]sourceData, error) } // sourceData holds the result of calling `katcSourceType.dataFunc`. It maps the @@ -98,36 +98,44 @@ func (r *rowTransformStep) UnmarshalJSON(data []byte) error { } } -// katcTableConfig is the configuration for a specific KATC table. The control server -// sends down these configurations. -type katcTableConfig struct { - SourceType katcSourceType `json:"source_type"` - Source string `json:"source"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported - Platform string `json:"platform"` - Columns []string `json:"columns"` - Query string `json:"query"` // Query to run against `path` - RowTransformSteps []rowTransformStep `json:"row_transform_steps"` -} +type ( + // katcTableConfig is the configuration for a specific KATC table. The control server + // sends down these configurations. + katcTableConfig struct { + Columns []string `json:"columns"` + katcTableDefinition + Overlays []katcTableConfigOverlay `json:"overlays"` + } + + katcTableConfigOverlay struct { + Filters map[string]string `json:"filters"` // determines if this overlay is applicable to this launcher installation + katcTableDefinition + } + + katcTableDefinition struct { + SourceType *katcSourceType `json:"source_type,omitempty"` + SourcePaths *[]string `json:"source_paths,omitempty"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported + SourceQuery *string `json:"source_query,omitempty"` // Query to run against each source path + RowTransformSteps *[]rowTransformStep `json:"row_transform_steps,omitempty"` + } +) // ConstructKATCTables takes stored configuration of KATC tables, parses the configuration, // and returns the constructed tables. func ConstructKATCTables(config map[string]string, slogger *slog.Logger) []osquery.OsqueryPlugin { plugins := make([]osquery.OsqueryPlugin, 0) + for tableName, tableConfigStr := range config { var cfg katcTableConfig if err := json.Unmarshal([]byte(tableConfigStr), &cfg); err != nil { slogger.Log(context.TODO(), slog.LevelWarn, - "unable to unmarshal config for Kolide ATC table, skipping", + "unable to unmarshal config for KATC table, skipping", "table_name", tableName, "err", err, ) continue } - if cfg.Platform != runtime.GOOS { - continue - } - t, columns := newKatcTable(tableName, cfg, slogger) plugins = append(plugins, table.NewPlugin(tableName, columns, t.generate)) } diff --git a/ee/katc/config_test.go b/ee/katc/config_test.go index 3214412c1..fed0272e1 100644 --- a/ee/katc/config_test.go +++ b/ee/katc/config_test.go @@ -21,27 +21,48 @@ func TestConstructKATCTables(t *testing.T) { { testCaseName: "snappy_sqlite", katcConfig: map[string]string{ - "kolide_snappy_sqlite_test": fmt.Sprintf(`{ + "kolide_snappy_sqlite_test": `{ "source_type": "sqlite", - "platform": "%s", "columns": ["data"], - "source": "/some/path/to/db.sqlite", - "query": "SELECT data FROM object_data JOIN object_store ON (object_data.object_store_id = object_store.id) WHERE object_store.name=\"testtable\";", - "row_transform_steps": ["snappy"] - }`, runtime.GOOS), + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data JOIN object_store ON (object_data.object_store_id = object_store.id) WHERE object_store.name=\"testtable\";", + "row_transform_steps": ["snappy"], + "overlays": [] + }`, }, expectedPluginCount: 1, }, { testCaseName: "indexeddb_leveldb", katcConfig: map[string]string{ - "kolide_indexeddb_leveldb_test": fmt.Sprintf(`{ + "kolide_indexeddb_leveldb_test": `{ "source_type": "indexeddb_leveldb", - "platform": "%s", "columns": ["data"], - "source": "/some/path/to/db.indexeddb.leveldb", - "query": "db.store", - "row_transform_steps": ["deserialize_chrome"] + "source_paths": ["/some/path/to/db.indexeddb.leveldb"], + "source_query": "db.store", + "row_transform_steps": ["deserialize_chrome"], + "overlays": [] + }`, + }, + expectedPluginCount: 1, + }, + { + testCaseName: "overlay", + katcConfig: map[string]string{ + "kolide_overlay_test": fmt.Sprintf(`{ + "source_type": "indexeddb_leveldb", + "columns": ["data"], + "source_paths": ["/some/path/to/db.indexeddb.leveldb"], + "source_query": "db.store", + "row_transform_steps": ["deserialize_chrome"], + "overlays": [ + { + "filters": { + "goos": "%s" + }, + "source_paths": ["/some/different/path/to/db.indexeddb.leveldb"] + } + ] }`, runtime.GOOS), }, expectedPluginCount: 1, @@ -49,25 +70,47 @@ func TestConstructKATCTables(t *testing.T) { { testCaseName: "multiple plugins", katcConfig: map[string]string{ - "test_1": fmt.Sprintf(`{ + "test_1": `{ "source_type": "sqlite", - "platform": "%s", "columns": ["data"], - "source": "/some/path/to/db.sqlite", - "query": "SELECT data FROM object_data;", - "row_transform_steps": ["snappy"] - }`, runtime.GOOS), - "test_2": fmt.Sprintf(`{ + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data;", + "row_transform_steps": ["snappy"], + "overlays": [] + }`, + "test_2": `{ "source_type": "sqlite", - "platform": "%s", "columns": ["col1", "col2"], - "source": "/some/path/to/a/different/db.sqlite", - "query": "SELECT col1, col2 FROM some_table;", - "row_transform_steps": ["camel_to_snake"] - }`, runtime.GOOS), + "source_paths": ["/some/path/to/a/different/db.sqlite"], + "source_query": "SELECT col1, col2 FROM some_table;", + "row_transform_steps": ["camel_to_snake"], + "overlays": [] + }`, }, expectedPluginCount: 2, }, + { + testCaseName: "skips invalid tables and returns valid tables", + katcConfig: map[string]string{ + "not_a_valid_table": `{ + "source_type": "not a real type", + "columns": ["col1", "col2"], + "source_paths": ["/some/path/to/a/different/db.sqlite"], + "source_query": "SELECT col1, col2 FROM some_table;", + "row_transform_steps": ["not a real row transform step"], + "overlays": [] + }`, + "valid_table": `{ + "source_type": "sqlite", + "columns": ["data"], + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data;", + "row_transform_steps": ["snappy"], + "overlays": [] + }`, + }, + expectedPluginCount: 1, + }, { testCaseName: "malformed config", katcConfig: map[string]string{ @@ -78,27 +121,26 @@ func TestConstructKATCTables(t *testing.T) { { testCaseName: "invalid table source", katcConfig: map[string]string{ - "kolide_snappy_test": fmt.Sprintf(`{ + "kolide_snappy_test": `{ "source_type": "unknown_source", - "platform": "%s", "columns": ["data"], - "source": "/some/path/to/db.sqlite", - "query": "SELECT data FROM object_data;" - }`, runtime.GOOS), + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data;", + "overlays": [] + }`, }, expectedPluginCount: 0, }, { testCaseName: "invalid data processing step type", katcConfig: map[string]string{ - "kolide_snappy_test": fmt.Sprintf(`{ + "kolide_snappy_test": `{ "source_type": "sqlite", - "platform": "%s", "columns": ["data"], - "source": "/some/path/to/db.sqlite", - "query": "SELECT data FROM object_data;", + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data;", "row_transform_steps": ["unknown_step"] - }`, runtime.GOOS), + }`, }, expectedPluginCount: 0, }, diff --git a/ee/katc/indexeddb_leveldb.go b/ee/katc/indexeddb_leveldb.go index c1a861328..41ab7b92b 100644 --- a/ee/katc/indexeddb_leveldb.go +++ b/ee/katc/indexeddb_leveldb.go @@ -15,39 +15,45 @@ import ( // found at the filepath in `sourcePattern`. It retrieves all rows from the database // and object store specified in `query`, which it expects to be in the format // `.`. -func indexeddbLeveldbData(ctx context.Context, slogger *slog.Logger, sourcePattern string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) { - pathPattern := sourcePatternToGlobbablePattern(sourcePattern) - leveldbs, err := filepath.Glob(pathPattern) - if err != nil { - return nil, fmt.Errorf("globbing for leveldb files: %w", err) - } - - // Extract database and table from query - dbName, objectStoreName, err := extractQueryTargets(query) - if err != nil { - return nil, fmt.Errorf("getting db and object store names: %w", err) - } +func indexeddbLeveldbData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, queryContext table.QueryContext) ([]sourceData, error) { + // Pull out path constraints from the query against the KATC table, to avoid querying more leveldb files than we need to. + pathConstraintsFromQuery := getPathConstraint(queryContext) - // Query databases results := make([]sourceData, 0) - for _, db := range leveldbs { - // Check to make sure `db` adheres to sourceConstraints - valid, err := checkSourceConstraints(db, sourceConstraints) + for _, sourcePath := range sourcePaths { + pathPattern := sourcePatternToGlobbablePattern(sourcePath) + leveldbs, err := filepath.Glob(pathPattern) if err != nil { - return nil, fmt.Errorf("checking source path constraints: %w", err) - } - if !valid { - continue + return nil, fmt.Errorf("globbing for leveldb files: %w", err) } - rowsFromDb, err := indexeddb.QueryIndexeddbObjectStore(db, dbName, objectStoreName) + // Extract database and table from query + dbName, objectStoreName, err := extractQueryTargets(query) if err != nil { - return nil, fmt.Errorf("querying %s: %w", db, err) + return nil, fmt.Errorf("getting db and object store names: %w", err) + } + + // Query databases + for _, db := range leveldbs { + // Check to make sure `db` adheres to pathConstraintsFromQuery. This is an + // optimization to avoid work, if osquery sqlite filtering is going to exclude it. + valid, err := checkPathConstraints(db, pathConstraintsFromQuery) + if err != nil { + return nil, fmt.Errorf("checking source path constraints: %w", err) + } + if !valid { + continue + } + + rowsFromDb, err := indexeddb.QueryIndexeddbObjectStore(db, dbName, objectStoreName) + if err != nil { + return nil, fmt.Errorf("querying %s: %w", db, err) + } + results = append(results, sourceData{ + path: db, + rows: rowsFromDb, + }) } - results = append(results, sourceData{ - path: db, - rows: rowsFromDb, - }) } return results, nil diff --git a/ee/katc/sqlite.go b/ee/katc/sqlite.go index f63eb384d..fe8b3407c 100644 --- a/ee/katc/sqlite.go +++ b/ee/katc/sqlite.go @@ -13,32 +13,38 @@ import ( ) // sqliteData is the dataFunc for sqlite KATC tables -func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePattern string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) { - pathPattern := sourcePatternToGlobbablePattern(sourcePattern) - sqliteDbs, err := filepath.Glob(pathPattern) - if err != nil { - return nil, fmt.Errorf("globbing for files with pattern %s: %w", pathPattern, err) - } +func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, queryContext table.QueryContext) ([]sourceData, error) { + // Pull out path constraints from the query against the KATC table, to avoid querying more sqlite dbs than we need to. + pathConstraintsFromQuery := getPathConstraint(queryContext) results := make([]sourceData, 0) - for _, sqliteDb := range sqliteDbs { - // Check to make sure `sqliteDb` adheres to sourceConstraints - valid, err := checkSourceConstraints(sqliteDb, sourceConstraints) + for _, sourcePath := range sourcePaths { + pathPattern := sourcePatternToGlobbablePattern(sourcePath) + sqliteDbs, err := filepath.Glob(pathPattern) if err != nil { - return nil, fmt.Errorf("checking source path constraints: %w", err) - } - if !valid { - continue + return nil, fmt.Errorf("globbing for files with pattern %s: %w", pathPattern, err) } - rowsFromDb, err := querySqliteDb(ctx, slogger, sqliteDb, query) - if err != nil { - return nil, fmt.Errorf("querying %s: %w", sqliteDb, err) + for _, sqliteDb := range sqliteDbs { + // Check to make sure `db` adheres to pathConstraintsFromQuery. This is an + // optimization to avoid work, if osquery sqlite filtering is going to exclude it. + valid, err := checkPathConstraints(sqliteDb, pathConstraintsFromQuery) + if err != nil { + return nil, fmt.Errorf("checking source path constraints: %w", err) + } + if !valid { + continue + } + + rowsFromDb, err := querySqliteDb(ctx, slogger, sqliteDb, query) + if err != nil { + return nil, fmt.Errorf("querying %s: %w", sqliteDb, err) + } + results = append(results, sourceData{ + path: sqliteDb, + rows: rowsFromDb, + }) } - results = append(results, sourceData{ - path: sqliteDb, - rows: rowsFromDb, - }) } return results, nil diff --git a/ee/katc/sqlite_test.go b/ee/katc/sqlite_test.go index 25b4f1b98..c1510510e 100644 --- a/ee/katc/sqlite_test.go +++ b/ee/katc/sqlite_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/kolide/launcher/pkg/log/multislogger" + "github.com/osquery/osquery-go/plugin/table" "github.com/stretchr/testify/require" ) @@ -59,7 +60,7 @@ func Test_sqliteData(t *testing.T) { } // Query data - results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), filepath.Join(sqliteDir, "*.sqlite"), "SELECT uuid, value FROM test_data;", nil) + results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), []string{filepath.Join(sqliteDir, "*.sqlite")}, "SELECT uuid, value FROM test_data;", table.QueryContext{}) require.NoError(t, err) // Confirm we have the correct number of `sourceData` returned (one per db) @@ -89,7 +90,7 @@ func Test_sqliteData_noSourcesFound(t *testing.T) { t.Parallel() tmpDir := t.TempDir() - results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), filepath.Join(tmpDir, "db.sqlite"), "SELECT * FROM data;", nil) + results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), []string{filepath.Join(tmpDir, "db.sqlite")}, "SELECT * FROM data;", table.QueryContext{}) require.NoError(t, err) require.Equal(t, 0, len(results)) } diff --git a/ee/katc/table.go b/ee/katc/table.go index d3aa31d0f..089fddb9b 100644 --- a/ee/katc/table.go +++ b/ee/katc/table.go @@ -2,34 +2,39 @@ package katc import ( "context" + "errors" "fmt" "log/slog" "regexp" + "runtime" "strings" "github.com/osquery/osquery-go/plugin/table" ) -const sourceColumnName = "source" +const pathColumnName = "path" // katcTable is a Kolide ATC table. It queries the source and transforms the response data // per the configuration in its `cfg`. type katcTable struct { - cfg katcTableConfig - columnLookup map[string]struct{} - slogger *slog.Logger + sourceType katcSourceType + sourcePaths []string + sourceQuery string + rowTransformSteps []rowTransformStep + columnLookup map[string]struct{} + slogger *slog.Logger } // newKatcTable returns a new table with the given `cfg`, as well as the osquery columns for that table. func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) (*katcTable, []table.ColumnDefinition) { columns := []table.ColumnDefinition{ { - Name: sourceColumnName, + Name: pathColumnName, Type: table.ColumnTypeText, }, } columnLookup := map[string]struct{}{ - sourceColumnName: {}, + pathColumnName: {}, } for i := 0; i < len(cfg.Columns); i += 1 { columns = append(columns, table.ColumnDefinition{ @@ -39,21 +44,72 @@ func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) ( columnLookup[cfg.Columns[i]] = struct{}{} } - return &katcTable{ - cfg: cfg, + k := katcTable{ columnLookup: columnLookup, - slogger: slogger.With( - "table_name", tableName, - "table_type", cfg.SourceType, - "table_source", cfg.Source, - ), - }, columns + slogger: slogger, + } + + if cfg.SourceType != nil { + k.sourceType = *cfg.SourceType + } + if cfg.SourcePaths != nil { + k.sourcePaths = *cfg.SourcePaths + } + if cfg.SourceQuery != nil { + k.sourceQuery = *cfg.SourceQuery + } + if cfg.RowTransformSteps != nil { + k.rowTransformSteps = *cfg.RowTransformSteps + } + + // Check overlays to see if any of the filters apply to us; + // use the overlay definition if so. + for _, overlay := range cfg.Overlays { + if !filtersMatch(overlay.Filters) { + continue + } + + if overlay.SourceType != nil { + k.sourceType = *overlay.SourceType + } + if overlay.SourcePaths != nil { + k.sourcePaths = *overlay.SourcePaths + } + if overlay.SourceQuery != nil { + k.sourceQuery = *overlay.SourceQuery + } + if overlay.RowTransformSteps != nil { + k.rowTransformSteps = *overlay.RowTransformSteps + } + + break + } + + // Add extra fields to slogger + k.slogger = slogger.With( + "table_name", tableName, + "table_type", cfg.SourceType, + ) + + return &k, columns +} + +func filtersMatch(filters map[string]string) bool { + // Currently, the only filter we expect is for OS. + if goos, goosFound := filters["goos"]; goosFound { + return goos == runtime.GOOS + } + return false } // generate handles queries against a KATC table. func (k *katcTable) generate(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { + if k.sourceType.dataFunc == nil { + return nil, errors.New("table source type not set") + } + // Fetch data from our table source - dataRaw, err := k.cfg.SourceType.dataFunc(ctx, k.slogger, k.cfg.Source, k.cfg.Query, getSourceConstraint(queryContext)) + dataRaw, err := k.sourceType.dataFunc(ctx, k.slogger, k.sourcePaths, k.sourceQuery, queryContext) if err != nil { return nil, fmt.Errorf("fetching data: %w", err) } @@ -62,13 +118,13 @@ func (k *katcTable) generate(ctx context.Context, queryContext table.QueryContex transformedResults := make([]map[string]string, 0) for _, s := range dataRaw { for _, dataRawRow := range s.rows { - // Make sure source is included in row data + // Make sure source's path is included in row data rowData := map[string]string{ - sourceColumnName: s.path, + pathColumnName: s.path, } // Run any needed transformations on the row data - for _, step := range k.cfg.RowTransformSteps { + for _, step := range k.rowTransformSteps { dataRawRow, err = step.transformFunc(ctx, k.slogger, dataRawRow) if err != nil { return nil, fmt.Errorf("running transform func %s: %w", step.name, err) @@ -103,30 +159,29 @@ func (k *katcTable) generate(ctx context.Context, queryContext table.QueryContex return filteredResults, nil } -// getSourceConstraint retrieves any constraints against the `source` column -func getSourceConstraint(queryContext table.QueryContext) *table.ConstraintList { - sourceConstraint, sourceConstraintExists := queryContext.Constraints[sourceColumnName] - if sourceConstraintExists { - return &sourceConstraint +// getPathConstraint retrieves any constraints against the `path` column +func getPathConstraint(queryContext table.QueryContext) *table.ConstraintList { + if pathConstraint, pathConstraintExists := queryContext.Constraints[pathColumnName]; pathConstraintExists { + return &pathConstraint } return nil } -// checkSourceConstraints validates whether a given `source` matches the given constraints. -func checkSourceConstraints(source string, sourceConstraints *table.ConstraintList) (bool, error) { - if sourceConstraints == nil { +// checkPathConstraints validates whether a given `path` matches the given constraints. +func checkPathConstraints(path string, pathConstraints *table.ConstraintList) (bool, error) { + if pathConstraints == nil { return true, nil } - for _, sourceConstraint := range sourceConstraints.Constraints { - switch sourceConstraint.Operator { + for _, pathConstraint := range pathConstraints.Constraints { + switch pathConstraint.Operator { case table.OperatorEquals: - if source != sourceConstraint.Expression { + if path != pathConstraint.Expression { return false, nil } case table.OperatorLike: // Transform the expression into a regex to test if we have a match. - likeRegexpStr := regexp.QuoteMeta(sourceConstraint.Expression) + likeRegexpStr := regexp.QuoteMeta(pathConstraint.Expression) // % matches zero or more characters likeRegexpStr = strings.Replace(likeRegexpStr, "%", `.*`, -1) // _ matches a single character @@ -137,13 +192,13 @@ func checkSourceConstraints(source string, sourceConstraints *table.ConstraintLi if err != nil { return false, fmt.Errorf("invalid LIKE statement: %w", err) } - if !r.MatchString(source) { + if !r.MatchString(path) { return false, nil } case table.OperatorGlob: // Transform the expression into a regex to test if we have a match. // Unlike LIKE, GLOB is case-sensitive. - globRegexpStr := regexp.QuoteMeta(sourceConstraint.Expression) + globRegexpStr := regexp.QuoteMeta(pathConstraint.Expression) // * matches zero or more characters globRegexpStr = strings.Replace(globRegexpStr, `\*`, `.*`, -1) // ? matches a single character @@ -152,19 +207,19 @@ func checkSourceConstraints(source string, sourceConstraints *table.ConstraintLi if err != nil { return false, fmt.Errorf("invalid GLOB statement: %w", err) } - if !r.MatchString(source) { + if !r.MatchString(path) { return false, nil } case table.OperatorRegexp: - r, err := regexp.Compile(sourceConstraint.Expression) + r, err := regexp.Compile(pathConstraint.Expression) if err != nil { return false, fmt.Errorf("invalid regex: %w", err) } - if !r.MatchString(source) { + if !r.MatchString(path) { return false, nil } default: - return false, fmt.Errorf("operator %v not valid source constraint", sourceConstraint.Operator) + return false, fmt.Errorf("operator %v not valid path constraint", pathConstraint.Operator) } } diff --git a/ee/katc/table_test.go b/ee/katc/table_test.go index 2c3465d09..8cba6ce69 100644 --- a/ee/katc/table_test.go +++ b/ee/katc/table_test.go @@ -81,23 +81,35 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { // At long last, our source is adequately configured. // Move on to constructing our KATC table. + sourceQuery := "SELECT data FROM object_data;" cfg := katcTableConfig{ - SourceType: katcSourceType{ - name: sqliteSourceType, - dataFunc: sqliteData, - }, - Platform: runtime.GOOS, - Columns: []string{expectedColumn}, - Source: filepath.Join(databaseDir, "%.sqlite"), // All sqlite files in the test directory - Query: "SELECT data FROM object_data;", - RowTransformSteps: []rowTransformStep{ - { - name: snappyDecodeTransformStep, - transformFunc: snappyDecode, + Columns: []string{expectedColumn}, + katcTableDefinition: katcTableDefinition{ + SourceType: &katcSourceType{ + name: sqliteSourceType, + dataFunc: sqliteData, + }, + SourcePaths: &[]string{filepath.Join("some", "incorrect", "path")}, + SourceQuery: &sourceQuery, + RowTransformSteps: &[]rowTransformStep{ + { + name: snappyDecodeTransformStep, + transformFunc: snappyDecode, + }, + { + name: deserializeFirefoxTransformStep, + transformFunc: deserializeFirefox, + }, }, + }, + Overlays: []katcTableConfigOverlay{ { - name: deserializeFirefoxTransformStep, - transformFunc: deserializeFirefox, + Filters: map[string]string{ + "goos": runtime.GOOS, + }, + katcTableDefinition: katcTableDefinition{ + SourcePaths: &[]string{filepath.Join(databaseDir, "%.sqlite")}, // All sqlite files in the test directory + }, }, }, } @@ -106,7 +118,7 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { // Make a query context restricting the source to our exact source sqlite database queryContext := table.QueryContext{ Constraints: map[string]table.ConstraintList{ - sourceColumnName: { + pathColumnName: { Constraints: []table.Constraint{ { Operator: table.OperatorEquals, @@ -123,8 +135,8 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { // Validate results require.Equal(t, 1, len(results), "exactly one row expected") - require.Contains(t, results[0], sourceColumnName, "missing source column") - require.Equal(t, sourceFilepath, results[0][sourceColumnName]) + require.Contains(t, results[0], pathColumnName, "missing source column") + require.Equal(t, sourceFilepath, results[0][pathColumnName]) require.Contains(t, results[0], expectedColumn, "expected column missing") require.Equal(t, expectedColumnValue, results[0][expectedColumn], "data mismatch") } @@ -134,14 +146,14 @@ func Test_checkSourcePathConstraints(t *testing.T) { for _, tt := range []struct { testCaseName string - source string + path string constraints table.ConstraintList valid bool errorExpected bool }{ { testCaseName: "equals", - source: filepath.Join("some", "path", "to", "a", "source"), + path: filepath.Join("some", "path", "to", "a", "source"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -155,7 +167,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "not equals", - source: filepath.Join("some", "path", "to", "a", "source"), + path: filepath.Join("some", "path", "to", "a", "source"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -169,7 +181,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "LIKE with % wildcard", - source: filepath.Join("a", "path", "to", "db.sqlite"), + path: filepath.Join("a", "path", "to", "db.sqlite"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -183,7 +195,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "LIKE with underscore wildcard", - source: filepath.Join("a", "path", "to", "db.sqlite"), + path: filepath.Join("a", "path", "to", "db.sqlite"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -197,7 +209,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "LIKE is case-insensitive", - source: filepath.Join("a", "path", "to", "db.sqlite"), + path: filepath.Join("a", "path", "to", "db.sqlite"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -210,7 +222,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "GLOB with * wildcard", - source: filepath.Join("another", "path", "to", "a", "source"), + path: filepath.Join("another", "path", "to", "a", "source"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -224,7 +236,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "GLOB with ? wildcard", - source: filepath.Join("another", "path", "to", "a", "source"), + path: filepath.Join("another", "path", "to", "a", "source"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -238,7 +250,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "regexp", - source: filepath.Join("test", "path", "to", "a", "source"), + path: filepath.Join("test", "path", "to", "a", "source"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -252,7 +264,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "invalid regexp", - source: filepath.Join("test", "path", "to", "a", "source"), + path: filepath.Join("test", "path", "to", "a", "source"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -266,7 +278,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "unsupported", - source: filepath.Join("test", "path", "to", "a", "source", "2"), + path: filepath.Join("test", "path", "to", "a", "source", "2"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -280,7 +292,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "multiple constraints where one does not match", - source: filepath.Join("test", "path", "to", "a", "source", "3"), + path: filepath.Join("test", "path", "to", "a", "source", "3"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -298,7 +310,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "multiple constraints where all match", - source: filepath.Join("test", "path", "to", "a", "source", "3"), + path: filepath.Join("test", "path", "to", "a", "source", "3"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -319,7 +331,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { t.Run(tt.testCaseName, func(t *testing.T) { t.Parallel() - valid, err := checkSourceConstraints(tt.source, &tt.constraints) + valid, err := checkPathConstraints(tt.path, &tt.constraints) if tt.errorExpected { require.Error(t, err, "expected error on checking constraints") } else { diff --git a/pkg/osquery/runtime/runner.go b/pkg/osquery/runtime/runner.go index d96790bf8..a1282c807 100644 --- a/pkg/osquery/runtime/runner.go +++ b/pkg/osquery/runtime/runner.go @@ -187,12 +187,12 @@ func (r *Runner) FlagsChanged(flagKeys ...keys.FlagKey) { // the katc_config subsystem. func (r *Runner) Ping() { r.slogger.Log(context.TODO(), slog.LevelDebug, - "Kolide ATC configuration changed, restarting instance to apply", + "KATC configuration changed, restarting instance to apply", ) if err := r.Restart(); err != nil { r.slogger.Log(context.TODO(), slog.LevelError, - "could not restart osquery instance after Kolide ATC configuration changed", + "could not restart osquery instance after KATC configuration changed", "err", err, ) } diff --git a/pkg/osquery/table/table.go b/pkg/osquery/table/table.go index e5e2d0517..7a72d21ea 100644 --- a/pkg/osquery/table/table.go +++ b/pkg/osquery/table/table.go @@ -77,22 +77,21 @@ func PlatformTables(k types.Knapsack, slogger *slog.Logger, currentOsquerydBinar return tables } -// kolideCustomAtcTables retrieves Kolide ATC config from the appropriate data store(s). -// For now, it just logs the configuration. In the future, it will handle indexeddb tables -// and others. +// kolideCustomAtcTables retrieves Kolide ATC config from the appropriate data store(s), +// then constructs the tables. func kolideCustomAtcTables(k types.Knapsack, slogger *slog.Logger) []osquery.OsqueryPlugin { // Fetch tables from KVStore or from startup settings config, err := katcFromDb(k) if err != nil { slogger.Log(context.TODO(), slog.LevelDebug, - "could not retrieve Kolide ATC config from store, may not have access -- falling back to startup settings", + "could not retrieve KATC config from store, may not have access -- falling back to startup settings", "err", err, ) config, err = katcFromStartupSettings(k) if err != nil { slogger.Log(context.TODO(), slog.LevelWarn, - "could not retrieve Kolide ATC config from startup settings", + "could not retrieve KATC config from startup settings", "err", err, ) return nil