-
Notifications
You must be signed in to change notification settings - Fork 507
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added a magic() VQL function (#1338)
Magic allows the detection of file types by applying various magic rules on the file content. Fixes: #1305
- Loading branch information
Showing
9 changed files
with
247 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,4 +17,5 @@ logs text eol=lf | |
| *.json binary | ||
| *.csv binary | ||
| *.zip binary | ||
| *.index binary | ||
| *.index binary | ||
| *.log binary | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -13,5 +13,3 @@ custom: | |
| - files: | ||
| - "artifacts/definitions/**/*.yaml" | ||
| - "docs/references/vql.yaml" | ||
|
|
||
| init: Init | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| Parameters: | ||
| ApacheMagic: | | ||
| 0 search/1024 "GET Apache Logs | ||
| !:strength + 100 | ||
| sampleLog: | | ||
| 127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207 | ||
| 127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207 | ||
| Queries: | ||
| # access.log should be just data here | ||
| - SELECT magic(path=srcDir + '/artifacts/testdata/files/notnbt.exe'), | ||
| magic(path=srcDir + '/artifacts/testdata/files/test.docx'), | ||
| magic(path=srcDir + '/artifacts/testdata/files/access.log') | ||
| FROM scope() | ||
|
|
||
| # Add some custom definitions. This should trigger our magic now. | ||
| - SELECT magic(path=srcDir + '/artifacts/testdata/files/access.log', | ||
| magic=ApacheMagic) | ||
| FROM scope() | ||
|
|
||
| # Using an accessor to get the data allows us to magic arbitrary strings. | ||
| - SELECT magic(path=srcDir + '/artifacts/testdata/files/notnbt.exe', | ||
| accessor="file"), | ||
| magic(path=sampleLog, accessor="data", magic=ApacheMagic) | ||
| FROM scope() | ||
|
|
||
| # When accessor is not specified we use libmagic itself to access | ||
| # files - this allows reporting on directories, symlinks etc. | ||
| - SELECT magic(path=srcDir + '/artifacts/testdata/files/') | ||
| FROM scope() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| SELECT magic(path=srcDir + '/artifacts/testdata/files/notnbt.exe'), magic(path=srcDir + '/artifacts/testdata/files/test.docx'), magic(path=srcDir + '/artifacts/testdata/files/access.log') FROM scope()[ | ||
| { | ||
| "magic(path=srcDir + '/artifacts/testdata/files/notnbt.exe')": "PE32 executable (console) Intel 80386, for MS Windows", | ||
| "magic(path=srcDir + '/artifacts/testdata/files/test.docx')": "Microsoft Word 2007+", | ||
| "magic(path=srcDir + '/artifacts/testdata/files/access.log')": "ASCII text" | ||
| } | ||
| ]SELECT magic(path=srcDir + '/artifacts/testdata/files/access.log', magic=ApacheMagic) FROM scope()[ | ||
| { | ||
| "magic(path=srcDir + '/artifacts/testdata/files/access.log', magic=ApacheMagic)": "Apache Logs, ASCII text" | ||
| } | ||
| ]SELECT magic(path=srcDir + '/artifacts/testdata/files/notnbt.exe', accessor="file"), magic(path=sampleLog, accessor="data", magic=ApacheMagic) FROM scope()[ | ||
| { | ||
| "magic(path=srcDir + '/artifacts/testdata/files/notnbt.exe', accessor=\"file\")": "PE32 executable (console) Intel 80386, for MS Windows", | ||
| "magic(path=sampleLog, accessor=\"data\", magic=ApacheMagic)": "Apache Logs, ASCII text" | ||
| } | ||
| ]SELECT magic(path=srcDir + '/artifacts/testdata/files/') FROM scope()[ | ||
| { | ||
| "magic(path=srcDir + '/artifacts/testdata/files/')": "directory" | ||
| } | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| // +build cgo | ||
|
|
||
| package tools | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| "github.com/Velocidex/go-magic/magic" | ||
| "github.com/Velocidex/go-magic/magic_files" | ||
| "github.com/Velocidex/ordereddict" | ||
| "www.velocidex.com/golang/velociraptor/glob" | ||
| vql_subsystem "www.velocidex.com/golang/velociraptor/vql" | ||
| "www.velocidex.com/golang/vfilter" | ||
| "www.velocidex.com/golang/vfilter/arg_parser" | ||
| ) | ||
|
|
||
| const ( | ||
| magicHandle = "$MagicHandle" | ||
| ) | ||
|
|
||
| type MagicFunctionArgs struct { | ||
| Path string `vfilter:"required,field=path,doc=Path to open and hash."` | ||
| Accessor string `vfilter:"optional,field=accessor,doc=The accessor to use"` | ||
| Type string `vfilter:"optional,field=type,doc=Magic type (can be empty or 'mime' or 'extension')"` | ||
| Magic string `vfilter:"optional,field=magic,doc=Additional magic to load"` | ||
| } | ||
|
|
||
| type MagicFunction struct{} | ||
|
|
||
| func (self MagicFunction) Call( | ||
| ctx context.Context, | ||
| scope vfilter.Scope, | ||
| args *ordereddict.Dict) vfilter.Any { | ||
|
|
||
| arg := &MagicFunctionArgs{} | ||
| err := arg_parser.ExtractArgsWithContext(ctx, scope, args, arg) | ||
| if err != nil { | ||
| scope.Log("magic: %v", err) | ||
| return vfilter.Null{} | ||
| } | ||
|
|
||
| magic_type := magic.MAGIC_NONE | ||
| switch arg.Type { | ||
| case "mime": | ||
| magic_type = magic.MAGIC_MIME | ||
| case "extension": | ||
| magic_type = magic.MAGIC_EXTENSION | ||
| case "": | ||
| magic_type = magic.MAGIC_NONE | ||
| default: | ||
| scope.Log("magic: unknown type %v", arg.Type) | ||
| return vfilter.Null{} | ||
| } | ||
|
|
||
| var handle *magic.Magic | ||
|
|
||
| // Cache key based on type and custom magic. | ||
| key := fmt.Sprintf("%s_%s_%d", magicHandle, arg.Type, len(arg.Magic)) | ||
| cached := vql_subsystem.CacheGet(scope, key) | ||
| switch t := cached.(type) { | ||
|
|
||
| case error: | ||
| return vfilter.Null{} | ||
|
|
||
| case nil: | ||
| handle = magic.NewMagicHandle(magic_type) | ||
| magic_files.LoadDefaultMagic(handle) | ||
|
|
||
| // Do we need to load additional magic tests? | ||
| if arg.Magic != "" { | ||
| handle.LoadBuffer(arg.Magic) | ||
| errors := handle.GetError() | ||
| if errors != "" { | ||
| scope.Log("magic: While loading custom magic: %v", errors) | ||
| } | ||
| } | ||
|
|
||
| // Attach the handle to the root destructor. | ||
| vql_subsystem.GetRootScope(scope). | ||
| AddDestructor(func() { handle.Close() }) | ||
| vql_subsystem.CacheSet(scope, key, handle) | ||
|
|
||
| case *magic.Magic: | ||
| handle = t | ||
|
|
||
| default: | ||
| // Unexpected value in cache. | ||
| return vfilter.Null{} | ||
| } | ||
|
|
||
| // Just let libmagic handle the path | ||
| if arg.Accessor == "" { | ||
| return handle.File(arg.Path) | ||
| } | ||
|
|
||
| // Read a header from the file and pass to the libmagic | ||
| accessor, err := glob.GetAccessor(arg.Accessor, scope) | ||
| if err != nil { | ||
| scope.Log("magic: %v", err) | ||
| return vfilter.Null{} | ||
| } | ||
|
|
||
| fd, err := accessor.Open(arg.Path) | ||
| if err != nil { | ||
| return vfilter.Null{} | ||
| } | ||
|
|
||
| buffer := make([]byte, 1024*64) | ||
| _, err = fd.Read(buffer) | ||
| if err != nil { | ||
| return vfilter.Null{} | ||
| } | ||
|
|
||
| return handle.Buffer(buffer) | ||
| } | ||
|
|
||
| func (self MagicFunction) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo { | ||
| return &vfilter.FunctionInfo{ | ||
| Name: "magic", | ||
| Doc: "Identify a file using magic rules.", | ||
| ArgType: type_map.AddType(scope, &MagicFunctionArgs{}), | ||
| Version: 1, | ||
| } | ||
| } | ||
|
|
||
| func init() { | ||
| vql_subsystem.RegisterFunction(&MagicFunction{}) | ||
| } |