forked from Velocidex/velociraptor
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Introduced a dedicated pathspec object (Velocidex#1440)
This replaces the url in certain accessors because it is more reliable It is now possible to nest zip files arbitrarily - if a compressed file is found in a zip file, the file is automatically extracted to a temp file for further processing.
- Loading branch information
Showing
19 changed files
with
760 additions
and
241 deletions.
There are no files selected for viewing
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| package test_utils | ||
|
|
||
| import ( | ||
| "context" | ||
| "time" | ||
|
|
||
| "github.com/Velocidex/ordereddict" | ||
| config_proto "www.velocidex.com/golang/velociraptor/config/proto" | ||
| "www.velocidex.com/golang/velociraptor/logging" | ||
| "www.velocidex.com/golang/velociraptor/services" | ||
| vql_subsystem "www.velocidex.com/golang/velociraptor/vql" | ||
| "www.velocidex.com/golang/vfilter" | ||
| ) | ||
|
|
||
| // A convenience function for running a query and getting back a set | ||
| // of rows. | ||
| func RunQuery( | ||
| config_obj *config_proto.Config, | ||
| query string, | ||
| env *ordereddict.Dict) ([]*ordereddict.Dict, error) { | ||
|
|
||
| builder := services.ScopeBuilder{ | ||
| Config: config_obj, | ||
| ACLManager: vql_subsystem.NullACLManager{}, | ||
| Logger: logging.NewPlainLogger( | ||
| config_obj, &logging.FrontendComponent), | ||
| Env: env, | ||
| } | ||
| manager, err := services.GetRepositoryManager() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| scope := manager.BuildScope(builder) | ||
| defer scope.Close() | ||
|
|
||
| ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) | ||
| defer cancel() | ||
|
|
||
| multi_vql, err := vfilter.MultiParse(query) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| rows := []*ordereddict.Dict{} | ||
| for _, vql := range multi_vql { | ||
| for row := range vql.Eval(ctx, scope) { | ||
| rows = append(rows, vfilter.RowToDict(ctx, scope, row)) | ||
| } | ||
| } | ||
|
|
||
| return rows, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| package glob | ||
|
|
||
| import ( | ||
| "net/url" | ||
|
|
||
| errors "github.com/pkg/errors" | ||
| "www.velocidex.com/golang/velociraptor/json" | ||
| ) | ||
|
|
||
| var ( | ||
| InvalidPathSpec = errors.New("Invalid PathSpec") | ||
| ) | ||
|
|
||
| /* | ||
| A PathSpec is a more precise indication of a path to open a source | ||
| of data. In Velociraptor, access to data is provided by the use of | ||
| "Accessors" - a registered driver capable of reading data from | ||
| certain sources. | ||
| Accessors can delegate to other accessors using the PathSpec. This | ||
| delegation allows an accessor to receive additional information in | ||
| order to properly create the filesystem abstraction. | ||
| For example, consider the "zip" accessor which is responsible for | ||
| reading compressed archives. In order to retrieve a file inside the | ||
| zip file, the accessor needs the following pieces of data: | ||
| 1. A delegate accessor to use to open the underlying zip file. | ||
| 2. A path to provide to the delegate accessor. | ||
| 3. The name of the zip member to open. | ||
| For example the following path spec: | ||
| {"Accessor": "file", | ||
| "DelegatePath": "/tmp/file.zip", | ||
| "Path": "zip_member.exe"} | ||
| Provides all this information. | ||
| PathSpecs are supposed to be serialized into strings and passed as | ||
| the filename to plugins that require file paths. The PathSpec is | ||
| just a more detailed path representation and is treated everywhere | ||
| as a plain string (json encoded). | ||
| Therefore the following path spec is valid for a recursive path | ||
| {"Accessor": "zip", | ||
| "DelegatePath": "{\"Accessor\": \"file\", \"DelegatePath\": \"/tmp/file.zip\", \"Path\": \"embedded.zip\"}", | ||
| "Path": "zip_member.exe"} | ||
| Given to the zip accessor, this PathSpec means to use the "zip" | ||
| accessor to open a member "embedded.zip" inside a file | ||
| "/tmp/file.zip", then to search within that embedded zip for a | ||
| "zip_member.exe" | ||
| For convenience, the PathSpec also supports a structured delegate so | ||
| the following serialization is also valid. | ||
| {"Accessor": "zip", | ||
| "Delegate": { | ||
| "Accessor": "file", | ||
| "DelegatePath": "/tmp/file.zip", | ||
| "Path": "embedded.zip" | ||
| }, | ||
| "Path": "zip_member.exe"} | ||
| ## Note: | ||
| In previous versions, the PathSpec abstraction was provided by | ||
| mapping URL parts to the fields above. This proved problematic | ||
| because URL encoding is lossy and not robust enough for round | ||
| tripping of all paths. | ||
| It also produces difficult to read paths. The old URL way is | ||
| deprecated but still supported - it will eventually be dropped. | ||
| */ | ||
| type PathSpec struct { | ||
| DelegateAccessor string `json:"DelegateAccessor,omitempty"` | ||
| DelegatePath string `json:"DelegatePath,omitempty"` | ||
| Delegate *PathSpec `json:"Delegate,omitempty"` | ||
| Path string `json:"Path,omitempty"` | ||
|
|
||
| // Keep track of if the pathspec came from a URL based for | ||
| // backwards compatibility. | ||
| url_based bool | ||
| } | ||
|
|
||
| func (self PathSpec) GetDelegatePath() string { | ||
| if self.Delegate != nil { | ||
| return self.Delegate.String() | ||
| } | ||
| return self.DelegatePath | ||
| } | ||
|
|
||
| func (self PathSpec) String() string { | ||
| if self.url_based { | ||
| result := url.URL{ | ||
| Scheme: self.DelegateAccessor, | ||
| Path: self.DelegatePath, | ||
| Fragment: self.Path, | ||
| } | ||
|
|
||
| return result.String() | ||
| } | ||
|
|
||
| return json.MustMarshalString(self) | ||
| } | ||
|
|
||
| func PathSpecFromString(parsed string) (*PathSpec, error) { | ||
| if len(parsed) == 0 { | ||
| return nil, InvalidPathSpec | ||
| } | ||
|
|
||
| // It is a serialized JSON object. | ||
| if parsed[0] == '{' { | ||
| result := &PathSpec{} | ||
| err := json.Unmarshal([]byte(parsed), result) | ||
| return result, err | ||
| } | ||
|
|
||
| // It can be a URL | ||
| parsed_url, err := url.Parse(parsed) | ||
| if err != nil { | ||
| return nil, InvalidPathSpec | ||
| } | ||
|
|
||
| // It looks like a windows path not a URL | ||
| if len(parsed_url.Scheme) == 1 { | ||
| return &PathSpec{ | ||
| DelegatePath: parsed, | ||
| }, nil | ||
| } | ||
|
|
||
| // Support urls for backwards compatibility. | ||
| return &PathSpec{ | ||
| DelegateAccessor: parsed_url.Scheme, | ||
| DelegatePath: parsed_url.Path, | ||
| Path: parsed_url.Fragment, | ||
| url_based: true, | ||
| }, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,22 +1,22 @@ | ||
| [ | ||
| { | ||
| "Base": "hello.zip#hello.txt", | ||
| "Base": "hello.txt", | ||
| "Data": "hello\n" | ||
| }, | ||
| { | ||
| "Base": "hello.zip#hello1.txt", | ||
| "Base": "hello1.txt", | ||
| "Data": "hello1\n" | ||
| }, | ||
| { | ||
| "Base": "hello.zip#hello2.txt", | ||
| "Base": "hello2.txt", | ||
| "Data": "hello2\n" | ||
| }, | ||
| { | ||
| "Base": "hello.zip#hello3.txt", | ||
| "Base": "hello3.txt", | ||
| "Data": "hello3\n" | ||
| }, | ||
| { | ||
| "Base": "hello.zip#hello4.txt", | ||
| "Base": "hello4.txt", | ||
| "Data": "hello4\n" | ||
| } | ||
| ] |
Oops, something went wrong.