diff --git a/artifacts/definitions/linux/kernel_modules.yaml b/artifacts/definitions/linux/kernel_modules.yaml new file mode 100644 index 00000000000..c27e978bbce --- /dev/null +++ b/artifacts/definitions/linux/kernel_modules.yaml @@ -0,0 +1,15 @@ +name: Linux.Proc.Modules +description: Module listing via /proc/modules. +parameters: + - name: ProcModules + default: /proc/modules +sources: + - precondition: | + SELECT OS From info() where OS = 'linux' + + queries: + - | + SELECT * from split_records( + filenames=ProcModules, + regex='\\s+', + columns=['Name', 'Size', 'UseCount', 'UsedBy', 'Status', 'Address']) diff --git a/artifacts/definitions/linux/known_hosts.yaml b/artifacts/definitions/linux/known_hosts.yaml new file mode 100644 index 00000000000..18591f15e60 --- /dev/null +++ b/artifacts/definitions/linux/known_hosts.yaml @@ -0,0 +1,30 @@ +name: Linux.Ssh.KnownHosts +description: Find and parse ssh known hosts files. +parameters: + - name: sshKnownHostsFiles + default: '.ssh/known_hosts*' +sources: + - precondition: | + SELECT OS From info() where OS = 'linux' + queries: + - | + // For each user on the system, search for authorized_keys files. + LET authorized_keys = SELECT * from foreach( + row={ + SELECT Uid, User, Homedir from Artifact.Linux.Sys.Users() + }, + query={ + SELECT FullPath, Mtime, Ctime, User, Uid from glob( + globs=Homedir + '/' + sshKnownHostsFiles) + }) + - | + // For each authorized keys file, extract each line on a different row. + // Note: This duplicates the path, user and uid on each key line. + SELECT * from foreach( + row=authorized_keys, + query={ + SELECT Uid, User, FullPath, Line from split_records( + filenames=FullPath, regex="\n", columns=["Line"]) + /* Ignore comment lines. */ + WHERE not Line =~ "^[^#]+#" + }) diff --git a/artifacts/definitions/linux/last_access.yaml b/artifacts/definitions/linux/last_access.yaml new file mode 100644 index 00000000000..f6e76663a62 --- /dev/null +++ b/artifacts/definitions/linux/last_access.yaml @@ -0,0 +1,92 @@ +name: Linux.Sys.LastUserLogin +description: Find and parse system wtmp files. This indicate when the + user last logged in. +parameters: + - name: wtmpGlobs + default: /var/log/wtmp* + + # This is automatically generated from dwarf symbols by Rekall: + # gcc -c -g -o /tmp/test.o /tmp/1.c + # rekall dwarfparser /tmp/test.o + + # And 1.c is: + # #include "utmp.h" + # struct utmp x; + + - name: wtmpProfile + default: | + { + "timeval": [8, { + "tv_sec": [0, ["int"]], + "tv_usec": [4, ["int"]] + }], + "exit_status": [4, { + "e_exit": [2, ["short int"]], + "e_termination": [0, ["short int"]] + }], + "timezone": [8, { + "tz_dsttime": [4, ["int"]], + "tz_minuteswest": [0, ["int"]] + }], + "utmp": [384, { + "__glibc_reserved": [364, ["Array", { + "count": 20, + "target": "char", + "target_args": null + }]], + "ut_addr_v6": [348, ["Array", { + "count": 4, + "target": "int", + "target_args": null + }]], + "ut_exit": [332, ["exit_status"]], + "ut_host": [76, ["String", { + "length": 256 + }]], + "ut_id": [40, ["String", { + "length": 4 + }]], + "ut_line": [8, ["String", { + "length": 32 + }]], + "ut_pid": [4, ["int"]], + "ut_session": [336, ["int"]], + "ut_tv": [340, ["timeval"]], + "ut_type": [0, ["Enumeration", { + "target": "short int", + "choices": { + "0": "EMPTY", + "1": "RUN_LVL", + "2": "BOOT_TIME", + "5": "INIT_PROCESS", + "6": "LOGIN_PROCESS", + "7": "USER_PROCESS", + "8": "DEAD_PROCESS" + } + }]], + "ut_user": [44, ["String", { + "length": 32 + }]] + }] + } + +sources: + - precondition: | + SELECT OS From info() where OS = 'linux' + queries: + - | + SELECT * from foreach( + row={ + SELECT FullPath from glob(globs=split(string=wtmpGlobs, sep=",")) + }, + query={ + SELECT ut_type, ut_id, ut_host as Host, + ut_user as User, + timestamp(epoch=ut_tv.tv_sec) as login_time + FROM binary_parse( + file=FullPath, + profile=wtmpProfile, + iterator="Array", + Target="wtmp" + ) + }) diff --git a/binary/iterators.go b/binary/iterators.go new file mode 100644 index 00000000000..8ba7aedef2a --- /dev/null +++ b/binary/iterators.go @@ -0,0 +1,58 @@ +package binary + +import ( + "encoding/json" +) + +// A parser that holds a logical array of elements. +type ArrayParserOptions struct { + Target string `vfilter:"required,field=target"` +} + +type ArrayParser struct { + *BaseParser + counter int64 + profile *Profile + options *ArrayParserOptions +} + +func (self ArrayParser) Copy() Parser { + return &self +} + +func (self *ArrayParser) ParseArgs(args *json.RawMessage) error { + return json.Unmarshal(*args, &self.options) +} + +// Produce the next iteration in the array. +func (self *ArrayParser) Next(base Object) Object { + result := self.Value(base) + self.counter += 1 + return result +} + +func (self *ArrayParser) Value(base Object) Object { + parser, pres := self.profile.getParser(self.options.Target) + if !pres { + return &ErrorObject{"Type not found"} + } + + return &BaseObject{ + name: base.Name(), + type_name: self.options.Target, + offset: base.Offset() + self.counter*parser.Size( + base.Offset(), base.Reader()), + reader: base.Reader(), + parser: parser, + } +} + +func NewArrayParser(type_name string, name string, + profile *Profile, options *ArrayParserOptions) *ArrayParser { + return &ArrayParser{&BaseParser{ + Name: name, type_name: type_name}, + 0, + profile, + options, + } +} diff --git a/binary/models.go b/binary/models.go index bcbaf166a2c..5a9de1e5287 100644 --- a/binary/models.go +++ b/binary/models.go @@ -6,44 +6,32 @@ import ( ) func AddModel(profile *Profile) { - profile.types["unsigned long long"] = &IntParser{ - name: "unsigned long long", - converter: func(buf []byte) uint64 { + + profile.types["unsigned long long"] = NewIntParser( + "unsigned long long", + func(buf []byte) uint64 { return uint64(binary.LittleEndian.Uint64(buf)) - }, - } - profile.types["unsigned short"] = &IntParser{ - name: "unsigned short", - converter: func(buf []byte) uint64 { + }) + profile.types["unsigned short"] = NewIntParser( + "unsigned short", func(buf []byte) uint64 { return uint64(binary.LittleEndian.Uint16(buf)) - }, - } - profile.types["int8"] = &IntParser{ - name: "int8", - converter: func(buf []byte) uint64 { + }) + profile.types["int8"] = NewIntParser( + "int8", func(buf []byte) uint64 { return uint64(buf[0]) - }, - } - profile.types["int16"] = &IntParser{ - name: "int16", - converter: func(buf []byte) uint64 { + }) + profile.types["int16"] = NewIntParser( + "int16", func(buf []byte) uint64 { return uint64(binary.LittleEndian.Uint16(buf)) - }, - } - profile.types["int32"] = &IntParser{ - name: "int32", - converter: func(buf []byte) uint64 { + }) + profile.types["int32"] = NewIntParser( + "int32", func(buf []byte) uint64 { return uint64(binary.LittleEndian.Uint32(buf)) - }, - } - profile.types["String"] = &StringParser{ - type_name: "string", - } + }) + profile.types["String"] = NewStringParser("string") + profile.types["Enumeration"] = NewEnumeration("Enumeration", profile) - profile.types["Enumeration"] = &Enumeration{ - profile: profile, - type_name: "Enumeration", - } + profile.types["Array"] = NewArrayParser("Array", "", profile, nil) // Aliases profile.types["int"] = profile.types["int32"] diff --git a/binary/parser.go b/binary/parser.go index defe433ebe5..f6f8a88c97b 100644 --- a/binary/parser.go +++ b/binary/parser.go @@ -8,9 +8,22 @@ import ( "io" "sort" "strings" - // utils "www.velocidex.com/golang/velociraptor/testing" ) +// Parsers are objects which know how to parse a particular +// type. Parsers are instantiated once and reused many times. They act +// upon an Object which represents a particular instance of a parser +// in a particular offset. + +// Here is an example: A struct foo may have 3 members. There is a +// Struct parser instantiated once which knows how to parse struct foo +// (i.e. all its fiels and their offsets). Once instantiated and +// stored in the Profile, the parser may be reused multiple times to +// parse multiple foo structs - each time, it produces an Object. + +// The Object struct contains the offset, and the parser that is used +// to parse it. + type Parser interface { SetName(name string) DebugString(offset int64, reader io.ReaderAt) string @@ -34,7 +47,13 @@ type Getter interface { Fields() []string } +type Iterator interface { + Value(base Object) Object + Next(base Object) Object +} + type Object interface { + Name() string AsInteger() uint64 AsString() string Get(field string) Object @@ -45,6 +64,7 @@ type Object interface { IsValid() bool Value() interface{} Fields() []string + Next() Object } type BaseObject struct { @@ -55,6 +75,10 @@ type BaseObject struct { parser Parser } +func (self *BaseObject) Name() string { + return self.name +} + func (self *BaseObject) Reader() io.ReaderAt { return self.reader } @@ -92,11 +116,20 @@ func (self *BaseObject) Get(field string) Object { return result } - switch self.parser.(type) { + switch t := self.parser.(type) { case Getter: - return self.parser.(Getter).Get(self, field) + return t.Get(self, field) default: - return NewErrorObject("Parser does not support Get") + return NewErrorObject("Parser does not support Get for " + field) + } +} + +func (self *BaseObject) Next() Object { + switch t := self.parser.(type) { + case Iterator: + return t.Next(self) + default: + return NewErrorObject("Parser does not support iteration") } } @@ -109,21 +142,17 @@ func (self *BaseObject) Size() int64 { } func (self *BaseObject) IsValid() bool { - buf := make([]byte, 8) - _, err := self.reader.ReadAt(buf, self.offset+self.Size()) - if err != nil { - return false - } - - return true + return self.parser.IsValid(self.offset, self.reader) } func (self *BaseObject) Value() interface{} { - switch self.parser.(type) { + switch t := self.parser.(type) { case Stringer: return self.AsString() case Integerer: return self.AsInteger() + case Iterator: + return t.Value(self) default: return self } @@ -147,6 +176,12 @@ func (self *BaseObject) MarshalJSON() ([]byte, error) { return buf, err } +// When an operation fails we return an error object. The error object +// can continue to be used in all operations and it will just carry +// itself over safely. This means that callers do not need to check +// for errors all the time: + +// a.Get("field").Next().Get("field") -> ErrorObject type ErrorObject struct { message string } @@ -155,6 +190,10 @@ func NewErrorObject(message string) *ErrorObject { return &ErrorObject{message} } +func (self *ErrorObject) Name() string { + return "Error: " + self.message +} + func (self *ErrorObject) Reader() io.ReaderAt { return nil } @@ -167,6 +206,10 @@ func (self *ErrorObject) Get(field string) Object { return self } +func (self *ErrorObject) Next() Object { + return self +} + func (self *ErrorObject) AsInteger() uint64 { return 0 } @@ -195,31 +238,51 @@ func (self *ErrorObject) Fields() []string { return []string{} } -type IntParser struct { - type_name string - name string +// Baseclass for parsers. +type BaseParser struct { + Name string size int64 - converter func(buf []byte) uint64 + type_name string } -func (self *IntParser) Copy() Parser { - result := *self - return &result +func (self *BaseParser) SetName(name string) { + self.Name = name } -func (self *IntParser) SetName(name string) { - self.name = name +func (self *BaseParser) DebugString(offset int64, reader io.ReaderAt) string { + return fmt.Sprintf("[%s] @ %#0x", self.type_name, offset) } -func (self *IntParser) AsInteger(offset int64, reader io.ReaderAt) uint64 { - buf := make([]byte, 8) +func (self *BaseParser) ShortDebugString(offset int64, reader io.ReaderAt) string { + return fmt.Sprintf("[%s] @ %#0x", self.type_name, offset) +} +func (self *BaseParser) Size(offset int64, reader io.ReaderAt) int64 { + return self.size +} + +func (self *BaseParser) IsValid(offset int64, reader io.ReaderAt) bool { + buf := make([]byte, self.size) _, err := reader.ReadAt(buf, offset) if err != nil { - return 0 + return false } + return true +} - return self.converter(buf) +// If a derived parser takes args. process them here. +func (self *BaseParser) ParseArgs(args *json.RawMessage) error { + return nil +} + +// Parse various sizes of ints. +type IntParser struct { + *BaseParser + converter func(buf []byte) uint64 +} + +func (self IntParser) Copy() Parser { + return &self } func (self *IntParser) DebugString(offset int64, reader io.ReaderAt) string { @@ -231,39 +294,38 @@ func (self *IntParser) ShortDebugString(offset int64, reader io.ReaderAt) string return fmt.Sprintf("%#0x", self.AsInteger(offset, reader)) } -func (self *IntParser) Size(offset int64, reader io.ReaderAt) int64 { - return self.size -} - -func (self *IntParser) IsValid(offset int64, reader io.ReaderAt) bool { +func (self *IntParser) AsInteger(offset int64, reader io.ReaderAt) uint64 { buf := make([]byte, 8) - _, err := reader.ReadAt(buf, offset) - if err != nil { - return false + n, err := reader.ReadAt(buf, offset) + if n == 0 || err != nil { + return 0 } - return true + return self.converter(buf) } -func (self *IntParser) ParseArgs(args *json.RawMessage) error { - return nil +func NewIntParser(type_name string, converter func(buf []byte) uint64) *IntParser { + return &IntParser{&BaseParser{ + type_name: type_name, + }, converter} +} + +// Parses strings. +type StringParserOptions struct { + Length int64 } type StringParser struct { - type_name string - Name string - options struct { - Length int64 - } + *BaseParser + options *StringParserOptions } -func (self *StringParser) Copy() Parser { - result := *self - return &result +func NewStringParser(type_name string) *StringParser { + return &StringParser{&BaseParser{type_name: type_name}, &StringParserOptions{}} } -func (self *StringParser) SetName(name string) { - self.Name = name +func (self StringParser) Copy() Parser { + return &self } func (self *StringParser) AsString(offset int64, reader io.ReaderAt) string { @@ -287,18 +349,8 @@ func (self *StringParser) Size(offset int64, reader io.ReaderAt) int64 { return int64(len(self.AsString(offset, reader))) } -func (self *StringParser) IsValid(offset int64, reader io.ReaderAt) bool { - buf := make([]byte, 8) - - _, err := reader.ReadAt(buf, offset) - if err != nil { - return false - } - return true -} - func (self *StringParser) DebugString(offset int64, reader io.ReaderAt) string { - return self.AsString(offset, reader) + return "[string '" + self.AsString(offset, reader) + "']" } func (self *StringParser) ShortDebugString(offset int64, reader io.ReaderAt) string { @@ -306,53 +358,25 @@ func (self *StringParser) ShortDebugString(offset int64, reader io.ReaderAt) str } func (self *StringParser) ParseArgs(args *json.RawMessage) error { - err := json.Unmarshal(*args, &self.options) - if err != nil { - return err - } - return nil + return json.Unmarshal(*args, &self.options) } type StructParser struct { - size int64 - type_name string - Name string - fields map[string]Parser + *BaseParser + fields map[string]*ParseAtOffset } -func (self *StructParser) Copy() Parser { - result := *self - return &result -} - -func (self *StructParser) SetName(name string) { - self.Name = name +func (self StructParser) Copy() Parser { + return &self } func (self *StructParser) Get(base Object, field string) Object { parser, pres := self.fields[field] if pres { - getter, ok := parser.(Getter) - if ok { - return getter.Get(base, field) - } - } - - return NewErrorObject("Field not known.") -} - -func (self *StructParser) Size(offset int64, reader io.ReaderAt) int64 { - return self.size -} - -func (self *StructParser) IsValid(offset int64, reader io.ReaderAt) bool { - buf := make([]byte, 8) - _, err := reader.ReadAt(buf, offset+self.size) - if err != nil { - return false + return parser.Get(base, field) } - return true + return NewErrorObject("Field " + field + " not known.") } func (self *StructParser) Fields() []string { @@ -388,36 +412,33 @@ func (self *StructParser) ShortDebugString(offset int64, reader io.ReaderAt) str return fmt.Sprintf("[%s] @ %#0x\n", self.type_name, offset) } -func (self *StructParser) AddParser(field string, parser Parser) { +func (self *StructParser) AddParser(field string, parser *ParseAtOffset) { self.fields[field] = parser } -func (self *StructParser) ParseArgs(args *json.RawMessage) error { - return nil -} - func NewStructParser(type_name string, size int64) *StructParser { - result := StructParser{ - size: size, - Name: type_name, - type_name: type_name, - fields: make(map[string]Parser)} + result := &StructParser{ + &BaseParser{type_name: type_name, size: size}, + make(map[string]*ParseAtOffset), + } - return &result + return result } type ParseAtOffset struct { - offset int64 - name string + // Field offset within the struct. + offset int64 + name string + + // The name of the parser to use and the params - will be + // dynamically resolved on first access. type_name string - profile *Profile - parser Parser - args *json.RawMessage -} + params *json.RawMessage -func (self *ParseAtOffset) Copy() Parser { - result := *self - return &result + profile *Profile + + // A local cache of the resolved parser for this field. + parser Parser } func (self *ParseAtOffset) Get(base Object, field string) Object { @@ -426,10 +447,14 @@ func (self *ParseAtOffset) Get(base Object, field string) Object { return &ErrorObject{"Type not found"} } - return &BaseObject{ - offset: base.Offset() + self.offset, - reader: base.Reader(), - parser: parser} + result := &BaseObject{ + name: field, + type_name: self.type_name, + offset: base.Offset() + self.offset, + reader: base.Reader(), + parser: parser, + } + return result } func (self *ParseAtOffset) Fields() []string { @@ -447,12 +472,11 @@ func (self *ParseAtOffset) Fields() []string { func (self *ParseAtOffset) DebugString(offset int64, reader io.ReaderAt) string { parser, pres := self.getParser(self.type_name) if !pres { - return "Type not found" + return self.name + ": Type " + self.type_name + " not found" } - return fmt.Sprintf( "%#03x %s %s", self.offset, self.name, - parser.ShortDebugString(self.offset+offset, reader)) + parser.DebugString(self.offset+offset, reader)) } func (self *ParseAtOffset) ShortDebugString(offset int64, reader io.ReaderAt) string { @@ -497,36 +521,37 @@ func (self *ParseAtOffset) getParser(name string) (Parser, bool) { return nil, false } + // Prepare a new parser based on the params. self.parser = parser.Copy() + self.parser.ParseArgs(self.params) + return self.parser, true } -func (self *ParseAtOffset) ParseArgs(args *json.RawMessage) error { - parser, ok := self.getParser(self.type_name) - if ok { - return parser.ParseArgs(args) - } - return nil +func (self *ParseAtOffset) ParseArgs(args *json.RawMessage) { + self.params = args +} + +type EnumerationOptions struct { + Choices map[string]string + Target string } type Enumeration struct { - type_name string - Name string - profile *Profile - parser Parser - options struct { - Choices map[string]string - Target string - } + *BaseParser + profile *Profile + parser Parser + options *EnumerationOptions } -func (self *Enumeration) Copy() Parser { - result := *self - return &result +func NewEnumeration(type_name string, profile *Profile) *Enumeration { + return &Enumeration{&BaseParser{ + type_name: type_name, + }, profile, nil, &EnumerationOptions{}} } -func (self *Enumeration) SetName(name string) { - self.Name = name +func (self Enumeration) Copy() Parser { + return &self } func (self *Enumeration) getParser() (Parser, bool) { @@ -540,7 +565,7 @@ func (self *Enumeration) getParser() (Parser, bool) { return nil, false } - self.parser = parser.Copy() + self.parser = parser return self.parser, true } @@ -590,9 +615,5 @@ func (self *Enumeration) IsValid(offset int64, reader io.ReaderAt) bool { } func (self *Enumeration) ParseArgs(args *json.RawMessage) error { - err := json.Unmarshal(*args, &self.options) - if err != nil { - return err - } - return nil + return json.Unmarshal(*args, &self.options) } diff --git a/binary/parser_test.go b/binary/parser_test.go index a01135b6268..fce320df0be 100644 --- a/binary/parser_test.go +++ b/binary/parser_test.go @@ -6,7 +6,6 @@ import ( "fmt" assert "github.com/stretchr/testify/assert" "testing" - // utils "www.velocidex.com/golang/velociraptor/testing" ) var ( @@ -24,7 +23,6 @@ func TestIntegerParser(t *testing.T) { offset: 0, parser: profile.types["unsigned long long"], } - assert.Equal(t, uint64(0x0807060504030201), base_obj.AsInteger()) } @@ -116,7 +114,9 @@ func TestUnpacking(t *testing.T) { t.Fatalf(err.Error()) } - test_struct := profile.Create("TestStruct", 2, reader) + test_struct, err := profile.Create("TestStruct", 2, reader, nil) + assert.NoError(t, err) + assert.Equal(t, uint64(0x0c0b0a0908070605), test_struct.Get("Field1").AsInteger()) diff --git a/binary/profile.go b/binary/profile.go index 528ad330130..a114429144e 100644 --- a/binary/profile.go +++ b/binary/profile.go @@ -3,9 +3,9 @@ package binary import ( "encoding/json" + "errors" "fmt" "io" - //utils "www.velocidex.com/golang/velociraptor/testing" ) type _Fields map[string][]*json.RawMessage @@ -86,40 +86,62 @@ func (self *Profile) ParseStructDefinitions(definitions string) error { return err } + // When we parse the JSON definition we place + // a delayed reference ParseAtOffset object as + // an intermediate. The struct will + // dereference its fields through the psuedo + // parser which will fetch the real parser + // dynamically. parser := &ParseAtOffset{ offset: offset, name: field_name, profile: self, - type_name: parser_name} + type_name: parser_name, + } parser.SetName(field_name) if len(params) == 2 { - err := parser.ParseArgs(¶ms[1]) - if err != nil { - return err - } + parser.ParseArgs(¶ms[1]) } - struct_parser.AddParser( - field_name, - parser) + struct_parser.AddParser(field_name, parser) } } return nil } -func (self *Profile) Create(type_name string, offset int64, reader io.ReaderAt) Object { - parser, pres := self.types[type_name] +// Create a new object of the specified type. For example: +// type_name = "Array" +// options = { "Target": "int"} +func (self *Profile) Create(type_name string, offset int64, + reader io.ReaderAt, options map[string]interface{}) (Object, error) { + var parser Parser + + profile_parser, pres := self.types[type_name] if !pres { - return &ErrorObject{ - fmt.Sprintf("Type name %s is not known.", type_name)} + return nil, errors.New( + fmt.Sprintf("Type name %s is not known.", type_name)) + } + + // We need a new copy of the parser since the params might be + // unique. + parser = profile_parser.Copy() + + // Convert the options map into json.RawMessage so we can make + // the parser parse it. + message, err := json.Marshal(options) + if err != nil { + return nil, err } + raw_message := json.RawMessage(message) + parser.ParseArgs(&raw_message) return &BaseObject{ offset: offset, reader: reader, type_name: type_name, name: type_name, - parser: parser} + parser: parser, + }, nil } diff --git a/vql/binary.go b/vql/binary.go index 75b6b14795b..52e83892eda 100644 --- a/vql/binary.go +++ b/vql/binary.go @@ -2,8 +2,9 @@ package vql import ( + "context" + "os" "www.velocidex.com/golang/velociraptor/binary" - //utils "www.velocidex.com/golang/velociraptor/testing" "www.velocidex.com/golang/vfilter" ) @@ -54,6 +55,104 @@ func (self _binaryFieldImpl) GetMembers(scope *vfilter.Scope, a vfilter.Any) []s } } +type _BinaryParserPluginArg struct { + Offset uint64 `vfilter:"optional,field=offset"` + File string `vfilter:"required,field=file"` + Profile string `vfilter:"required,field=profile"` + Iterator string `vfilter:"required,field=iterator"` +} + +type _BinaryParserPlugin struct{} + +func (self _BinaryParserPlugin) Call( + ctx context.Context, + scope *vfilter.Scope, + args *vfilter.Dict) <-chan vfilter.Row { + output_chan := make(chan vfilter.Row) + + arg := &_BinaryParserPluginArg{} + err := vfilter.ExtractArgs(scope, args, arg) + if err != nil { + scope.Log("%s: %s", self.Name(), err.Error()) + close(output_chan) + return output_chan + } + + // Extract additional args + options := make(map[string]interface{}) + for k, v := range *args.ToDict() { + switch k { + case "offset", "file", "profile", "iterator": + continue + default: + options[k] = v + } + } + + go func() { + defer close(output_chan) + file, err := os.Open(arg.File) + if err != nil { + scope.Log("%s: %s", self.Name(), err.Error()) + return + } + + // Only close the file when the context (and the VQL + // query) is fully done because we are releasing + // objects which may reference the file. These objects + // may participate in WHERE clause and so will be + // referenced after the plugin is terminated. + go func() { + for { + select { + case <-ctx.Done(): + file.Close() + return + } + } + }() + + profile := binary.NewProfile() + binary.AddModel(profile) + + err = profile.ParseStructDefinitions(arg.Profile) + if err != nil { + scope.Log("%s: %s", self.Name(), err.Error()) + return + } + + options := make(map[string]interface{}) + options["Target"] = "utmp" + array, err := profile.Create("Array", 0, file, options) + if err != nil { + scope.Log("%s: %s", self.Name(), err.Error()) + return + } + for { + value := array.Next() + if !value.IsValid() { + break + } + + output_chan <- value + } + }() + + return output_chan +} + +func (self _BinaryParserPlugin) Name() string { + return "binary_parse" +} + +func (self _BinaryParserPlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo { + return &vfilter.PluginInfo{ + Name: "binary_parse", + Doc: "List last logged in users based on wtmp records.", + } +} + func init() { - exportedProtocolImpl = append(exportedProtocolImpl, &_binaryFieldImpl{}) + RegisterProtocol(&_binaryFieldImpl{}) + RegisterPlugin(&_BinaryParserPlugin{}) } diff --git a/vql/functions/functions.go b/vql/functions/functions.go index f4d318ce69a..70eb3d2b82b 100644 --- a/vql/functions/functions.go +++ b/vql/functions/functions.go @@ -2,11 +2,38 @@ package functions import ( "context" + "encoding/base64" "strconv" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" vfilter "www.velocidex.com/golang/vfilter" ) +type _Base64Decode struct { + String string `vfilter:"required,field=string"` +} + +func (self _Base64Decode) Call( + ctx context.Context, + scope *vfilter.Scope, + args *vfilter.Dict) vfilter.Any { + arg := &_Base64Decode{} + err := vfilter.ExtractArgs(scope, args, arg) + if err != nil { + scope.Log("%s: %s", self.Name(), err.Error()) + return vfilter.Null{} + } + + result, err := base64.StdEncoding.DecodeString(arg.String) + if err != nil { + return vfilter.Null{} + } + return string(result) +} + +func (self _Base64Decode) Name() string { + return "base64decode" +} + type _ToIntArgs struct { String string `vfilter:"required,field=string"` } diff --git a/vql/users_linux.go b/vql/users_linux.go index 65531d58f70..b0bfa242bcd 100644 --- a/vql/users_linux.go +++ b/vql/users_linux.go @@ -2,10 +2,10 @@ package vql import ( - "bytes" + _ "bytes" + "context" "os" "www.velocidex.com/golang/velociraptor/binary" - // utils "www.velocidex.com/golang/velociraptor/testing" "www.velocidex.com/golang/vfilter" ) @@ -75,53 +75,94 @@ var ( ` ) -func extractUserRecords( +type _UsersPluginArg struct { + File string `vfilter:"optional,field=file"` +} + +type _UsersPlugin struct{} + +func (self _UsersPlugin) Call( + ctx context.Context, scope *vfilter.Scope, - args *vfilter.Dict) []vfilter.Row { - var result []vfilter.Row - filename := "/var/log/wtmp" - arg, pres := args.Get("filename") - if pres { - filename, _ = arg.(string) - } + args *vfilter.Dict) <-chan vfilter.Row { + output_chan := make(chan vfilter.Row) - file, err := os.Open(filename) + arg := &_UsersPluginArg{} + err := vfilter.ExtractArgs(scope, args, arg) if err != nil { - return result + scope.Log("%s: %s", "users", err.Error()) + close(output_chan) + return output_chan } - defer file.Close() - - profile := binary.NewProfile() - binary.AddModel(profile) - err = profile.ParseStructDefinitions(UTMP_PROFILE) - if err != nil { - return result + // Default location. + if arg.File == "" { + arg.File = "/var/log/wtmp" } - // We make a copy of the data to avoid race - // conditions. Otherwise we might close the file before - // VFilter finishes analyzing the returned object and might - // require a new read. This only works because there are no - // free pointers. - for { - buf := make([]byte, profile.StructSize("utmp", 0, file)) - _, err := file.Read(buf) + go func() { + defer close(output_chan) + file, err := os.Open(arg.File) if err != nil { - break + scope.Log("%s: %s", self.Name(), err.Error()) + return } - reader := bytes.NewReader(buf) - obj := profile.Create("utmp", 0, reader) - result = append(result, obj) - } - return result + // Only close the file when the context (and the VQL + // query) is fully done because we are releasing + // objects which may reference the file. These objects + // may participate in WHERE clause and so will be + // referenced after the plugin is terminated. + go func() { + for { + select { + case <-ctx.Done(): + file.Close() + return + } + } + }() + + profile := binary.NewProfile() + binary.AddModel(profile) + + err = profile.ParseStructDefinitions(UTMP_PROFILE) + if err != nil { + scope.Log("%s: %s", self.Name(), err.Error()) + return + } + + options := make(map[string]interface{}) + options["Target"] = "utmp" + array, err := profile.Create("Array", 0, file, options) + if err != nil { + scope.Log("%s: %s", self.Name(), err.Error()) + return + } + for { + value := array.Next() + if !value.IsValid() { + break + } + + output_chan <- value + } + }() + + return output_chan +} + +func (self _UsersPlugin) Name() string { + return "users" +} + +func (self _UsersPlugin) Info(type_map *vfilter.TypeMap) *vfilter.PluginInfo { + return &vfilter.PluginInfo{ + Name: "users", + Doc: "List last logged in users based on wtmp records.", + } } func init() { - exportedPlugins = append(exportedPlugins, - vfilter.GenericListPlugin{ - PluginName: "users", - Function: extractUserRecords, - }) + RegisterPlugin(&_UsersPlugin{}) }