diff --git a/artifacts/definitions/Windows/Remediation/Sinkhole.yaml b/artifacts/definitions/Windows/Remediation/Sinkhole.yaml index a36baee5f1c..01ee5fb975d 100644 --- a/artifacts/definitions/Windows/Remediation/Sinkhole.yaml +++ b/artifacts/definitions/Windows/Remediation/Sinkhole.yaml @@ -96,7 +96,7 @@ sources: Line FROM parse_lines(filename=HostsFile) WHERE - Line + Record AND Line AND NOT Line =~ '^#' -- Parse a URL to get domain name. diff --git a/artifacts/definitions/Windows/Search/FileFinder.yaml b/artifacts/definitions/Windows/Search/FileFinder.yaml index 5a632920e75..04095ce8020 100644 --- a/artifacts/definitions/Windows/Search/FileFinder.yaml +++ b/artifacts/definitions/Windows/Search/FileFinder.yaml @@ -66,7 +66,7 @@ parameters: sources: - query: | LET file_search = SELECT FullPath, - Sys.mft as Inode, + get(item=Sys, field="mft") as Inode, Mode.String AS Mode, Size, Mtime AS Modified, Atime AS ATime, diff --git a/artifacts/testdata/server/testcases/mock.in.yaml b/artifacts/testdata/server/testcases/mock.in.yaml index e60b998b3af..93709ed5851 100644 --- a/artifacts/testdata/server/testcases/mock.in.yaml +++ b/artifacts/testdata/server/testcases/mock.in.yaml @@ -125,12 +125,12 @@ Parameters: "Family": 2, "Type": 1, "Laddr": { - "ip": "172.168.101.128", - "port": 64371 + "IP": "172.168.101.128", + "Port": 64371 }, "Raddr": { - "ip": "10.179.67.176", - "port": 443 + "IP": "10.179.67.176", + "Port": 443 }, "Status": "ESTAB", "Pid": 4888, diff --git a/artifacts/testdata/server/testcases/mock.out.yaml b/artifacts/testdata/server/testcases/mock.out.yaml index d3b21857b6a..b90f6d0c00d 100644 --- a/artifacts/testdata/server/testcases/mock.out.yaml +++ b/artifacts/testdata/server/testcases/mock.out.yaml @@ -148,10 +148,10 @@ LET X <= SELECT mock(plugin='info', results=[dict(OS='windows', foo='bar'), dict "Family": "IPv4", "Type": "TCP", "Status": "ESTAB", - "Laddr.IP": null, - "Laddr.Port": null, - "Raddr.IP": null, - "Raddr.Port": null, + "Laddr.IP": "172.168.101.128", + "Laddr.Port": 64371, + "Raddr.IP": "10.179.67.176", + "Raddr.Port": 443, "Timestamp": "2019-12-07T03:30:58Z", "_Source": "Windows.Network.NetstatEnriched/Netstat" } diff --git a/artifacts/testdata/server/testcases/pe.in.yaml b/artifacts/testdata/server/testcases/pe.in.yaml index d3cfbe97b96..ed6fe7b9f59 100644 --- a/artifacts/testdata/server/testcases/pe.in.yaml +++ b/artifacts/testdata/server/testcases/pe.in.yaml @@ -19,7 +19,7 @@ Queries: # Test Associative protocol. - SELECT PEInfo.VersionInformation.CompanyName AS CompanyName FROM X - WHERE CompanyName =~ "Microsoft" + WHERE PEInfo.VersionInformation AND CompanyName =~ "Microsoft" # Test membership protocol - SELECT Name diff --git a/artifacts/testdata/server/testcases/pe.out.yaml b/artifacts/testdata/server/testcases/pe.out.yaml index dd31c53cc61..46cab0fd1b1 100644 --- a/artifacts/testdata/server/testcases/pe.out.yaml +++ b/artifacts/testdata/server/testcases/pe.out.yaml @@ -64,7 +64,7 @@ SELECT basename(path=FullPath) AS Name, parse_pe(file=FullPath) as PEInfo FROM g "ntoskrnl.exe!MmGetPhysicalMemoryRanges" ] } -]LET X = SELECT basename(path=FullPath) AS Name, parse_pe(file=FullPath) as PEInfo FROM glob(globs=srcDir + "/artifacts/**10/*.{exe,sys}")[]SELECT PEInfo.VersionInformation.CompanyName AS CompanyName FROM X WHERE CompanyName =~ "Microsoft"[ +]LET X = SELECT basename(path=FullPath) AS Name, parse_pe(file=FullPath) as PEInfo FROM glob(globs=srcDir + "/artifacts/**10/*.{exe,sys}")[]SELECT PEInfo.VersionInformation.CompanyName AS CompanyName FROM X WHERE PEInfo.VersionInformation AND CompanyName =~ "Microsoft"[ { "CompanyName": "Microsoft Corporation" }, diff --git a/bin/golden.go b/bin/golden.go index 5867a423fad..c414f5271a7 100644 --- a/bin/golden.go +++ b/bin/golden.go @@ -24,12 +24,14 @@ import ( "log" "os" "path/filepath" + "regexp" "runtime/pprof" "strings" "time" "github.com/Velocidex/ordereddict" "github.com/Velocidex/yaml/v2" + errors "github.com/pkg/errors" "github.com/sergi/go-diff/diffmatchpatch" "github.com/shirou/gopsutil/process" kingpin "gopkg.in/alecthomas/kingpin.v2" @@ -164,10 +166,11 @@ func runTest(fixture *testFixture, container, err := reporting.NewContainer(tmpfile.Name(), "", 5) kingpin.FatalIfError(err, "Can not create output container") + log_writer := &MemoryLogWriter{config_obj: config_obj} builder := services.ScopeBuilder{ Config: config_obj, ACLManager: vql_subsystem.NewRoleACLManager("administrator"), - Logger: log.New(&LogWriter{config_obj}, "Velociraptor: ", 0), + Logger: log.New(log_writer, "Velociraptor: ", 0), Uploader: container, Env: ordereddict.NewDict(). Set("GoldenOutput", tmpfile.Name()). @@ -219,6 +222,15 @@ func runTest(fixture *testFixture, } } + res, err := log_writer.Matches("Symbol .+ not found") + if err != nil { + return result, err + } + + if res { + return result, errors.New("Symbol not found error!") + } + return result, nil } @@ -307,3 +319,30 @@ func init() { return true }) } + +type MemoryLogWriter struct { + config_obj *config_proto.Config + logs []string +} + +func (self *MemoryLogWriter) Write(b []byte) (int, error) { + self.logs = append(self.logs, string(b)) + + logging.GetLogger(self.config_obj, &logging.ClientComponent).Info("%v", string(b)) + return len(b), nil +} + +func (self *MemoryLogWriter) Matches(pattern string) (bool, error) { + re, err := regexp.Compile(pattern) + if err != nil { + return false, err + } + + for _, line := range self.logs { + if re.FindString(line) != "" { + return true, nil + } + } + + return false, nil +} diff --git a/go.mod b/go.mod index 3c3be513270..38983c67315 100644 --- a/go.mod +++ b/go.mod @@ -114,7 +114,7 @@ require ( gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect - howett.net/plist v0.0.0-20181124034731-591f970eefbb + howett.net/plist v0.0.0-20201203080718-1454fab16a06 www.velocidex.com/golang/evtx v0.0.2-0.20201104041743-4b6cdb206c95 www.velocidex.com/golang/go-ese v0.0.0-20200111070159-4b7484475321 www.velocidex.com/golang/go-ntfs v0.1.2-0.20201111050421-bbba6f6a13d3 @@ -122,7 +122,7 @@ require ( www.velocidex.com/golang/go-prefetch v0.0.0-20200722101157-37e4751dd5ca www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196 www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500 - www.velocidex.com/golang/vfilter v0.0.0-20210621143251-8e57fc2e83c6 + www.velocidex.com/golang/vfilter v0.0.0-20210630033040-9c8261d8c095 www.velocidex.com/golang/vtypes v0.0.0-20210624153356-79a8d2c1b823 ) diff --git a/go.sum b/go.sum index 294f958ce30..350ef1b35bc 100644 --- a/go.sum +++ b/go.sum @@ -942,8 +942,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= -howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +howett.net/plist v0.0.0-20201203080718-1454fab16a06 h1:QDxUo/w2COstK1wIBYpzQlHX/NqaQTcf9jyz347nI58= +howett.net/plist v0.0.0-20201203080718-1454fab16a06/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= @@ -965,7 +965,7 @@ www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196/go.mod h1:i www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500 h1:XqZddiAbjPIsTZcEPbqqqABS/ZV5SB7j33eczNsqD60= www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500/go.mod h1:DVzloLH8L+oF3zma1Jisaat5bGF+4VLggDcYlIp00ns= www.velocidex.com/golang/vfilter v0.0.0-20210515085940-25d96b94dafb/go.mod h1:KB724xBNYh4lgipyGwsvx0/5hXRqsKjmrMrkSjGESvU= -www.velocidex.com/golang/vfilter v0.0.0-20210621143251-8e57fc2e83c6 h1:EPExL5jjHBmDNn/jWd9LmDS7sgBIf9ppYf5YpH+Aw10= -www.velocidex.com/golang/vfilter v0.0.0-20210621143251-8e57fc2e83c6/go.mod h1:KB724xBNYh4lgipyGwsvx0/5hXRqsKjmrMrkSjGESvU= +www.velocidex.com/golang/vfilter v0.0.0-20210630033040-9c8261d8c095 h1:boDFsEOosv+a+6SnfP8SFJQGH45WQ2ukj5UJf9Qsses= +www.velocidex.com/golang/vfilter v0.0.0-20210630033040-9c8261d8c095/go.mod h1:KB724xBNYh4lgipyGwsvx0/5hXRqsKjmrMrkSjGESvU= www.velocidex.com/golang/vtypes v0.0.0-20210624153356-79a8d2c1b823 h1:7NLuLQkIiTKI0aQt5MVPs+5e5bFpWm6Z18qdfDQ77LE= www.velocidex.com/golang/vtypes v0.0.0-20210624153356-79a8d2c1b823/go.mod h1:PIG8uSY330pJd620KPksZpTaAsX3sIMiiNJQihZph6c=