Made symbol not found a fatal error in golden. (Velocidex#1127)

artifacts/definitions/Windows/Remediation/Sinkhole.yaml

@@ -96,7 +96,7 @@ sources:
             Line
         FROM parse_lines(filename=HostsFile)
         WHERE
-            Line
+            Record AND Line
             AND NOT Line =~ '^#'
 
       -- Parse a URL to get domain name.

artifacts/definitions/Windows/Search/FileFinder.yaml

@@ -66,7 +66,7 @@ parameters:
 sources:
   - query: |
       LET file_search = SELECT FullPath,
-               Sys.mft as Inode,
+               get(item=Sys, field="mft") as Inode,
                Mode.String AS Mode, Size,
                Mtime AS Modified,
                Atime AS ATime,

artifacts/testdata/server/testcases/mock.in.yaml

@@ -125,12 +125,12 @@ Parameters:
        "Family": 2,
        "Type": 1,
        "Laddr": {
-        "ip": "172.168.101.128",
-        "port": 64371
+        "IP": "172.168.101.128",
+        "Port": 64371
        },
        "Raddr": {
-        "ip": "10.179.67.176",
-        "port": 443
+        "IP": "10.179.67.176",
+        "Port": 443
        },
        "Status": "ESTAB",
        "Pid": 4888,

artifacts/testdata/server/testcases/mock.out.yaml

@@ -148,10 +148,10 @@ LET X <= SELECT mock(plugin='info', results=[dict(OS='windows', foo='bar'), dict
   "Family": "IPv4",
   "Type": "TCP",
   "Status": "ESTAB",
-  "Laddr.IP": null,
-  "Laddr.Port": null,
-  "Raddr.IP": null,
-  "Raddr.Port": null,
+  "Laddr.IP": "172.168.101.128",
+  "Laddr.Port": 64371,
+  "Raddr.IP": "10.179.67.176",
+  "Raddr.Port": 443,
   "Timestamp": "2019-12-07T03:30:58Z",
   "_Source": "Windows.Network.NetstatEnriched/Netstat"
  }

artifacts/testdata/server/testcases/pe.in.yaml

@@ -19,7 +19,7 @@ Queries:
 
   # Test Associative protocol.
   - SELECT PEInfo.VersionInformation.CompanyName AS CompanyName FROM X
-    WHERE CompanyName =~ "Microsoft"
+    WHERE PEInfo.VersionInformation AND CompanyName =~ "Microsoft"
 
   # Test membership protocol
   - SELECT Name

artifacts/testdata/server/testcases/pe.out.yaml

@@ -64,7 +64,7 @@ SELECT basename(path=FullPath) AS Name, parse_pe(file=FullPath) as PEInfo FROM g
    "ntoskrnl.exe!MmGetPhysicalMemoryRanges"
   ]
  }
-]LET X = SELECT basename(path=FullPath) AS Name, parse_pe(file=FullPath) as PEInfo FROM glob(globs=srcDir + "/artifacts/**10/*.{exe,sys}")[]SELECT PEInfo.VersionInformation.CompanyName AS CompanyName FROM X WHERE CompanyName =~ "Microsoft"[
+]LET X = SELECT basename(path=FullPath) AS Name, parse_pe(file=FullPath) as PEInfo FROM glob(globs=srcDir + "/artifacts/**10/*.{exe,sys}")[]SELECT PEInfo.VersionInformation.CompanyName AS CompanyName FROM X WHERE PEInfo.VersionInformation AND CompanyName =~ "Microsoft"[
  {
   "CompanyName": "Microsoft Corporation"
  },

bin/golden.go

@@ -24,12 +24,14 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"regexp"
 	"runtime/pprof"
 	"strings"
 	"time"
 
 	"github.com/Velocidex/ordereddict"
 	"github.com/Velocidex/yaml/v2"
+	errors "github.com/pkg/errors"
 	"github.com/sergi/go-diff/diffmatchpatch"
 	"github.com/shirou/gopsutil/process"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
@@ -164,10 +166,11 @@ func runTest(fixture *testFixture,
 	container, err := reporting.NewContainer(tmpfile.Name(), "", 5)
 	kingpin.FatalIfError(err, "Can not create output container")
 
+	log_writer := &MemoryLogWriter{config_obj: config_obj}
 	builder := services.ScopeBuilder{
 		Config:     config_obj,
 		ACLManager: vql_subsystem.NewRoleACLManager("administrator"),
-		Logger:     log.New(&LogWriter{config_obj}, "Velociraptor: ", 0),
+		Logger:     log.New(log_writer, "Velociraptor: ", 0),
 		Uploader:   container,
 		Env: ordereddict.NewDict().
 			Set("GoldenOutput", tmpfile.Name()).
@@ -219,6 +222,15 @@ func runTest(fixture *testFixture,
 		}
 	}
 
+	res, err := log_writer.Matches("Symbol .+ not found")
+	if err != nil {
+		return result, err
+	}
+
+	if res {
+		return result, errors.New("Symbol not found error!")
+	}
+
 	return result, nil
 }
 
@@ -307,3 +319,30 @@ func init() {
 		return true
 	})
 }
+
+type MemoryLogWriter struct {
+	config_obj *config_proto.Config
+	logs       []string
+}
+
+func (self *MemoryLogWriter) Write(b []byte) (int, error) {
+	self.logs = append(self.logs, string(b))
+
+	logging.GetLogger(self.config_obj, &logging.ClientComponent).Info("%v", string(b))
+	return len(b), nil
+}
+
+func (self *MemoryLogWriter) Matches(pattern string) (bool, error) {
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return false, err
+	}
+
+	for _, line := range self.logs {
+		if re.FindString(line) != "" {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}

go.mod

@@ -114,15 +114,15 @@ require (
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
-	howett.net/plist v0.0.0-20181124034731-591f970eefbb
+	howett.net/plist v0.0.0-20201203080718-1454fab16a06
 	www.velocidex.com/golang/evtx v0.0.2-0.20201104041743-4b6cdb206c95
 	www.velocidex.com/golang/go-ese v0.0.0-20200111070159-4b7484475321
 	www.velocidex.com/golang/go-ntfs v0.1.2-0.20201111050421-bbba6f6a13d3
 	www.velocidex.com/golang/go-pe v0.1.1-0.20210524015317-07c8b305094e
 	www.velocidex.com/golang/go-prefetch v0.0.0-20200722101157-37e4751dd5ca
 	www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196
 	www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500
-	www.velocidex.com/golang/vfilter v0.0.0-20210621143251-8e57fc2e83c6
+	www.velocidex.com/golang/vfilter v0.0.0-20210630033040-9c8261d8c095
 	www.velocidex.com/golang/vtypes v0.0.0-20210624153356-79a8d2c1b823
 )
 

go.sum

@@ -942,8 +942,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M=
-howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
+howett.net/plist v0.0.0-20201203080718-1454fab16a06 h1:QDxUo/w2COstK1wIBYpzQlHX/NqaQTcf9jyz347nI58=
+howett.net/plist v0.0.0-20201203080718-1454fab16a06/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
@@ -965,7 +965,7 @@ www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196/go.mod h1:i
 www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500 h1:XqZddiAbjPIsTZcEPbqqqABS/ZV5SB7j33eczNsqD60=
 www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500/go.mod h1:DVzloLH8L+oF3zma1Jisaat5bGF+4VLggDcYlIp00ns=
 www.velocidex.com/golang/vfilter v0.0.0-20210515085940-25d96b94dafb/go.mod h1:KB724xBNYh4lgipyGwsvx0/5hXRqsKjmrMrkSjGESvU=
-www.velocidex.com/golang/vfilter v0.0.0-20210621143251-8e57fc2e83c6 h1:EPExL5jjHBmDNn/jWd9LmDS7sgBIf9ppYf5YpH+Aw10=
-www.velocidex.com/golang/vfilter v0.0.0-20210621143251-8e57fc2e83c6/go.mod h1:KB724xBNYh4lgipyGwsvx0/5hXRqsKjmrMrkSjGESvU=
+www.velocidex.com/golang/vfilter v0.0.0-20210630033040-9c8261d8c095 h1:boDFsEOosv+a+6SnfP8SFJQGH45WQ2ukj5UJf9Qsses=
+www.velocidex.com/golang/vfilter v0.0.0-20210630033040-9c8261d8c095/go.mod h1:KB724xBNYh4lgipyGwsvx0/5hXRqsKjmrMrkSjGESvU=
 www.velocidex.com/golang/vtypes v0.0.0-20210624153356-79a8d2c1b823 h1:7NLuLQkIiTKI0aQt5MVPs+5e5bFpWm6Z18qdfDQ77LE=
 www.velocidex.com/golang/vtypes v0.0.0-20210624153356-79a8d2c1b823/go.mod h1:PIG8uSY330pJd620KPksZpTaAsX3sIMiiNJQihZph6c=