Replaced makefile with mage.

actions/vql.go

@@ -110,7 +110,6 @@ func (self *VQLClientAction) StartQuery(
 			// seconds.
 			max_wait = 100
 		}
-
 		result_chan := vfilter.GetResponseChannel(
 			vql, ctx, scope, max_rows, max_wait)
 		for {
@@ -129,7 +128,6 @@ func (self *VQLClientAction) StartQuery(
 				Response:  string(result.Payload),
 				Timestamp: uint64(time.Now().UTC().UnixNano() / 1000),
 			}
-
 			responder.Log("Time %v: %s: Sending response part %d %s (%d rows).",
 				(response.Timestamp-now)/1000000,
 				query.Name,

appveyor.yml

@@ -10,18 +10,14 @@ install:
     echo %GOPATH%
     go version
     go env
-    go get github.com/UnnoTed/fileb0x
-    go install github.com/UnnoTed/fileb0x
-    fileb0x artifacts/b0x.yaml
-    fileb0x config/b0x.yaml
     go get github.com/golang/dep
     go get -u github.com/golang/dep/cmd/dep
     dep ensure
 build_script:
 - cmd: |
     regedit /S artifacts/testdata/windows/init.reg
-    go build -o buildOutput\velociraptor.exe -i ./bin/
+    go run make.go -v windows
 test_script:
-- cmd: buildOutput\velociraptor.exe golden artifacts\testdata\windows\
+- cmd: output\velociraptor.exe golden artifacts\testdata\windows\
 on_finish:
   - ps: Get-ChildItem artifacts/testdata/windows/*.out.yaml | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name }

artifacts/assets.go

@@ -2,12 +2,14 @@ package artifacts
 
 import (
 	"strings"
+
 	"www.velocidex.com/golang/velociraptor/artifacts/assets"
 	config "www.velocidex.com/golang/velociraptor/config"
 	logging "www.velocidex.com/golang/velociraptor/logging"
 )
 
 func register(config_obj *config.Config) error {
+	assets.Init()
 	files, err := assets.WalkDirs("", false)
 	if err != nil {
 		return err

artifacts/b0x.yaml

@@ -12,3 +12,4 @@ output: "ab0x.go"
 custom:
   - files:
       - "artifacts/definitions/**/*.yaml"
+init: Init

artifacts/definitions/events/stats.yaml

@@ -8,4 +8,13 @@ parameters:
 sources:
   - queries:
       - |
-        SELECT UnixNano FROM clock(period=atoi(string=Frequency))
+        SELECT * from foreach(
+         row={
+           SELECT UnixNano FROM clock(period=atoi(string=Frequency))
+         },
+         query={
+           SELECT UnixNano / 1000000000 as Timestamp,
+                  Times.user + Times.system as CPU,
+                  MemoryInfo.RSS as RSS
+           FROM pslist(pid=getpid())
+         })

bin/frontend.go

@@ -18,6 +18,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/config"
 	crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto"
 	"www.velocidex.com/golang/velociraptor/flows"
+	"www.velocidex.com/golang/velociraptor/gui/assets"
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/server"
 )
@@ -85,6 +86,7 @@ func start_frontend(config_obj *config.Config, server_obj *server.Server) {
 	// Start the hunt dispatcher.
 	_, err := flows.GetHuntDispatcher(config_obj)
 	kingpin.FatalIfError(err, "Hunt dispatcher")
+
 	server_obj.Info("Frontend is ready to handle client requests at %s", listenAddr)
 	atomic.StoreInt32(&healthy, 1)
 	if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
@@ -372,6 +374,9 @@ func init() {
 			// Parse the artifacts database to detect errors early.
 			getRepository(config_obj)
 
+			// Load the assets into memory.
+			assets.Init()
+
 			go func() {
 				err := api.StartServer(config_obj, server_obj)
 				kingpin.FatalIfError(

config/b0x.yaml

@@ -12,3 +12,4 @@ output: "ab0x.go"
 custom:
   - files:
       - "config/default.yaml"
+init: Init

gui/assets/init.go

@@ -0,0 +1,5 @@
+// +build !release
+
+package assets
+
+func Init() {}

gui/b0x.yaml

@@ -32,3 +32,4 @@ custom:
       - "**/*~"
       - "**/*.js"
       - "**/*.scss"
+init: Init

magefile.go

@@ -0,0 +1,197 @@
+//+build mage
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/magefile/mage/mg"
+	"github.com/magefile/mage/sh"
+)
+
+var (
+	assets = map[string]string{
+		"artifacts/b0x.yaml": "artifacts/assets/ab0x.go",
+		"config/b0x.yaml":    "config/ab0x.go",
+		"gui/b0x.yaml":       "gui/assets/ab0x.go",
+	}
+
+	mingw_xcompiler = "x86_64-w64-mingw32-gcc"
+	name            = "velociraptor"
+)
+
+func Linux() error {
+	if err := os.Mkdir("output", 0700); err != nil && !os.IsExist(err) {
+		return fmt.Errorf("failed to create output: %v", err)
+	}
+
+	err := ensure_assets()
+	if err != nil {
+		return err
+	}
+
+	env := make(map[string]string)
+	err = sh.RunWith(
+		env,
+		mg.GoCmd(), "build",
+		"-o", filepath.Join("output", name),
+		"-tags", "release",
+		"-ldflags=-s -w "+flags(),
+		"./bin/")
+
+	if err != nil {
+		return err
+	}
+
+	return err
+}
+
+// Builds a Development binary. This does not embed things like GUI
+// resources to allow them to be loaded from the local directory.
+func Dev() error {
+	if err := os.Mkdir("output", 0700); err != nil && !os.IsExist(err) {
+		return fmt.Errorf("failed to create output: %v", err)
+	}
+
+	err := ensure_assets()
+	if err != nil {
+		return err
+	}
+
+	env := make(map[string]string)
+	err = sh.RunWith(
+		env,
+		mg.GoCmd(), "build",
+		"-o", filepath.Join("output", name),
+		"-tags", "devel",
+		"-ldflags=-s -w "+flags(),
+		"./bin/")
+
+	if err != nil {
+		return err
+	}
+
+	return err
+}
+
+// Cross compile the windows binary using mingw
+func Windows() error {
+	if err := os.Mkdir("output", 0700); err != nil && !os.IsExist(err) {
+		return fmt.Errorf("failed to create output: %v", err)
+	}
+
+	err := ensure_assets()
+	if err != nil {
+		return err
+	}
+
+	env := make(map[string]string)
+	if mingwxcompiler_exists() {
+		env["CC"] = mingw_xcompiler
+		env["CGO_ENABLED"] = "1"
+	} else {
+		fmt.Printf("Windows cross compiler %v not found. Disabling cgo modules.")
+	}
+
+	env["GOOS"] = "windows"
+	env["GOARCH"] = "amd64"
+
+	err = sh.RunWith(
+		env,
+		mg.GoCmd(), "build",
+		"-o", filepath.Join("output", name+".exe"),
+		"-tags", "release",
+		"-ldflags=-s -w "+flags(),
+		"./bin/")
+
+	if err != nil {
+		return err
+	}
+
+	return err
+}
+
+func Clean() error {
+	for _, target := range assets {
+		err := sh.Rm(target)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func flags() string {
+	timestamp := time.Now().Format(time.RFC3339)
+	return fmt.Sprintf(`-X "www.velocidex.com/golang/velociraptor/config.build_time=%s" -X "www.velocidex.com/golang/velociraptor/config.commit_hash=%s"`, timestamp, hash())
+}
+
+// hash returns the git hash for the current repo or "" if none.
+func hash() string {
+	hash, _ := sh.Output("git", "rev-parse", "--short", "HEAD")
+	return hash
+}
+
+func fileb0x(asset string) error {
+	err := sh.Run("fileb0x", asset)
+	if err != nil {
+		err = sh.Run(mg.GoCmd(), "get", "github.com/UnnoTed/fileb0x")
+		if err != nil {
+			return err
+		}
+
+		err = sh.Run("fileb0x", asset)
+	}
+
+	return err
+}
+
+func ensure_assets() error {
+	for asset, target := range assets {
+		before := timestamp_of(target)
+		err := fileb0x(asset)
+		if err != nil {
+			return err
+		}
+		// Only do this if the file has changed.
+		if before != timestamp_of(target) {
+			err = replace_string_in_file(target, "func init()", "func Init()")
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func mingwxcompiler_exists() bool {
+	err := sh.Run(mingw_xcompiler, "--version")
+	return err == nil
+}
+
+// Temporarily manipulate the generated file until
+// https://github.com/UnnoTed/fileb0x/pull/46 goes in.
+func replace_string_in_file(filename string, old string, new string) error {
+	read, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return err
+	}
+	newContents := strings.Replace(string(read), old, new, -1)
+	return ioutil.WriteFile(filename, []byte(newContents), 0)
+}
+
+func timestamp_of(path string) int64 {
+	stat, err := os.Stat(path)
+	if os.IsNotExist(err) || err != nil {
+		return 0
+	}
+
+	return stat.ModTime().UnixNano()
+}

make.go

@@ -0,0 +1,11 @@
+// +build ignore
+
+package main
+
+import (
+	"os"
+
+	"github.com/magefile/mage/mage"
+)
+
+func main() { os.Exit(mage.Main()) }

vql/functions/pid.go

@@ -0,0 +1,29 @@
+package functions
+
+import (
+	"context"
+	"os"
+
+	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
+	"www.velocidex.com/golang/vfilter"
+)
+
+type GetPidFunction struct{}
+
+func (self *GetPidFunction) Call(ctx context.Context,
+	scope *vfilter.Scope,
+	args *vfilter.Dict) vfilter.Any {
+
+	return os.Getpid()
+}
+
+func (self GetPidFunction) Info(type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
+	return &vfilter.FunctionInfo{
+		Name: "getpid",
+		Doc:  "Returns the current pid of the process.",
+	}
+}
+
+func init() {
+	vql_subsystem.RegisterFunction(&GetPidFunction{})
+}