Switch json encoder to support time serialization. (Velocidex#507)

The encoding/json does not support context aware encoding. This changes wraps all json operations and redirects to github.com/Velocidex/json which supports context aware encoding. We set up overrides for time objects which control the RFC 3339 encoding. It is possible now to control timezone by simply setting the VQL: LET TZ <= "Australia/Brisbane" Will force all tables in this cell to be rendered in local time. Otherwise times are rendered in UTC.
simiaoxiaoseng · Jul 24, 2020 · a54c649 · a54c649
1 parent 672fd28
commit a54c649
Show file tree

Hide file tree

Showing 70 changed files with 589 additions and 529 deletions.
diff --git a/actions/vql.go b/actions/vql.go
@@ -152,7 +152,10 @@ func (self VQLClientAction) StartQuery(
 		}
 
 		result_chan := vfilter.GetResponseChannel(
-			vql, sub_ctx, scope, int(max_row), int(max_wait))
+			vql, sub_ctx, scope,
+			vql_subsystem.MarshalJson(scope),
+			int(max_row),
+			int(max_wait))
 	run_query:
 		for {
 			select {

diff --git a/api/auth.go b/api/auth.go
@@ -19,13 +19,13 @@ package api
 
 import (
 	"context"
-	"encoding/json"
 
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/json"
 )
 
 var (

diff --git a/api/authenticators/azure.go b/api/authenticators/azure.go
@@ -18,7 +18,6 @@
 package authenticators
 
 import (
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -33,6 +32,7 @@ import (
 	"golang.org/x/oauth2/microsoft"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/constants"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/logging"
 )
 

diff --git a/api/authenticators/basic.go b/api/authenticators/basic.go
@@ -2,14 +2,14 @@ package authenticators
 
 import (
 	"context"
-	"encoding/json"
 	"net/http"
 
 	"github.com/gorilla/csrf"
 	"github.com/sirupsen/logrus"
 	"www.velocidex.com/golang/velociraptor/acls"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/users"
 )

diff --git a/api/authenticators/github.go b/api/authenticators/github.go
@@ -18,7 +18,6 @@
 package authenticators
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -32,6 +31,7 @@ import (
 	"golang.org/x/oauth2/github"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/constants"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/logging"
 )
 

diff --git a/api/authenticators/google.go b/api/authenticators/google.go
@@ -20,7 +20,6 @@ package authenticators
 import (
 	"crypto/rand"
 	"encoding/base64"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -38,6 +37,7 @@ import (
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/constants"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/logging"
 	users "www.velocidex.com/golang/velociraptor/users"
 )

diff --git a/api/authenticators/logging.go b/api/authenticators/logging.go
@@ -2,12 +2,12 @@ package authenticators
 
 import (
 	"context"
-	"encoding/json"
 	"net/http"
 
 	"github.com/sirupsen/logrus"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/logging"
 )
 

diff --git a/api/authenticators/saml.go b/api/authenticators/saml.go
@@ -2,7 +2,6 @@ package authenticators
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -14,6 +13,7 @@ import (
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/crypto"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/users"
 )

diff --git a/api/csv.go b/api/csv.go
@@ -82,7 +82,7 @@ func getTable(
 			}
 
 			row_data := make([]string, 0, len(result.Columns))
-			for _, key := range row.Keys() {
+			for _, key := range result.Columns {
 				value, _ := row.Get(key)
 				row_data = append(row_data, csv.AnyToString(value))
 			}

diff --git a/api/handlers.go b/api/handlers.go
@@ -19,12 +19,12 @@ package api
 
 import (
 	"context"
-	"encoding/json"
 	"net/http"
 
 	"github.com/sirupsen/logrus"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/logging"
 )
 

diff --git a/api/notebooks.go b/api/notebooks.go
@@ -5,7 +5,6 @@ import (
 	"encoding/base32"
 	"encoding/base64"
 	"encoding/binary"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -24,6 +23,7 @@ import (
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/datastore"
 	file_store "www.velocidex.com/golang/velociraptor/file_store"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/reporting"
 	"www.velocidex.com/golang/velociraptor/services"

diff --git a/api/query.go b/api/query.go
@@ -116,7 +116,9 @@ func streamQuery(
 				fmt.Printf("Running %v\n", vql.ToString(scope))
 
 				result_chan := vfilter.GetResponseChannel(
-					vql, stream.Context(), scope, int(arg.MaxRow), int(arg.MaxWait))
+					vql, stream.Context(), scope,
+					vql_subsystem.MarshalJson(scope),
+					int(arg.MaxRow), int(arg.MaxWait))
 
 				for result := range result_chan {
 					fmt.Printf("Got %v\n", string(result.Payload))

diff --git a/api/reports.go b/api/reports.go
@@ -1,7 +1,6 @@
 package api
 
 import (
-	"encoding/json"
 	"strings"
 
 	errors "github.com/pkg/errors"
@@ -10,6 +9,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/artifacts"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/constants"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/reporting"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 )

diff --git a/api/vfs.go b/api/vfs.go
@@ -62,7 +62,6 @@ GetVFSDownloadInfoPath().
 package api
 
 import (
-	"encoding/json"
 	"fmt"
 	"strings"
 
@@ -72,6 +71,7 @@ import (
 	datastore "www.velocidex.com/golang/velociraptor/datastore"
 	file_store "www.velocidex.com/golang/velociraptor/file_store"
 	flows_proto "www.velocidex.com/golang/velociraptor/flows/proto"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/paths"
 	"www.velocidex.com/golang/velociraptor/utils"
 )
@@ -177,7 +177,7 @@ func renderDBVFS(
 			}
 		}
 
-		encoded_rows, err := json.MarshalIndent(rows, "", " ")
+		encoded_rows, err := json.MarshalIndent(rows)
 		if err != nil {
 			return nil, err
 		}
@@ -228,7 +228,7 @@ func renderFileStore(
 		}
 	}
 
-	encoded_rows, err := json.MarshalIndent(rows, "", " ")
+	encoded_rows, err := json.MarshalIndent(rows)
 	if err != nil {
 		return nil, err
 	}

diff --git a/api/vql.go b/api/vql.go
@@ -18,7 +18,6 @@
 package api
 
 import (
-	"encoding/json"
 	"io"
 
 	"github.com/Velocidex/ordereddict"
@@ -28,6 +27,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/artifacts"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/file_store/csv"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/logging"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/vfilter"

diff --git a/artifacts/definitions/Linux/Sys/Pslist.yaml b/artifacts/definitions/Linux/Sys/Pslist.yaml
@@ -0,0 +1,19 @@
+name: Linux.Sys.Pslist
+description: |
+  List processes and their running binaries.
+
+parameters:
+  - name: processRegex
+    default: .
+
+precondition: SELECT OS From info() where OS = 'linux'
+
+sources:
+  - query: |
+        SELECT Pid, Ppid, Name, Cmdline, Exe,
+               hash(path=Exe) as Hash,
+               Username, timestamp(epoch=CreateTime/1000) AS CreatedTime,
+               MemoryInfo.RSS AS RSS,
+               Exe =~ "\\(deleted\\)$" AS Deleted
+        FROM pslist()
+        WHERE Name =~ processRegex
diff --git a/artifacts/definitions/Server/Information/Users.yaml b/artifacts/definitions/Server/Information/Users.yaml
@@ -8,23 +8,23 @@ type: SERVER
 
 parameters:
   - name: ClientId
-    default:
+    default: C.56a8dfd31eb1fa6f
+
   - name: StandardUserAccounts
     description: Well known SIDs to hide from the output.
     default: "(-5..$|S-1-5-18|S-1-5-19|S-1-5-20)"
 
 sources:
-  - queries:
-      - |
+  - query: |
         // Get the most recent collection of our user listing.
-        LET last_user_listing = SELECT flow_id
+        LET last_user_listing = SELECT session_id AS flow_id
            FROM flows(client_id=ClientId)
-           WHERE context.artifacts =~'Windows.Sys.Users'
-           ORDER BY LastActive DESC LIMIT 1
+           WHERE artifacts_with_results =~'Windows.Sys.Users'
+           ORDER BY LastActive
+           DESC LIMIT 1
 
-      - |
         /* For each Windows.Sys.Users collection, extract the user
-           names. Hide standard SIDs.
+           names, but hide standard SIDs.
         */
         SELECT * FROM foreach(
             row=last_user_listing,

diff --git a/artifacts/definitions/Windows/System/Pslist.yaml b/artifacts/definitions/Windows/System/Pslist.yaml
@@ -6,6 +6,8 @@ parameters:
   - name: processRegex
     default: .
 
+precondition: SELECT OS From info() where OS = 'windows'
+
 sources:
   - queries:
       - |

diff --git a/bin/config.go b/bin/config.go
@@ -23,7 +23,6 @@ import (
 	"crypto/rand"
 	"crypto/x509"
 	"encoding/base64"
-	"encoding/json"
 	"encoding/pem"
 	"fmt"
 	"os"
@@ -38,6 +37,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/config"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/crypto"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/logging"
 )
 

diff --git a/bin/fuse.go b/bin/fuse.go
@@ -1,4 +1,4 @@
-// +build linux
+// +build disabled
 
 package main
 

diff --git a/bin/fuse_windows.go b/bin/fuse_windows.go
@@ -1,4 +1,4 @@
-// +build windows
+// +build disabled
 
 /*
 A fuse implementation using cgofuse and WinFSp.

diff --git a/bin/golden.go b/bin/golden.go
@@ -139,7 +139,9 @@ func runTest(fixture *testFixture) (string, error) {
 		}
 
 		result_chan := vfilter.GetResponseChannel(
-			vql, context.Background(), scope, 1000, 1000)
+			vql, context.Background(), scope,
+			vql_subsystem.MarshalJsonIndent(scope),
+			1000, 1000)
 		for {
 			query_result, ok := <-result_chan
 			if !ok {

diff --git a/bin/grant.go b/bin/grant.go
@@ -1,7 +1,6 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"strings"
@@ -10,6 +9,7 @@ import (
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 	"www.velocidex.com/golang/velociraptor/acls"
 	acl_proto "www.velocidex.com/golang/velociraptor/acls/proto"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/utils"
 )