Tools setup page now allows specifying tool url (Velocidex#804)

Sometimes users need to host tools on external resources (e.g. S3
bucket). This change allows this url to be configured in the GUI so
all endpoints can download the tool from the specified URL.

Also this change includes some fixes to hunt and flow view refresh and
a fix to Windows.KapeFiles.Target that broke due to typed parameters.

artifacts/definitions/Windows/Collectors/File.yaml

@@ -26,7 +26,8 @@ sources:
       # Generate the collection globs for each device
       - LET specs = SELECT RootDevice + "\\" + Glob AS Glob
             FROM collectionSpec
-            WHERE log(message="Processing Device " + RootDevice + " With " + Accessor)
+            WHERE log(message=format(format="Processing Device %v with %v",
+                      args=[RootDevice, Accessor]))
 
       # Join all the collection rules into a single Glob plugin. This ensure we
       # only make one pass over the filesystem. We only want LFNs.

artifacts/definitions/Windows/KapeFiles/Targets.yaml

@@ -844,7 +844,7 @@ parameters:
       222,Setupapi.log XP,USBDevices,Windows/setupapi.log,lazy_ntfs,
       223,Setupapi.log Win7+,USBDevices,Windows\inf/setupapi.dev.log,lazy_ntfs,
       224,Setupapi.log Win7+,USBDevices,Windows.old\Windows\inf/setupapi.dev.log,lazy_ntfs,
-      225,Windows Your Phone - All Databases,Apps,Users\*\AppData\Local\Packages\Microsoft.YourPhone_8wekyb3d8bbwe\LocalCache\Indexed/**10,lazy_ntfs,Locates all Your Phone database files 
+      225,Windows Your Phone - All Databases,Apps,Users\*\AppData\Local\Packages\Microsoft.YourPhone_8wekyb3d8bbwe\LocalCache\Indexed/**10,lazy_ntfs,Locates all Your Phone database files
       226,Prefetch,Prefetch,Windows\prefetch/*.pf,lazy_ntfs,
       227,Prefetch,Prefetch,Windows.old\Windows\prefetch/*.pf,lazy_ntfs,
       228,Syscache,Program Execution,System Volume Information/Syscache.hve,lazy_ntfs,
@@ -1670,14 +1670,13 @@ sources:
                    -- For VSS we always need to parse NTFS
                    SELECT * FROM Artifact.Windows.Collectors.VSS(
                       RootDevice=Device, Accessor="ntfs",
-                      collectionSpec=serialize(item=rule_specs_ntfs, format="csv"))
+                      collectionSpec=rule_specs_ntfs)
                }, b={
                    SELECT * FROM Artifact.Windows.Collectors.VSS(
                       RootDevice=Device,
                       Accessor=if(condition=DontBeLazy,
                                   then="ntfs", else="lazy_ntfs"),
-                      collectionSpec=serialize(
-                        item=rule_specs_lazy_ntfs, format="csv"))
+                      collectionSpec=rule_specs_lazy_ntfs)
                })
            }, else={
              SELECT * FROM chain(
@@ -1687,8 +1686,7 @@ sources:
                    SELECT * FROM Artifact.Windows.Collectors.File(
                       RootDevice=Device,
                       Accessor="ntfs",
-                      collectionSpec=serialize(
-                          item=rule_specs_ntfs, format="csv"))
+                      collectionSpec=rule_specs_ntfs)
                }, b={
 
                    -- Prefer the auto accessor if possible since it
@@ -1698,10 +1696,10 @@ sources:
                       RootDevice=Device,
                       Accessor=if(condition=UseAutoAccessor,
                                   then="auto", else="lazy_ntfs"),
-                      collectionSpec=serialize(
-                         item=rule_specs_lazy_ntfs, format="csv"))
+                      collectionSpec=rule_specs_lazy_ntfs)
                })
            })
+
       SELECT * FROM all_results WHERE _Source =~ "Metadata"
 
   - name: Uploads
@@ -1762,5 +1760,3 @@ reports:
       {{ end }}
 
       {{ Query $query | Table }}
-
-

artifacts/obfuscation.go

@@ -3,6 +3,7 @@ package artifacts
 import (
 	"regexp"
 
+	"github.com/pkg/errors"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/crypto"
@@ -41,7 +42,7 @@ func Obfuscate(
 		// forms. This removes comments.
 		ast, err := vfilter.Parse(query.VQL)
 		if err != nil {
-			return err
+			return errors.Wrap(err, "While parsing VQL: "+query.VQL)
 		}
 
 		// TODO: Compress the AST.

bin/config_interactive.go

@@ -65,7 +65,7 @@ What OS will the server be deployed on?
 	}
 
 	// https://docs.microsoft.com/en-us/troubleshoot/windows-server/identity/naming-conventions-for-computer-domain-site-ou#dns-host-names
-	url_validator = regexValidator("^[a-z0-9.-A-Z]+$")
+	url_validator = regexValidator("^[a-z0-9.A-Z]+$")
 	port_question = &survey.Input{
 		Message: "Enter the frontend port to listen on.",
 		Default: "8000",

gui/velociraptor/src/components/flows/client-flows-view.js

@@ -53,9 +53,6 @@ class ClientFlowsView extends React.Component {
             return;
         }
 
-        let selected_flow_id = this.props.match && this.props.match.params &&
-            this.props.match.params.flow_id;
-
         // Cancel any in flight calls.
         this.source.cancel();
         this.source = axios.CancelToken.source();
@@ -67,24 +64,28 @@ class ClientFlowsView extends React.Component {
             if (response.cancel) return;
 
             let flows = response.data.items || [];
-            let selected_flow = {};
+            let selected_flow_id = this.state.currentFlow.session_id;
 
             // If the router specifies a selected flow id, we select it.
             if (!this.state.init_router) {
-                for(var i=0;i<flows.length;i++) {
-                    let flow=flows[i];
-                    if (flow.session_id === selected_flow_id) {
-                        selected_flow = flow;
-                        break;
-                    }
-                };
+                selected_flow_id = this.props.match && this.props.match.params &&
+                    this.props.match.params.flow_id;
 
                 // If we can not find the selected_flow we just select the first one
-                if (_.isEmpty(selected_flow) && !_.isEmpty(flows)){
-                    selected_flow = flows[0];
+                if (!selected_flow_id && !_.isEmpty(flows)){
+                    selected_flow_id = flows[0].session_id;
                 }
 
-                this.setState({init_router: true, currentFlow: selected_flow});
+                this.setState({init_router: true});
+            }
+
+            // Update the current selected flow with the new data.
+            if (selected_flow_id) {
+                _.each(flows, flow=>{
+                    if(flow.session_id == selected_flow_id) {
+                        this.setState({currentFlow: flow});
+                    };
+                });
             }
 
             this.setState({flows: flows, loading: false});

gui/velociraptor/src/components/hunts/hunts.js

@@ -29,15 +29,15 @@ class VeloHunts extends React.Component {
     }
 
     componentDidMount = () => {
-        this.source = axios.CancelToken.source();
         this.get_hunts_source = axios.CancelToken.source();
         this.list_hunts_source = axios.CancelToken.source();
         this.interval = setInterval(this.fetchHunts, POLL_TIME);
         this.fetchHunts();
     }
 
     componentWillUnmount() {
-        this.source.cancel();
+        this.list_hunts_source.cancel();
+        this.get_hunts_source.cancel();
         clearInterval(this.interval);
     }
 
@@ -87,25 +87,33 @@ class VeloHunts extends React.Component {
             if (response.cancel) return;
 
             let hunts = response.data.items || [];
+            let selected_hunt_id = this.state.selected_hunt && this.state.selected_hunt.hunt_id;
+
+            // If the router specifies a selected flow id, we select
+            // it but only once.
+            if (!this.state.init_router && !selected_hunt_id) {
+                selected_hunt_id = this.props.match && this.props.match.params &&
+                    this.props.match.params.hunt_id;
 
-            // If the router specifies a selected flow id, we select it.
-            if (!this.state.init_router) {
-                for(var i=0;i<hunts.length;i++) {
-                    let hunt=hunts[i];
-                    if (hunt.hunt_id === selected_hunt_id) {
-                        this.loadFullHunt(hunt);
-                        break;
-                    }
-                };
                 this.setState({init_router: true});
             }
 
+            // Update the selected hunt in the state.
+            if (selected_hunt_id) {
+                _.each(hunts, hunt=>{
+                    if (hunt.hunt_id === selected_hunt_id) {
+                        this.setState({selected_hunt: hunt});
+                    };
+                });
+            };
+
             this.setState({hunts: hunts});
         });
 
-        // Get the full hunt information from the server based on the hunt
-        // metadata
-        if (!this.state.init_router && selected_hunt_id) {
+        // Get the full hunt information from the server based on the
+        // hunt metadata - this is done every poll interval to ensure
+        // hunt stats are updated.
+        if (selected_hunt_id) {
             this.get_hunts_source.cancel();
             this.get_hunts_source = axios.CancelToken.source();
 

gui/velociraptor/src/components/tools/tool-viewer.js

@@ -22,28 +22,27 @@ export default class ToolViewer extends React.Component {
     };
 
     componentDidMount = () => {
-
         this.fetchToolInfo();
     }
 
     componentDidUpdate = (prevProps, prevState, rootNode) => {
-        if (!this.state.initialized_from_parent && this.props.name) {
+        if (this.props.name != prevProps.name) {
             this.fetchToolInfo();
         }
     }
 
     fetchToolInfo = () => {
         api.get("v1/GetToolInfo",
                 {name: this.props.name}).then((response) => {
-            this.setState({tool: response.data, initialized_from_parent: true});
+            this.setState({tool: response.data});
         });
     }
 
     state = {
         showDialog: false,
-        initialized_from_parent: false,
         tool: {},
         tool_file: null,
+        remote_url: "",
     }
 
     uploadFile = () => {
@@ -57,6 +56,16 @@ export default class ToolViewer extends React.Component {
         });
     }
 
+    setServeUrl = url=>{
+        let tool = Object.assign({}, this.state.tool);
+        tool.url = this.state.remote_url;
+        tool.hash = "";
+        tool.filename = "";
+        tool.github_project = "";
+        tool.serve_locally = false;
+        tool.materialize = true;
+        this.setToolInfo(tool);
+    };
 
     setToolInfo = (tool) => {
         this.setState({inflight: true});
@@ -342,7 +351,8 @@ export default class ToolViewer extends React.Component {
                           As an admin you can manually upload a
                           binary to be used as that tool. This will override the
                           upstream URL setting and provide your tool to all
-                          artifacts that need it.
+                          artifacts that need it. Alternative, set a URL for clients
+                          to fetch tools from.
                         </Card.Text>
                         <Form className="selectable">
                           <InputGroup className="mb-3">
@@ -370,6 +380,24 @@ export default class ToolViewer extends React.Component {
                             </Form.File>
                           </InputGroup>
                         </Form>
+                        <Form className="selectable">
+                          <InputGroup>
+                            <InputGroup.Prepend>
+                              <InputGroup.Text  as="button"
+                                disabled={this.state.inflight || !this.state.remote_url}
+                                onClick={this.setServeUrl}>
+                                { this.state.inflight ?
+                                  <FontAwesomeIcon icon="spinner" spin /> :
+                                  "Set Serve URL" }
+                              </InputGroup.Text>
+                            </InputGroup.Prepend>
+                            <Form.Control as="input"
+                                          value={this.state.remote_url}
+                                          onChange={e=>this.setState(
+                                              {remote_url: e.currentTarget.value})}
+                            />
+                          </InputGroup>
+                        </Form>
                       </Card.Body>
                     </Card>
                   </CardDeck>

scripts/kape_files.py

@@ -214,14 +214,13 @@ def format(ctx):
                    -- For VSS we always need to parse NTFS
                    SELECT * FROM Artifact.Windows.Collectors.VSS(
                       RootDevice=Device, Accessor="ntfs",
-                      collectionSpec=serialize(item=rule_specs_ntfs, format="csv"))
+                      collectionSpec=rule_specs_ntfs)
                }, b={
                    SELECT * FROM Artifact.Windows.Collectors.VSS(
                       RootDevice=Device,
                       Accessor=if(condition=DontBeLazy,
                                   then="ntfs", else="lazy_ntfs"),
-                      collectionSpec=serialize(
-                        item=rule_specs_lazy_ntfs, format="csv"))
+                      collectionSpec=rule_specs_lazy_ntfs)
                })
            }, else={
              SELECT * FROM chain(
@@ -231,8 +230,7 @@ def format(ctx):
                    SELECT * FROM Artifact.Windows.Collectors.File(
                       RootDevice=Device,
                       Accessor="ntfs",
-                      collectionSpec=serialize(
-                          item=rule_specs_ntfs, format="csv"))
+                      collectionSpec=rule_specs_ntfs)
                }, b={
 
                    -- Prefer the auto accessor if possible since it
@@ -242,8 +240,7 @@ def format(ctx):
                       RootDevice=Device,
                       Accessor=if(condition=UseAutoAccessor,
                                   then="auto", else="lazy_ntfs"),
-                      collectionSpec=serialize(
-                         item=rule_specs_lazy_ntfs, format="csv"))
+                      collectionSpec=rule_specs_lazy_ntfs)
                })
            })
       SELECT * FROM all_results WHERE _Source =~ "Metadata"

services/launcher/compiler.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"path"
 	"regexp"
-	"strings"
 
 	errors "github.com/pkg/errors"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
@@ -20,12 +19,20 @@ import (
 
 var (
 	artifact_in_query_regex = regexp.MustCompile(`Artifact\.([^\s\(]+)\(`)
+	escape_regex            = regexp.MustCompile("(^[0-9]|[\"'_ ])")
 )
 
 func escape_name(name string) string {
 	return regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(name, "_")
 }
 
+func maybeEscape(name string) string {
+	if escape_regex.FindString(name) != "" {
+		return "`" + name + "`"
+	}
+	return name
+}
+
 func Compile(config_obj *config_proto.Config,
 	repository services.Repository, artifact *artifacts_proto.Artifact,
 	result *actions_proto.VQLCollectorArgs) error {
@@ -45,9 +52,7 @@ func Compile(config_obj *config_proto.Config,
 
 		// If the variable contains spaces we need to escape
 		// the name in backticks.
-		if strings.Contains(name, " ") {
-			name = "`" + name + "`"
-		}
+		name = maybeEscape(name)
 
 		switch parameter.Type {
 		case "int", "int64":

services/launcher/fixtures/TestParameterTypesDeps.golden

@@ -0,0 +1,15 @@
+[
+ {
+  "IntValue": 9,
+  "CSVValue": [
+   {
+    "Col1": "Value1",
+    "Col2": "Value2"
+   },
+   {
+    "Col1": "Value3",
+    "Col2": "Value4"
+   }
+  ]
+ }
+]