Implemented Artifact packs.

- Artifacts can now express many tables, each with their own name and
  description.
- Descriptions can be formatted using go templates. It is possible to
  express additional queries of the same artifact within the
  description. This allows the text description to present highlights
  of the main artifact collected.

actions/proto/vql.proto

@@ -11,6 +11,9 @@ message VQLRequest {
     string Name = 2 [(sem_type) = {
             description: "The name of this query. This should be descriptive to indicate what type of informaiton the query retrieves.",
         }];
+    string Description = 3 [(sem_type) = {
+            description: "What this query is supposed to do.",
+        }];
     string VQL = 1 [(sem_type) = {
             description: "The VQL query to execute on the client.",
         }];

artifacts/artifacts.go

@@ -139,6 +139,28 @@ func (self *Repository) GetQueryDependencies(query string) []string {
 	return result
 }
 
+// Expand artifact packs (i.e. those with multiple sources) into their
+// respective artifacts.
+func (self *Repository) GetArtifactNames(names []string) []string {
+	result := []string{}
+
+name_list:
+	for _, name := range names {
+		definition, pres := self.Data[name]
+		if pres {
+			for _, source := range definition.Sources {
+				if source.Name == "" {
+					result = append(result, name)
+					continue name_list
+				}
+
+				result = append(result, source.Name)
+			}
+		}
+	}
+	return result
+}
+
 func (self *Repository) PopulateArtifactsVQLCollectorArgs(
 	request *actions_proto.VQLCollectorArgs) {
 	dependencies := make(map[string]bool)
@@ -194,16 +216,49 @@ func Compile(artifact *artifacts_proto.Artifact,
 		})
 	}
 
-	source_precondition := ""
 	for idx, source := range artifact.Sources {
-		prefix := fmt.Sprintf("%s_%d", escape_name(artifact.Name), idx)
-		source_result := ""
+		// If a precondition is defined at the artifact level, the
+		// source may override it.
+		source_precondition := artifact.Precondition
+		source_precondition_var := ""
 		if source.Precondition != "" {
-			source_precondition = "precondition_" + prefix
+			source_precondition = source.Precondition
+		}
+
+		// If the source has specialized name and description
+		// we use it otherwise take the name and description
+		// from the artifact itself. This allows us to create
+		// an artifact pack which contains multiple related
+		// artifacts in the sources list.
+
+		// NOTE: The client does not receive the actual name
+		// or description because we compress the
+		// VQLCollectorArgs object before we sent it to them
+		// (i.e. substitute the strings with place holders).
+		// It is therefore safe to include confidential
+		// information in the description or name properties
+		// of an artifact (Although obviously the client can
+		// see the actual VQL query that its running).
+		name := artifact.Name
+		description := artifact.Description
+
+		if source.Name != "" {
+			name = source.Name
+		}
+
+		if source.Description != "" {
+			description = source.Description
+		}
+
+		prefix := fmt.Sprintf("%s_%d", escape_name(name), idx)
+		source_result := ""
+
+		if source_precondition != "" {
+			source_precondition_var = "precondition_" + prefix
 			result.Query = append(result.Query,
 				&actions_proto.VQLRequest{
-					VQL: "LET " + source_precondition + " = " +
-						source.Precondition,
+					VQL: "LET " + source_precondition_var + " = " +
+						source_precondition,
 				})
 		}
 
@@ -250,15 +305,17 @@ func Compile(artifact *artifacts_proto.Artifact,
 
 		if source.Precondition != "" {
 			result.Query = append(result.Query, &actions_proto.VQLRequest{
-				Name: "Artifact " + artifact.Name,
+				Name:        "Artifact " + name,
+				Description: description,
 				VQL: fmt.Sprintf(
 					"SELECT * FROM if(then=%s, condition=%s)",
-					source_result, source_precondition),
+					source_result, source_precondition_var),
 			})
 		} else {
 			result.Query = append(result.Query, &actions_proto.VQLRequest{
-				Name: "Artifact " + artifact.Name,
-				VQL:  "SELECT * FROM " + source_result,
+				Name:        "Artifact " + name,
+				Description: description,
+				VQL:         "SELECT * FROM " + source_result,
 			})
 		}
 	}

artifacts/definitions/linux/acpi_tables.yaml

@@ -1,6 +1,7 @@
 name: Linux.Sys.ACPITables
 description: Firmware ACPI functional table common metadata and content.
-reference: https://osquery.io/schema/3.2.6#acpi_tables
+reference:
+  - https://osquery.io/schema/3.2.6#acpi_tables
 parameters:
   - name: kLinuxACPIPath
     default: /sys/firmware/acpi/tables

artifacts/definitions/linux/apt_sources.yaml

@@ -14,7 +14,8 @@ description: |
   We then try to open the file. If the file exists we parse some
   metadata from it. If not we leave those columns empty.
 
-reference: "https://osquery.io/schema/3.2.6#apt_sources"
+reference:
+  - https://osquery.io/schema/3.2.6#apt_sources
 parameters:
   - name: linuxAptSourcesGlobs
     description: Globs to find apt source *.list files.

artifacts/definitions/packs/windows/persistance.yaml

@@ -0,0 +1,34 @@
+name: Windows.Packs.Persistence
+description: |
+  This artifact pack collects various persistence mechanisms in Windows.
+
+precondition:
+  SELECT OS from info() where OS = "windows"
+
+sources:
+  - name: WMI Event Filters
+    description: |
+      {{ DocFrom "Windows.Persistence.PermanentWMIEvents" }}
+
+    queries:
+      - SELECT * FROM Artifact.Windows.Persistence.PermanentWMIEvents()
+
+  - name: Startup Items
+    description: |
+      {{ DocFrom "Windows.Sys.StartupItems" }}
+
+    queries:
+      - SELECT * FROM Artifact.Windows.Sys.StartupItems()
+
+  - name: Debug Bootstraping
+    description: |
+      {{ DocFrom "Windows.Persistence.Debug" }}
+
+      If there are any rows in the table below then executing the
+      program will also launch the program listed under the Debugger
+      column.
+
+      {{ Query "SELECT Program, Debugger FROM Rows" }}
+
+    queries:
+      - SELECT * FROM Artifact.Windows.Persistence.Debug()

artifacts/definitions/windows/appcompat_shims.yaml

@@ -4,8 +4,8 @@ description: |
   table presents the AppCompat Shim information from the registry in a
   nice format.
 
-reference: |
-  http://files.brucon.org/2015/Tomczak_and_Ballenthin_Shims_for_the_Win.pdf
+reference:
+  - http://files.brucon.org/2015/Tomczak_and_Ballenthin_Shims_for_the_Win.pdf
 
 parameters:
   - name: shimKeys

artifacts/definitions/windows/events/failed_logon.yaml

@@ -35,7 +35,7 @@ parameters:
 
 sources:
   - precondition:
-      SELECT OS From info() where OS = 'windows'
+      SELECT OS FROM info() where OS = 'windows'
     queries:
       - |
         LET failed_logon = SELECT EventData as FailedEventData,

artifacts/definitions/windows/firewall_rules.yaml

@@ -1,7 +1,7 @@
 name: Windows.Sys.FirewallRules
 description: List windows firewall rules.
 reference:
-  https://social.technet.microsoft.com/Forums/azure/en-US/aaed9c6a-fb8b-4d43-8b69-9f4e0f619a8c/how-to-check-the-windows-firewall-settings-from-netsh-command?forum=winserverGP
+  - https://social.technet.microsoft.com/Forums/azure/en-US/aaed9c6a-fb8b-4d43-8b69-9f4e0f619a8c/how-to-check-the-windows-firewall-settings-from-netsh-command?forum=winserverGP
 
 parameters:
   - name: regKey

artifacts/definitions/windows/physical_memory_ranges.yaml

@@ -1,7 +1,8 @@
 name: Windows.Sys.PhysicalMemoryRanges
 description: List Windows physical memory ranges.
-reference: |
-  https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/wdm/ns-wdm-_cm_resource_list
+reference:
+  - https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/wdm/ns-wdm-_cm_resource_list
+
 parameters:
   - name: physicalMemoryKey
     default: HKEY_LOCAL_MACHINE\HARDWARE\RESOURCEMAP\System Resources\Physical Memory\.Translated

artifacts/definitions/windows/programs.yaml

@@ -3,7 +3,8 @@ description: |
   Represents products as they are installed by Windows Installer. A product generally
   correlates to one installation package on Windows. Some fields may be blank as Windows
   installation details are left to the discretion of the product author.
-reference: https://github.com/facebook/osquery/blob/master/specs/windows/programs.table
+reference:
+  - https://github.com/facebook/osquery/blob/master/specs/windows/programs.table
 
 parameters:
   - name: programKeys

artifacts/definitions/windows/startup_items.yaml

@@ -1,7 +1,8 @@
 name: Windows.Sys.StartupItems
 description: Applications that will be started up from the various run key locations.
-reference: |
-  https://docs.microsoft.com/en-us/windows/desktop/setupapi/run-and-runonce-registry-keys
+reference:
+  - https://docs.microsoft.com/en-us/windows/desktop/setupapi/run-and-runonce-registry-keys
+
 parameters:
   - name: runKeyGlobs
     default: >