Added a geoip() VQL function. (Velocidex#1178)

api/events.go

@@ -250,7 +250,8 @@ func listAvailableEventArtifacts(
 		return nil, err
 	}
 
-	// Analyzer analyses the path name from disk and adds to the events list.
+	// getAllArtifacts analyses the path name from disk and adds
+	// to the events list.
 	seen := make(map[string]*api_proto.AvailableEvent)
 	err = getAllArtifacts(self.config, path_manager.GetRootPath(), seen)
 	if err != nil {
@@ -283,9 +284,18 @@ func getAllArtifacts(
 
 	return file_store_factory.Walk(log_path,
 		func(full_path api.FSPathSpec, info os.FileInfo) error {
+			// Walking the events directory will give us
+			// all the day json files. Each day json file
+			// is contained in a directory structure which
+			// reflects the name of the artifact, for
+			// example:
+
+			// <log_path>/Server.Monitor.Health/Prometheus/2021-08-01.json
+			// Corresponds to the artifact Server.Monitor.Health/Prometheus
 			if !info.IsDir() && info.Size() > 0 {
-				relative_path := full_path.Components()[len(log_path.Components()):]
-				artifact_name := strings.Join(relative_path, ".")
+				relative_path := full_path.Dir().
+					Components()[len(log_path.Components()):]
+				artifact_name := strings.Join(relative_path, "/")
 				event, pres := seen[artifact_name]
 				if !pres {
 					event = &api_proto.AvailableEvent{

artifacts/definitions/Server/Enrichment/GeoIP.yaml

@@ -0,0 +1,31 @@
+name: Server.Enrichment.GeoIP
+description: |
+  This artifact can use the MaxMind database to Geo resolve an IP
+  address. You will need to provide a valid GeoIP database.
+
+  You can obtain a free to use (gratis but not libre) database from
+  https://www.maxmind.com/ or you can pay for a more accurate option.
+
+  After storing the database somewhere on your server, you should the
+  location in the server metadata screen to it under the key "GeoIPDB"
+  (for example `/usr/shared/GeoLite2-City_20210803/GeoLite2-City.mmdb`)
+
+  Alternatively you can import this artifact to gain access to the
+  utility functions (or just copy them into your own artifact).
+
+export: |
+  LET DB = server_metadata().GeoIPDB
+  LET Country(IP) = geoip(db=DB, ip=IP).registered_country.names.en
+  LET State(IP) = geoip(db=DB, ip=IP).subdivisions[0].names.en
+  LET City(IP) = geoip(db=DB, ip=IP).city.names.en
+
+parameters:
+  - name: IP
+    description: An IP to lookup
+
+sources:
+  - query: |
+      SELECT Country(IP=_value) AS Country,
+             State(IP=_value) AS State,
+             City(IP=_value) AS City
+      FROM foreach(row=IP)

docs/references/vql.yaml

@@ -61,6 +61,30 @@
     type: string
     description: Required name prefix
   category: server
+- name: atexit
+  description: |
+    Install a query to run when the query is unwound. This is used to
+    clean up when the query ends.
+
+    For example:
+
+    ```vql
+    LET _ = atexit(query={
+      SELECT rm(filename="Foobar.txt") FROM scope()
+    })
+    ```
+  type: Function
+  args:
+  - name: query
+    type: Any
+    description: A VQL Query to parse and execute.
+    required: true
+  - name: env
+    type: ordereddict.Dict
+    description: A dict of args to insert into the scope.
+  - name: timeout
+    type: uint64
+    description: How long to wait for destructors to run (default 60 seconds).
 - name: atoi
   description: Convert a string to an int.
   type: Function
@@ -425,9 +449,11 @@
   args:
   - name: path
     type: string
-    description: A VFS path to compress
-    repeated: true
+    description: A path to compress
     required: true
+  - name: output
+    type: string
+    description: A path to write the output - default is the path with a .gz extension
   category: server
 - name: connections
   description: List all active connections
@@ -451,6 +477,9 @@
   - name: permissions
     type: string
     description: Required permissions (e.g. 'x').
+  - name: append
+    type: bool
+    description: If true we append to the target file otherwise truncate it
   category: basic
 - name: count
   description: Counts the items.
@@ -742,6 +771,26 @@
     description: A path with environment escapes
     required: true
   category: basic
+- name: favorites_save
+  description: Save a collection into the favorites.
+  type: Function
+  args:
+  - name: name
+    type: string
+    description: A name for this collection template.
+    required: true
+  - name: description
+    type: string
+    description: A description for the template.
+  - name: specs
+    type: LazyExpr
+    description: The collection request spec that will be saved. We use this to create
+      the new collection.
+    required: true
+  - name: type
+    type: string
+    description: The type of favorite.
+    required: true
 - name: fifo
   description: |
     Executes 'query' and cache a number of rows from it. For each invocation
@@ -952,6 +1001,41 @@
     type: Any
     description: An array of elements to apply into the format string.
   category: basic
+- name: gcs_pubsub_publish
+  description: Publish a message to Google PubSub.
+  type: Function
+  args:
+  - name: topic
+    type: string
+    description: The topic to publish to
+    required: true
+  - name: project_id
+    type: string
+    description: The project id to publish to
+    required: true
+  - name: msg
+    type: Any
+    description: Message to publish to Pubsub
+    required: true
+  - name: credentials
+    type: string
+    description: The credentials to use
+    required: true
+- name: geoip
+  description: |
+    Lookup an IP Address using the MaxMind GeoIP database. You can get
+    a copy of the database from https://www.maxmind.com/. The database
+    must be locally accessible on the server.
+  type: Function
+  args:
+  - name: ip
+    type: string
+    description: IP Address to lookup.
+    required: true
+  - name: db
+    type: string
+    description: Path to the MaxMind GeoIP Database.
+    required: true
 - name: get
   description: |
     Gets the member field from item.
@@ -1266,6 +1350,10 @@
   - name: HuntId
     type: string
     required: true
+  - name: FlowId
+    type: string
+    description: If a flow id is specified we do not create a new flow, but instead
+      add this flow_id to the hunt.
   category: server
 - name: hunt_flows
   description: Retrieve the flows launched by a hunt.
@@ -1731,10 +1819,42 @@
     we watch the global journal which contains event logs from all clients.
   type: Plugin
   args:
+  - name: client_id
+    type: string
+    description: The client id to extract
+  - name: flow_id
+    type: string
+    description: A flow ID (client or server artifacts)
+  - name: hunt_id
+    type: string
+    description: Retrieve sources from this hunt (combines all results from all clients)
   - name: artifact
     type: string
-    description: The event artifact name to watch
-    required: true
+    description: The name of the artifact collection to fetch
+  - name: source
+    type: string
+    description: An optional named source within the artifact
+  - name: start_time
+    type: Any
+    description: Start return events from this date (for event sources)
+  - name: end_time
+    type: Any
+    description: Stop end events reach this time (event sources).
+  - name: notebook_id
+    type: string
+    description: The notebook to read from (shoud also include cell id)
+  - name: notebook_cell_id
+    type: string
+    description: The notebook cell read from (shoud also include notebook id)
+  - name: notebook_cell_table
+    type: int64
+    description: A notebook cell can have multiple tables.)
+  - name: start_row
+    type: int64
+    description: Start reading the result set from this row
+  - name: count
+    type: int64
+    description: Maximum number of clients to fetch (default unlimited)'
   category: server
 - name: netstat
   description: Collect network information.
@@ -2499,7 +2619,8 @@
   args:
   - name: pid
     type: int64
-    description: A process ID to list. If not provided list all processes.
+    description: A pid to list. If this is provided we are able to operate much faster
+      by only opening a single process.
   category: plugin
 - name: query
   description: Launch a subquery and materialize it into a list of rows.
@@ -2518,7 +2639,7 @@
     description: A VQL Query to parse and execute.
     required: true
   - name: env
-    type: Any
+    type: ordereddict.Dict
     description: A dict of args to insert into the scope.
 - name: rand
   description: Selects a random number.
@@ -2663,6 +2784,14 @@
   - name: sep
     type: string
     description: Separator to use (default native)
+- name: rm
+  description: Remove a file from the filesystem using the API.
+  type: Function
+  args:
+  - name: filename
+    type: string
+    description: Filename to remove.
+    required: true
 - name: rot13
   description: Apply rot13 deobfuscation to the string.
   type: Function
@@ -3572,8 +3701,7 @@
   args:
   - name: artifact
     type: string
-    description: The event artifact name to watch
-    required: true
+    description: The artifact to watch
   category: event
 - name: watch_syslog
   description: 'Watch a syslog file and stream events from it. '

go.mod

@@ -79,6 +79,7 @@ require (
 	github.com/microcosm-cc/bluemonday v1.0.15
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/olekukonko/tablewriter v0.0.4
+	github.com/oschwald/maxminddb-golang v1.8.0 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.12.0
 	github.com/pquerna/cachecontrol v0.0.0-20200921180117-858c6e7e6b7e // indirect

go.sum

@@ -441,6 +441,8 @@ github.com/octago/sflags v0.2.0/go.mod h1:G0bjdxh4qPRycF74a2B8pU36iTp9QHGx0w0dFZ
 github.com/olekukonko/tablewriter v0.0.0-20180912035003-be2c049b30cc/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8=
 github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
+github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph9p/UMXK/Hk=
+github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
 github.com/paulmach/orb v0.1.5 h1:GUcATabvxciqEzGd+c01/9ek3B6pUp9OdcIHFSDDSSg=
 github.com/paulmach/orb v0.1.5/go.mod h1:pPwxxs3zoAyosNSbNKn1jiXV2+oovRDObDKfTvRegDI=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -683,6 +685,7 @@ golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

gui/velociraptor/src/components/events/timeline-viewer.js

@@ -430,7 +430,7 @@ export default class EventTimelineViewer extends React.Component {
                  >
                    <TimelineMarkers>
                      <CustomMarker
-                       date={this.state.start_time} >
+                       date={this.state.start_time || new Date()} >
                        { ({ styles, date }) => {
                            styles.backgroundColor = undefined;
                            styles.width = undefined;

result_sets/timed/reader.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/Velocidex/ordereddict"
 	"www.velocidex.com/golang/velociraptor/file_store/api"
-	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/paths"
 	"www.velocidex.com/golang/velociraptor/result_sets"
 	"www.velocidex.com/golang/velociraptor/timelines"
@@ -51,7 +50,6 @@ type TimedResultSetReader struct {
 
 func (self *TimedResultSetReader) GetAvailableFiles(
 	ctx context.Context) []*api.ResultSetFileProperties {
-	json.Dump(self.files[0].Path)
 	return self.files
 }
 

services/launcher/compiler.go

@@ -133,9 +133,21 @@ func resolveImports(config_obj *config_proto.Config,
 	// itself declares exports for itself (by default each
 	// artifact imports its own exports).
 	if artifact.Export != "" {
-		result.Query = append(result.Query, &actions_proto.VQLRequest{
-			VQL: artifact.Export,
-		})
+		scope := vql_subsystem.MakeScope()
+
+		// Support multiple queries in the export section.
+		queries, err := vfilter.MultiParse(artifact.Export)
+		if err != nil {
+			return fmt.Errorf("While parsing export in %s: %w",
+				artifact.Name, err)
+		}
+
+		for _, q := range queries {
+			result.Query = append(result.Query,
+				&actions_proto.VQLRequest{
+					VQL: q.ToString(scope),
+				})
+		}
 	}
 
 	if artifact.Imports == nil {

services/repository/filestore.go

@@ -49,11 +49,12 @@ func InitializeGlobalRepositoryFromFilestore(
 				string(data), false /* validate */)
 			if err != nil {
 				logger.Info("Unable to load custom "+
-					"artifact %s: %v", path, err)
+					"artifact %s: %v",
+					path.AsClientPath(), err)
 				return nil
 			}
 			artifact_obj.Raw = string(data)
-			logger.Info("Loaded %s", path)
+			logger.Info("Loaded %s", path.AsClientPath())
 
 			return nil
 		})