Added more Linux artifacts (Velocidex#209)

* ssh login watcher.
* process execution monitor.
* split_lines can now handle compression.

artifacts/definitions/Linux/Events/ProcessExecutions.yaml

@@ -0,0 +1,49 @@
+name: Linux.Events.ProcessExecutions
+description: |
+  This artifact collects process execution logs from the Linux kernel.
+
+  This artifact relied on the presence of `auditctl` usually included
+  in the auditd package. On Ubuntu you can install it using:
+
+  ```
+  apt-get install auditd
+  ```
+
+precondition: SELECT OS From info() where OS = 'linux'
+
+type: CLIENT_EVENT
+
+parameters:
+  - name: pathToAuditctl
+    default: /usr/sbin/auditctl
+    description: We depend on auditctl to install the correct process execution rules.
+
+sources:
+  - queries:
+     # Install the auditd rule if possible.
+     - LET _ = SELECT * FROM execve(argv=[pathToAuditctl, "-a",
+          "exit,always", "-F", "arch=b64", "-S", "execve", "-k", "procmon"])
+
+     - LET exec_log = SELECT timestamp(string=Timestamp) AS Time, Sequence,
+           atoi(string=Process.PID) AS Pid,
+           atoi(string=Process.PPID) AS Ppid,
+           Process.PPID AS PPID,
+           atoi(string=Summary.Actor.Primary) AS UserId,
+           Process.Title AS CmdLine,
+           Process.Exe AS Exe,
+           Process.CWD AS CWD
+       FROM audit()
+       WHERE "procmon" in Tags AND Result = 'success'
+
+     # Cache Uid -> Username mapping.
+     - LET users <= SELECT User, atoi(string=Uid) AS Uid
+       FROM Artifact.Linux.Sys.Users()
+
+     # Enrich the original artifact with more data.
+     - SELECT Time, Pid, Ppid, UserId,
+              { SELECT User from users WHERE Uid = UserId} AS User,
+              regex_replace(source=read_file(filename= "/proc/" + PPID + "/cmdline"),
+                            replace=" ", re="[\\0]") AS Parent,
+              CmdLine,
+              Exe, CWD
+       FROM exec_log

artifacts/definitions/Linux/Events/SSHLogin.yaml

@@ -0,0 +1,31 @@
+name: Linux.Events.SSHLogin
+description: |
+  This monitoring artifact watches the auth.log file for new
+  successful SSH login events and relays them back to the server.
+
+reference:
+  - https://www.elastic.co/blog/grokking-the-linux-authorization-logs
+
+type: CLIENT_EVENT
+
+parameters:
+  - name: syslogAuthLogPath
+    default: /var/log/auth.log
+
+  - name: SSHGrok
+    description: A Grok expression for parsing SSH auth lines.
+    default: >-
+      %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: %{DATA:event} %{DATA:method} for (invalid user )?%{DATA:user} from %{IPORHOST:ip} port %{NUMBER:port} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?
+
+sources:
+  - queries:
+      # Basic syslog parsing via GROK expressions.
+      - LET success_login = SELECT grok(grok=SSHGrok, data=Line) AS Event, Line
+        FROM watch_syslog(filename=syslogAuthLogPath)
+        WHERE Event.program = "sshd" AND Event.event = "Accepted"
+      - SELECT timestamp(string=Event.timestamp) AS Time,
+              Event.user AS User,
+              Event.method AS Method,
+              Event.IP AS SourceIP,
+              Event.pid AS Pid
+        FROM success_login

artifacts/definitions/Linux/Search/FileFinder.yaml

@@ -0,0 +1,115 @@
+name: Linux.Search.FileFinder
+description: |
+  Find files on the filesystem using the filename or content.
+
+
+  ## Performance Note
+
+  This artifact can be quite expensive, especially if we search file
+  content. It will require opening each file and reading its entire
+  content. To minimize the impact on the endpoint we recommend this
+  artifact is collected with a rate limited way (about 20-50 ops per
+  second).
+
+  This artifact is useful in the following scenarios:
+
+    * We need to locate all the places on our network where customer
+      data has been copied.
+
+    * We’ve identified malware in a data breach, named using short
+      random strings in specific folders and need to search for other
+      instances across the network.
+
+    * We believe our user account credentials have been dumped and
+      need to locate them.
+
+    * We need to search for exposed credit card data to satisfy PCI
+      requirements.
+
+    * We have a sample of data that has been disclosed and need to
+      locate other similar files
+
+
+precondition:
+  SELECT * FROM info() where OS = 'linux'
+
+parameters:
+  - name: SearchFilesGlob
+    default: /home/*/**
+    description: Use a glob to define the files that will be searched.
+
+  - name: Keywords
+    default:
+    description: A comma delimited list of strings to search for.
+
+  - name: Upload_File
+    default: N
+    type: bool
+
+  - name: Calculate_Hash
+    default: N
+    type: bool
+
+  - name: MoreRecentThan
+    default: ""
+    type: timestamp
+
+  - name: ModifiedBefore
+    default: ""
+    type: timestamp
+
+
+sources:
+  - queries:
+    - LET file_search = SELECT FullPath,
+               Sys.mft as Inode,
+               Mode.String AS Mode, Size,
+               Mtime.Sec AS Modified,
+               timestamp(epoch=Atime.Sec) AS ATime,
+               timestamp(epoch=Mtime.Sec) AS MTime,
+               timestamp(epoch=Ctime.Sec) AS CTime, IsDir
+        FROM glob(globs=SearchFilesGlob,
+                  accessor="file")
+
+    - LET more_recent = SELECT * FROM if(
+        condition=MoreRecentThan,
+        then={
+          SELECT * FROM file_search
+          WHERE Modified > parse_float(string=MoreRecentThan)
+        }, else=file_search)
+
+    - LET modified_before = SELECT * FROM if(
+        condition=ModifiedBefore,
+        then={
+          SELECT * FROM more_recent
+          WHERE Modified < parse_float(string=ModifiedBefore)
+        }, else=more_recent)
+
+    - LET keyword_search = SELECT * FROM if(
+        condition=Keywords,
+        then={
+          SELECT * FROM foreach(
+            row={
+               SELECT * FROM modified_before
+            },
+            query={
+               SELECT FullPath, Inode, Mode,
+                      Size, Modified, ATime, MTime, CTime,
+                      str(str=String.Data) As Keywords
+
+               FROM yara(files=FullPath,
+                         key=Keywords,
+                         rules="wide nocase ascii:"+Keywords,
+                         accessor="file")
+            })
+        }, else=modified_before)
+
+    - SELECT FullPath, Inode, Mode, Size, Modified, ATime,
+             MTime, CTime, Keywords,
+               if(condition=(Upload_File = "Y" and NOT IsDir ),
+                  then=upload(file=FullPath,
+                              accessor="file")) AS Upload,
+               if(condition=(Calculate_Hash = "Y" and NOT IsDir ),
+                  then=hash(path=FullPath,
+                            accessor="file")) AS Hash
+      FROM keyword_search

artifacts/definitions/Linux/Ssh/LoginLogs.yaml

@@ -0,0 +1,35 @@
+name: Linux.Syslog.SSHLogin
+description: |
+  Parses the auth logs to determine all SSH login attempts.
+
+reference:
+  - https://www.elastic.co/blog/grokking-the-linux-authorization-logs
+
+type: CLIENT
+
+parameters:
+  - name: syslogAuthLogPath
+    default: /var/log/auth.log*
+
+  - name: SSHGrok
+    description: A Grok expression for parsing SSH auth lines.
+    default: >-
+      %{SYSLOGTIMESTAMP:Timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: %{DATA:event} %{DATA:method} for (invalid user )?%{DATA:user} from %{IPORHOST:ip} port %{NUMBER:port} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?
+
+sources:
+  - queries:
+      # Basic syslog parsing via GROK expressions.
+      - SELECT timestamp(string=Event.Timestamp) AS Time,
+               Event.IP AS IP,
+               Event.event AS Result,
+               Event.method AS Method,
+               Event.user AS AttemptedUser,
+               FullPath
+        FROM foreach(
+          row={
+              SELECT FullPath FROM glob(globs=syslogAuthLogPath)
+          }, query={
+              SELECT grok(grok=SSHGrok, data=Line) AS Event, FullPath
+              FROM parse_lines(filename=FullPath)
+              WHERE Event.program = "sshd"
+          })

artifacts/definitions/Linux/Ssh/PrivateKeys.yaml

@@ -0,0 +1,26 @@
+name: Linux.Ssh.PrivateKeys
+description: |
+  SSH Private keys can be either encrypted or unencrypted. Unencrypted
+  private keys are more risky because an attacker can use them without
+  needing to unlock them with a password.
+
+  This artifact searches for private keys in the usual locations and
+  also records if they are encrypted or not.
+
+  ## references
+  - https://attack.mitre.org/techniques/T1145/
+
+precondition: SELECT OS From info() where OS = 'linux'
+
+parameters:
+  - name: KeyGlobs
+    default: /home/*/.ssh/id_{rsa,dsa}
+
+sources:
+  - queries:
+      - SELECT FullPath,
+               timestamp(epoch=Mtime.Sec) AS Mtime,
+               if(condition={
+                     SELECT * from yara(rules="wide ascii:ENCRYPTED", files=FullPath)
+                  }, then="Yes", else="No") AS Encrypted
+        FROM glob(globs=KeyGlobs)

artifacts/definitions/Linux/Sys/Crontab.yaml

@@ -4,12 +4,15 @@ description: |
 parameters:
   - name: cronTabGlob
     default: /etc/crontab,/etc/cron.d/**,/var/at/tabs/**,/var/spool/cron/**,/var/spool/cron/crontabs/**
+  - name: cronTabScripts
+    default: /etc/cron.daily/*,/etc/cron.hourly/*,/etc/cron.monthly/*,/etc/cron.weekly/*
+
+precondition: SELECT OS From info() where OS = 'linux'
+
 sources:
-  - precondition: |
-      SELECT OS From info() where OS = 'linux'
+  - name: CronTabs
     queries:
-      - |
-        LET raw = SELECT * FROM foreach(
+      - LET raw = SELECT * FROM foreach(
           row={
             SELECT FullPath from glob(globs=split(string=cronTabGlob, sep=","))
           },
@@ -26,6 +29,7 @@ sources:
                  "(?P<DayOfMonth>[^\\s]+)\\s+"+
                  "(?P<Month>[^\\s]+)\\s+"+
                  "(?P<DayOfWeek>[^\\s]+)\\s+"+
+                 "(?P<User>[^\\s]+)\\s+"+
                  "(?P<Command>.+)$"]) as Record
 
             /* Read lines from the file and filter ones that start with "#" */
@@ -34,8 +38,8 @@ sources:
                regex="\n", columns=["data"]) WHERE not data =~ "^\\s*#"
             }) WHERE Record.Command
 
-      - |
-        SELECT Record.Event AS Event,
+      - SELECT Record.Event AS Event,
+               Record.User AS User,
                Record.Minute AS Minute,
                Record.Hour AS Hour,
                Record.DayOfMonth AS DayOfMonth,
@@ -44,3 +48,8 @@ sources:
                Record.Command AS Command,
                FullPath AS Path
         FROM raw
+
+  - name: Uploaded
+    queries:
+      - SELECT FullPath, upload(filename=FullPath) AS Upload
+        FROM glob(globs=split(string=cronTabGlob + "," + cronTabScripts, sep=","))

artifacts/definitions/Linux/Sys/Maps.yaml

@@ -0,0 +1,41 @@
+name: Linux.Sys.Maps
+description: |
+  A running binary may link other binaries into its address
+  space. These shared objects contain exported functions which may be
+  used by the binary.
+
+  This artifact parses the /proc/<pid>/maps to emit all mapped files
+  into the process.
+
+precondition: SELECT OS From info() where OS = 'linux'
+
+parameters:
+  - name: processRegex
+    description: A regex applied to process names.
+    default: .
+
+sources:
+  - queries:
+      - LET processes = SELECT Pid, Name, Username
+        FROM pslist()
+        WHERE Name =~ processRegex
+      - SELECT Pid, Name, Username,
+               "0x" + Record.Start AS StartHex,
+               "0x" + Record.End AS EndHex,
+               Record.Perm AS Perm,
+               atoi(string="0x" + Record.Size) AS Size,
+               "0x" + Record.Size AS SizeHex,
+               Record.Filename AS Filename,
+               if(condition=Record.Deleted, then=TRUE, else=FALSE) AS Deleted
+        FROM foreach(
+          row=processes,
+          query={
+            SELECT parse_string_with_regex(
+                    string=Line,
+                    regex="(?P<Start>^[^-]+)-(?P<End>[^\\s]+)\\s+(?P<Perm>[^\\s]+)\\s+(?P<Size>[^\\s]+)\\s+[^\\s]+\\s+(?P<PermInt>[^\\s]+)\\s+(?P<Filename>.+?)(?P<Deleted> \\(deleted\\))?$") AS Record,
+                  Pid, Name, Username
+            FROM parse_lines(
+               filename=format(format="/proc/%d/maps", args=[Pid]),
+               accessor='file'
+            )
+          })

artifacts/definitions/Linux/Sys/SUID.yaml

@@ -0,0 +1,38 @@
+name: Linux.Sys.SUID
+description: |
+  When the setuid or setgid bits are set on Linux or macOS for an
+  application, this means that the application will run with the
+  privileges of the owning user or group respectively [1]. Normally an
+  application is run in the current user’s context, regardless of
+  which user or group owns the application. There are instances where
+  programs need to be executed in an elevated context to function
+  properly, but the user running them doesn’t need the elevated
+  privileges. Instead of creating an entry in the sudoers file, which
+  must be done by root, any user can specify the setuid or setgid flag
+  to be set for their own applications. These bits are indicated with
+  an "s" instead of an "x" when viewing a file's attributes via ls
+  -l. The chmod program can set these bits with via bitmasking, chmod
+  4777 [file] or via shorthand naming, chmod u+s [file].
+
+  An adversary can take advantage of this to either do a shell escape
+  or exploit a vulnerability in an application with the setsuid or
+  setgid bits to get code running in a different user’s
+  context. Additionally, adversaries can use this mechanism on their
+  own malware to make sure they're able to execute in elevated
+  contexts in the future [2].
+
+  ## References:
+  - https://attack.mitre.org/techniques/T1166/
+
+parameters:
+  - name: GlobExpression
+    default: /usr/**
+
+sources:
+  - queries:
+      - SELECT Mode.String AS Mode,
+               FullPath, Size,
+               timestamp(epoch=Mtime.Sec) AS Mtime,
+               Sys.Uid AS OwnerID,
+               Sys.Gid AS GroupID
+        FROM glob(globs=GlobExpression) WHERE Mode =~ '^u'