Fix AptSources deb822 parsing bug and add deb822 test (#2926)

#2851 was merged a bit too early. My mistake, I should have marked it as a draft. A misplaced `WHERE URIs` prevented deb822 parsing to work altogether. This is now fixed, and I've included some sources files to the tests, along with another lists file that includes two common options. I've added these new files to the tests Sources and SourcesFlattened, but not SourcesCacheFiles. I've also fixed the VQL suggestions after realising that `get()` doesn't work in WHERE as I assumed it would (see discussion [here](https://discord.com/channels/624244632552734750/865024982399320114/1147470384359284746) with a detailed explanation from Mike).
Velocidex · Sep 8, 2023 · 3707c2f · 3707c2f
1 parent 39ce693
commit 3707c2f
Show file tree

Hide file tree

Showing 6 changed files with 285 additions and 25 deletions.
diff --git a/artifacts/definitions/Linux/Debian/AptSources.yaml b/artifacts/definitions/Linux/Debian/AptSources.yaml
@@ -18,7 +18,7 @@ description: |
   to find source code for packages.
 
   `*.sources` files (deb822-style format) are in the form of key–value
-  lines, and as opposed to the one–line format, they can contain
+  lines, and as opposed to the one–line format, they may contain
   multiple URIs, components and types (deb/deb-src), along with
   embedded GPG keys. Example:
 
@@ -29,8 +29,9 @@ description: |
   Components: main contrib non-free
   ```
 
-  The exported function parse_aptsources(OSPath, flatten) parses
-  both formats and returns a (optionally flattened) table with
+  The exported function `parse_aptsources(OSPath, flatten)` parses
+  both formats and returns an (optionally flattened) table with
+
    - OSPath
    - Types (deb/deb-src)
    - Components (e.g. main/contrib/non-free/restricted,universe)
@@ -39,30 +40,36 @@ description: |
    - _Transport (e.g. http/https/file/cdrom/ftp)
    - URIs (e.g. http://us.archive.ubuntu.com/ubuntu/)
 
-  Any option is added to an individual column. Typical options are
+  Any option is added to an individual column. The most common options
+  are
+
    - Architectures (e.g. amd64/i386/armel)
    - Signed-By (e.g. /usr/share/keyrings/osquery.gpg)
 
   All known option names are transformed to the plural PascalCase
   variants as listed in the sources.list man page. Any undocumented
   options will still be included in the results, with names unchanged.
   Options in the one-line format of the form "lang+=de"/"arch-=i386"
-  will be in columns like "Languages-Add"/"Architectures-Remove", matching
-  the option names having the same effect in deb822.
+  will be put in columns like "Languages-Add"/"Architectures-Remove",
+  matching the option names having the same effect in deb822.
 
   Entries in deb822 sources files may be disabled by including
   "Enabled: no" instead of commenting out all lines. If this field
   is not present with a falsly value, the entry is enabled. Use the
   exported functions DebTrue()/DebFalse() to correctly parse all
-  accepted true/false strings, or use the VQL suggestion "Enabled"
-  to filter on this column (true), if present.
+  accepted true/false strings, or use the VQL suggestion "Only enabled
+  sources" to filter on this column (true), if present.
 
   If the GPG key is embedded in a .sources file, the whole GPG key
   will be included in the cell. Otherwise the value will be a file
-  path.
+  path. Use the VQL suggestion "Hide embedded GPG keys" to replace
+  embedded GPG keys with "(embedded)" in the results. In order to
+  inspect the keys themselves (files or embedded data), use the
+  exchange artifact Linux.Debian.GPGKeys.
 
-  If flatten is False, multi–value fields (like Components) will
-  be combined in a single-space-separated string in each row.
+  If the function parameter "flatten" is False, multi–value fields
+  (like Components) will be combined in a single space-separated
+  string in each row.
 
   In addition to the two apt sources tables, a third table correlates
   information from InRelease and Release files to provide additional
@@ -303,10 +310,6 @@ export: |
             columns='Section',
             regex='^ #', record_regex='''\n{2,}'''
         )
-        /* Sections may be empty due to several newlines or comments on their own
-           separated by newlines. Ensure that at least one field is present
-           (URIs are mandatory): */
-        WHERE URIs
 
         LET Deb822_Flattened_(OSPath) = SELECT * FROM foreach(
             row=Deb822Sections(OSPath=OSPath),
@@ -317,6 +320,9 @@ export: |
                 )
             })}
         )
+        /* DEB822_Sections() may produce empty rows. Exclude these by filtering
+           for a required column, like URIs: */
+        WHERE URIs
 
         /* Parse a deb822 sources file with options in individual columns.
            Note that, as opposed to DebOneLine and Deb822_Flattened, this
@@ -329,6 +335,7 @@ export: |
                 column='Contents'
             )}
         )
+        WHERE URIs
 
         /* Parse a deb822 sources file with options in individual columns, flattened: */
         LET Deb822_Flattened(OSPath) = SELECT * FROM flatten(query={
@@ -384,14 +391,34 @@ sources:
       - type: vql_suggestion
         name: Only enabled sources
         template: |
-            SELECT * FROM source(artifact='Custom.Linux.Debian.AptSources/Sources')
-            WHERE get(field='Enabled', default='yes') =~ '(?i)^(?:yes|true|with|on|enable)$'
+            /*
+            # Sources (enabled only)
+            */
+            SELECT * FROM source()
+            WHERE Enabled =~ '(?i)^(?:yes|true|with|on|enable)$' || true
 
       - type: vql_suggestion
         name: Trusted sources (apt-secure bypassed)
         template: |
-            SELECT * FROM source(artifact='Custom.Linux.Debian.AptSources/Sources')
-            WHERE get(field='Trusted', default='') =~ '(?i)^(?:yes|true|with|on|enable)$'
+            /*
+            # "Trusted" sources (apt-secure bypassed)
+
+            When the Trusted option is true, apt does not verify the GPG
+            signature of the Release files of the repository, and it also
+            doe not warn about this.
+            */
+            SELECT * FROM source()
+            WHERE Trusted =~ '(?i)^(?:yes|true|with|on|enable)$' || false
+
+      - type: vql_suggestion
+        name: Hide embedded GPG keys
+        template: |
+            /*
+            # Sources (embedded GPG keys hidden)
+            */
+            SELECT *, if(condition=get(field='Signed-By')=~'BEGIN PGP PUBLIC KEY',
+                then='(embedded)', else=get(field='Signed-By')) AS `Signed-By`
+                FROM source()
 
   - name: SourcesFlattened
     query: |

diff --git a/artifacts/testdata/files/debian/sources.list/misc.sources b/artifacts/testdata/files/debian/sources.list/misc.sources
@@ -0,0 +1,50 @@
+X-Repolib-Name: MongoDb
+Enabled: yes
+Types: deb deb-src
+URIs: https://repo.mongodb.org/apt/ubuntu
+Suites: focal/mongodb-org/6.0
+ jammy/mongodb-org/testing
+Components: multiverse
+architectures: amd64,arm64
+signed-by:
+ -----BEGIN PGP PUBLIC KEY BLOCK-----
+  Comment: Use "gpg --dearmor" for unpacking
+# Comment
+ Version: GnuPG v1
+ .
+ mQINBGPILWABEACqeWP/ktugdlWEyk7YTXo3n19+5Om4AlSdIyKv49vAlKtzCfMA
+ QkZq3mfvjXiKMuLnL2VeElAJQIYcPoqnHf6tJbdrNv4AX2uI1cTsvGW7YS/2WNwJ
+ C/+vBa4o+yA2CG/MVWZRbtOjkFF/W07yRFtNHAcgdmpIjdWgSnPQr9eIqLuWXIhy
+ H7EerKsba227Vd/HfvKnAy30Unlsdywy7wi1FupzGJck0TPoOVGmsSpSyIQu9A4Z
+ uC6TE/NcJHvaN0JuHwM+bQo9oWirGsZ1NCoVqSY8/sasdUc7T9r90MbUcH674YAR
+ 8OKYVBzU0wch4VTFhfHZecKHQnZf+V4dmP9oXnu4fY0/0w3l4jaew7Ind7kPg3yN
+ hvgAkBK8yRAbSu1NOtHDNiRoHGEQFgct6trVOvCqHbN/VToLNtGk0rhKGOp8kuSF
+ OJ02PJPxF3/zHGP8n8khCjUJcrilYPqRghZC8ZWnCj6GJVg6WjwLi+hPwNMi8xK6
+ cjKhRW3eCy5Wcn73PzVBX9f7fSeFDJec+IfS47eNkxunHAOUMXa2+D+1xSWgEfK0
+ PClfyWPgLIXY2pGQ6v8l3A6P5gJv4o38/E1h1RTcO3H1Z6cgZLIORZHPyAj50SPQ
+ cjzftEcz56Pl/Cyw3eMYC3qlbABBgsdeb6KB6G5dkNxI4or3MgmxcwfnkwARAQAB
+ tDdNb25nb0RCIDcuMCBSZWxlYXNlIFNpZ25pbmcgS2V5IDxwYWNrYWdpbmdAbW9u
+ Z29kYi5jb20+iQI+BBMBAgAoBQJjyC1gAhsDBQkJZgGABgsJCAcDAgYVCAIJCgsE
+ FgIDAQIeAQIXgAAKCRAWDSa7F4W6OM+eD/sE7KbJyRNWyPCRTqqJXrXvyPqZtbFX
+ 8sio0lQ8ghn4f7lmb7LnFroUsmBeWaYirM8O3b2+iQ9oj4GeR3gbRZsEhFXQfL54
+ SfrmG9hrWWpJllgPP7Six+jrzcjvkf1TENqw4jRP+cJhuihH1Gfizo9ktwwoN9Yr
+ m7vgh+focEEmx8dysS38ApLxKlUEfTsE9bYsClgqyY1yrt3v4IpGbf66yfyBHNgY
+ sObR3sngDRVbap7PwNyREGsuAFfKr/Dr37HfrjY7nsn3vH7hbDpSBh+H7a0b/chS
+ mM60aaG4biWpvmSC7uxA/t0gz+NQuC4HL+qyNPUxvyIO+TwlaXfCI6ixazyrH+1t
+ F7Bj5mVsne7oeWjRrSz85jK3Tpn9tj3Fa7PCDA6auAlPK8Upbhuoajev4lIydNd2
+ 70yO0idm/FtpX5a8Ck7KSHDvEnXpN70imayoB4Fs2Kigi2BdZOOdib16o5F/9cx9
+ piNa7HotHCLTfR6xRmelGEPWKspU1Sm7u2A5vWgjfSab99hiNQ89n+I7BcK1M3R1
+ w/ckl6qBtcxz4Py+7jYIJL8BYz2tdreWbdzWzjv+XQ8ZgOaMxhL9gtlfyYqeGfnp
+ hYW8LV7a9pavxV2tLuVjMM+05ut/d38IkTV7OSJgisbSGcmycXIzxsipyXJVGMZt
+ MFw3quqJhQMRsA==
+ =gbRM
+ -----END PGP PUBLIC KEY BLOCK-----
+
+# Comment
+Types: deb
+URIs: https://dl.winehq.org/wine-builds/debian
+Suites: bookworm
+Components: main
+Architectures: amd64 i386
+Signed-By:
+ /etc/apt/keyrings/winehq-archive.key
diff --git a/artifacts/testdata/files/debian/sources.list/osquery.list b/artifacts/testdata/files/debian/sources.list/osquery.list
@@ -0,0 +1 @@
+deb [arch=amd64 signed-by=/usr/share/keyrings/osquery.gpg] https://pkg.osquery.io/deb deb main
diff --git a/artifacts/testdata/files/debian/sources.list → ...ta/files/debian/sources.list/sources.list b/artifacts/testdata/files/debian/sources.list → ...ta/files/debian/sources.list/sources.list
diff --git a/artifacts/testdata/server/testcases/debian.in.yaml b/artifacts/testdata/server/testcases/debian.in.yaml
@@ -2,14 +2,14 @@ Queries:
  - SELECT OSPath.Basename AS OSPath, *
    FROM Artifact.Linux.Debian.AptSources(
       source="Sources",
-      linuxAptSourcesGlobs=[dict(ListGlobs=srcDir + "/artifacts/testdata/files/debian/sources.list")],
+      linuxAptSourcesGlobs=[dict(ListGlobs=srcDir + "/artifacts/testdata/files/debian/sources.list/*")],
       aptCacheDirectory=srcDir + "/artifacts/testdata/files/debian/"
    )
 
  - SELECT OSPath.Basename AS OSPath, *
    FROM Artifact.Linux.Debian.AptSources(
       source="SourcesFlattened",
-      linuxAptSourcesGlobs=[dict(ListGlobs=srcDir + "/artifacts/testdata/files/debian/sources.list")],
+      linuxAptSourcesGlobs=[dict(ListGlobs=srcDir + "/artifacts/testdata/files/debian/sources.list/*")],
       aptCacheDirectory=srcDir + "/artifacts/testdata/files/debian/"
    )
 
@@ -21,7 +21,7 @@ Queries:
 
    FROM Artifact.Linux.Debian.AptSources(
       source="SourcesCacheFiles",
-      linuxAptSourcesGlobs=[dict(ListGlobs=srcDir + "/artifacts/testdata/files/debian/sources.list")],
+      linuxAptSourcesGlobs=[dict(ListGlobs=srcDir + "/artifacts/testdata/files/debian/sources.list/sources.list")],
       aptCacheDirectory=srcDir + "/artifacts/testdata/files/debian/"
    )
    WHERE Record