tenable_io: fix flakey test and timestamp handling

packages/tenable_io/_dev/deploy/docker/files/config.yml

@@ -27,7 +27,7 @@ rules:
                 {"id":13670,"name":"Fedora Core 1 : kernel-2.4.22-1.2140.nptl (2003-047)","attributes":{"vpr":{"score":5.5,"updated":"2018-07-19T00:00:00Z","drivers":{"age_of_vuln":{"lower_bound":366,"upper_bound":730},"threat_recency":{"lower_bound":366,"upper_bound":730},"exploit_code_maturity":"UNPROVEN","cvss_impact_score_predicted":false,"cvss3_impact_score":5.9,"threat_intensity_last28":"VERY_LOW","threat_sources_last28":["No recorded events"],"product_coverage":"LOW"}},"plugin_modification_date":"2018-07-19T00:00:00Z","plugin_version":"1.17","exploited_by_malware":false,"description":"Various RTC drivers had the potential to leak...","unsupported_by_vendor":false,"cvss_temporal_score":0,"patch_publication_date":"2004-01-07T00:00:00Z","see_also":["http://example.com/u?07bc9e7f"],"default_account":false,"exploit_available":false,"cve":["CVE-2003-0984"],"exploit_framework_canvas":false,"cvss_base_score":4.6,"solution":"Update the affected packages.","cvss_vector":{"raw":"AV:L/AC:L/Au:N/C:P/I:P/A:P","AccessVector":"Local-access","Availability-Impact":"Partial","Authentication":"None required","Integrity-Impact":"Partial","AccessComplexity":"Low","Confidentiality-Impact":"Partial"},"exploit_framework_exploithub":false,"cpe":["p-cpe:/a:fedoraproject:fedora:kernel-source","cpe:/o:fedoraproject:fedora_core:1","p-cpe:/a:fedoraproject:fedora:kernel-BOOT","p-cpe:/a:fedoraproject:fedora:kernel-debuginfo","p-cpe:/a:fedoraproject:fedora:kernel","p-cpe:/a:fedoraproject:fedora:kernel-doc","p-cpe:/a:fedoraproject:fedora:kernel-smp"],"plugin_publication_date":"2004-07-23T00:00:00Z","exploit_framework_core":false,"in_the_news":false,"has_patch":true,"xref":["FEDORA:2003-047"],"malware":false,"exploit_framework_d2_elliot":false,"xrefs":[{"type":"FEDORA","id":"2003-047"}],"risk_factor":"Medium","synopsis":"The remote Fedora Core host is missing a security update.","cvss3_temporal_score":0,"exploited_by_nessus":false,"cvss3_base_score":0,"exploit_framework_metasploit":false,"plugin_type":"local"}},
                 {"id":32,"name":"Siemens S7-300 Series PLC CPU Firmware <= 3.2.11 DoS","attributes":{"intel_type":"SENSOR","synopsis":"A Siemens S7-300 programmable logic controller (PLC) has been detected which is vulnerable to a Denial of Service (DoS) attack vector.","description":"Siemens S7-300 PLC central processing units (CPUs) contain an unspecified flaw that may allow a remote attacker to use a specially crafted packet to cause the device to enter defect mode until a cold restart is performed.","solution":"Upgrade the firmware to version 3.2.12 or later.","see_also":[],"plugin_publication_date":"2019-05-21T00:00:00Z","vuln_publication_date":"2016-06-08T00:00:00Z","patch_publication_date":"2016-06-08T00:00:00Z","has_patch":true,"exploit_available":false,"risk_factor":"HIGH","plugin_modification_date":"2019-09-30T00:00:00Z","always_run":false,"compliance":false,"cvss_vector":{"raw":"AV:N/AC:L/Au:N/C:N/I:N/A:C","AccessVector":"Network","AccessComplexity":"Low","Authentication":"None required","Confidentiality-Impact":"None","Integrity-Impact":"None","Availability-Impact":"Complete"},"cvss_temporal_vector":{"raw":"E:U/RL:OF/RC:C","Exploitability":"Unproven","RemediationLevel":"Official Fix","ReportConfidence":"Confirmed"},"cvss_temporal_score":5.8,"cvss3_vector":{"raw":"AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H","AttackVector":"Network","AttackComplexity":"Low","PrivilegesRequired":"None","UserInteraction":"None","Scope":"Unchanged","Confidentiality-Impact":"None","Integrity-Impact":"None","Availability-Impact":"High"},"cvss3_temporal_vector":{"raw":"E:U/RL:O/RC:C","ExploitCodeMaturity":"Unproven","RemediationLevel":"Official Fix","ReportConfidence":"Confirmed"},"cvss3_temporal_score":6.5,"cvss3_base_score":7.5,"cve":["CVE-2016-3949"],"bid":[91133,23432,234234,235],"xref":[],"xrefs":[],"vpr":{"score":3.6,"drivers":{"age_of_vuln":{"lower_bound":731},"exploit_code_maturity":"UNPROVEN","cvss_impact_score_predicted":false,"threat_intensity_last28":"VERY_LOW","threat_sources_last28":["No recorded events"],"product_coverage":"LOW"},"updated":"2021-02-07T05:31:54Z"}}}
               ]
-            },"size":1000,"params":{"page":1,"size":1000,"last_updated":""},"total_count":2}
+            },"size":2,"params":{"page":1,"size":1000,"last_updated":""},"total_count":2}
   - path: /vulns/export
     methods: ["POST"]
     responses:

packages/tenable_io/changelog.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "3.2.1"
+  changes:
+    - description: Fix pagination progression logic.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10940
+    - description: Fix timestamp logic.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10940
 - version: "3.2.0"
   changes:
     - description: Improve error reporting for API request failures.

packages/tenable_io/data_stream/plugin/agent/stream/cel.yml.hbs

@@ -38,43 +38,45 @@ program: |
     },
   }).do_request().as(resp,
     resp.StatusCode == 200 ?
-      bytes(resp.Body).decode_json().as(body, {
-        "events": has(body.?data.plugin_details) ? body.data.plugin_details.map(e, { "message": e.encode_json() }) : [{}],
-        "cursor": {
-          "last_update_time": (
-            has(body.?data.plugin_details) && body.data.plugin_details.size() > 0 ?
-              (
-                has(state.?cursor.last_update_time) && body.data.plugin_details.map(e, e.attributes.plugin_modification_date).max() < state.cursor.last_update_time ?
-                  state.cursor.last_update_time
-                :
-                  body.data.plugin_details.map(e, e.attributes.plugin_modification_date).max()
-              )
+      bytes(resp.Body).decode_json().as(body,
+        (int(state.?count.orValue(0))+int(body.size)).as(count,
+          (count < int(body.total_count)).as(want_more, {
+            "events": has(body.?data.plugin_details) ?
+              body.data.plugin_details.map(e, { "message": e.encode_json() })
             :
-              state.?cursor.last_update_time.orValue(string(timestamp(0)))
-          ),
-          "first_update_time": (
-            has(state.?cursor.first_update_time) && has(body.data.plugin_details) ?
-              (
-                body.data.plugin_details.size() > 0 ?
+              [],
+            "cursor": {
+              "last_update_time": (
+                has(body.?data.plugin_details) && body.data.plugin_details.size() > 0 ?
+                  ([?state.?cursor.last_update_time] + body.data.plugin_details.map(e,
+                    e.attributes.plugin_modification_date
+                  )).map(t,
+                    timestamp(t)
+                  ).max()
+                :
+                  state.?cursor.last_update_time.orValue(string(timestamp(0)))
+              ),
+              "first_update_time": (
+                has(state.?cursor.first_update_time) && has(body.data.plugin_details) ?
                   (
-                    body.size == int(state.batch_size) && state.want_more ?
+                    want_more ?
                       state.cursor.first_update_time
                     :
                       state.cursor.last_update_time
                   )
                 :
-                  state.cursor.last_update_time
-              )
-            :
-              string(timestamp(0))
-          ),
-        },
-        "next_page": body.size == int(state.batch_size) ? int(state.next_page)+1 : 1,
-        "want_more": body.size == int(state.batch_size),
-        "access_key": state.access_key,
-        "secret_key": state.secret_key,
-        "batch_size": state.batch_size,
-      })
+                  string(timestamp(0))
+              ),
+            },
+            "next_page": want_more ? int(state.next_page)+1 : 1,
+            "want_more": want_more,
+            "count": count,
+            "access_key": state.access_key,
+            "secret_key": state.secret_key,
+            "batch_size": state.batch_size,
+          })
+        )
+      )
     :
       {
         "events": {

packages/tenable_io/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml

@@ -21,6 +21,10 @@ processors:
   - json:
       field: event.original
       target_field: json
+  - fail:
+      tag: cel_failure
+      if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null
+      message: error message set and no data to process
   - drop:
       if: ctx.json?.data?.plugin_details != null && ctx.json.data.plugin_details.isEmpty()
   - fingerprint:

packages/tenable_io/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: tenable_io
 title: Tenable Vulnerability Management
-version: "3.2.0"
+version: "3.2.1"
 description: Collect logs from Tenable Vulnerability Management with Elastic Agent.
 type: integration
 categories: