Fix: Limit query size for affected products update

The update of the affected products is now limited to 1000
CVEs per SQL query. This number can be changed with the
--affected-products-query-size option.
Also, an index has been added to make the queries a little faster.

This is meant to prevent the WAL from becoming very large during the
affected products update.

Author: timopollmeier

doc/gvmd.8

@@ -13,6 +13,12 @@ It manages the storage of any vulnerability management configurations and of the
 \fB-h, --help\f1
 Show help options.
 .TP
+\fB--affected-products-query-size=\fINUMBER\fB\f1
+Sets the number of CVEs to process per query when updating the affected products. Defaults to 1000. 
+.TP
+\fB--auth-timeout=\fITIMEOUT\fB\f1
+Sets the authentication timeout time for the cached authentication. Defaults to 15 minutes. 
+.TP
 \fB--broker-address=\fIADDRESS\fB\f1
 Sets the address for the publish-subscribe message (MQTT) broker. Defaults to localhost:9138. Set to empty to disable. 
 .TP

doc/gvmd.8.xml

@@ -52,6 +52,24 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
         <p>Show help options.</p>
       </optdesc>
     </option>
+    <option>
+      <p><opt>--affected-products-query-size=<arg>NUMBER</arg></opt></p>
+      <optdesc>
+        <p>
+          Sets the number of CVEs to process per query when updating
+          the affected products. Defaults to 1000.
+        </p>
+      </optdesc>
+    </option>
+    <option>
+      <p><opt>--auth-timeout=<arg>TIMEOUT</arg></opt></p>
+      <optdesc>
+        <p>
+          Sets the authentication timeout time for the cached authentication.
+          Defaults to 15 minutes.
+        </p>
+      </optdesc>
+    </option>
     <option>
       <p><opt>--broker-address=<arg>ADDRESS</arg></opt></p>
       <optdesc>

doc/gvmd.html

@@ -37,6 +37,20 @@ <h2>Options</h2>
 
 
 
+      <p><b>--affected-products-query-size=<em>NUMBER</em></b></p>
+      
+        <p>Sets the number of CVEs to process per query when updating
+          the affected products. Defaults to 1000.</p>
+      
+    
+    
+      <p><b>--auth-timeout=<em>TIMEOUT</em></b></p>
+      
+        <p>Sets the authentication timeout time for the cached authentication.
+          Defaults to 15 minutes.</p>
+      
+    
+    
       <p><b>--broker-address=<em>ADDRESS</em></b></p>
 
         <p>Sets the address for the publish-subscribe message (MQTT) broker.

src/gvmd.c

@@ -1865,6 +1865,8 @@ gvmd (int argc, char** argv, char *env[])
   static gchar *scanner_key_priv = NULL;
   static int scanner_connection_retry = SCANNER_CONNECTION_RETRY_DEFAULT;
   static int schedule_timeout = SCHEDULE_TIMEOUT_DEFAULT;
+  static int affected_products_query_size
+    = AFFECTED_PRODUCTS_QUERY_SIZE_DEFAULT;
   static int secinfo_commit_size = SECINFO_COMMIT_SIZE_DEFAULT;
   static gchar *delete_scanner = NULL;
   static gchar *verify_scanner = NULL;
@@ -1911,6 +1913,12 @@ gvmd (int argc, char** argv, char *env[])
   GOptionContext *option_context;
   static GOptionEntry option_entries[]
     = {
+        { "affected-products-query-size", '\0', 0, G_OPTION_ARG_INT,
+          &affected_products_query_size,
+          "Sets the number of CVEs to process per query when updating"
+          " the affected products. Defaults to "
+          G_STRINGIFY (AFFECTED_PRODUCTS_QUERY_SIZE_DEFAULT) ".",
+          "<number>" },
         { "auth-timeout", '\0', 0, G_OPTION_ARG_INT,
           &auth_timeout,
           "Sets the authentication timeout time for the cached authentication."
@@ -2364,6 +2372,8 @@ gvmd (int argc, char** argv, char *env[])
 
   /* Set SQL sizes */
 
+  set_affected_products_query_size (affected_products_query_size);
+
   set_secinfo_commit_size (secinfo_commit_size);
 
   set_vt_ref_insert_size (vt_ref_insert_size);

src/manage_pg.c

@@ -3781,6 +3781,8 @@ manage_db_init_indexes (const gchar *name)
            " ON scap2.cpes (severity);");
       sql ("CREATE INDEX cpes_by_uuid"
            " ON scap2.cpes (uuid);");
+      sql ("CREATE INDEX cpes_by_cpe_name_id"
+           " ON scap2.cpes(cpe_name_id);");
 
       sql ("CREATE INDEX afp_cpe_idx"
            " ON scap2.affected_products (cpe);");

src/manage_sql_secinfo.c

@@ -71,6 +71,11 @@
  */
 #define CPE_MAX_CHUNK_SIZE 10000
 
+/**
+ * @brief Query size for affected products updates.
+ */
+static int affected_products_query_size = AFFECTED_PRODUCTS_QUERY_SIZE_DEFAULT;
+
 /**
  * @brief Commit size for updates.
  */
@@ -4055,6 +4060,24 @@ update_scap_cves ()
   return 0;
 }
 
+static void
+exec_affected_products_sql (const char *cve_ids_str)
+{
+  sql ("INSERT INTO scap2.affected_products"
+        "  SELECT DISTINCT scap2.cpe_match_nodes.cve_id, scap2.cpes.id"
+        "    FROM scap2.cpe_match_nodes, scap2.cpe_nodes_match_criteria,"
+        "         scap2.cpe_matches, scap2.cpes"
+        "    WHERE scap2.cpe_match_nodes.cve_id IN (%s)"
+        "      AND scap2.cpe_match_nodes.id ="
+        "            scap2.cpe_nodes_match_criteria.node_id"
+        "      AND scap2.cpe_nodes_match_criteria.vulnerable = 1"
+        "      AND scap2.cpe_nodes_match_criteria.match_criteria_id ="
+        "            scap2.cpe_matches.match_criteria_id"
+        "      AND scap2.cpe_matches.cpe_name_id = scap2.cpes.cpe_name_id"
+        "  ON CONFLICT DO NOTHING;",
+        cve_ids_str);
+}
+
 /**
  * @brief Update SCAP affected products.
  *
@@ -4063,17 +4086,40 @@ update_scap_cves ()
 static void
 update_scap_affected_products ()
 {
+  iterator_t cves_iter;
+  GString *cve_ids_buffer;
   g_info ("Updating affected products");
 
-  sql ("INSERT INTO scap2.affected_products"
-       "  SELECT DISTINCT scap2.cpe_match_nodes.cve_id, scap2.cpes.id"
-       "    FROM scap2.cpe_match_nodes, scap2.cpe_nodes_match_criteria,"
-       "         scap2.cpe_matches, scap2.cpes"
-       "    WHERE scap2.cpe_match_nodes.id = scap2.cpe_nodes_match_criteria.node_id"
-       "      AND scap2.cpe_nodes_match_criteria.vulnerable = 1"
-       "      AND scap2.cpe_nodes_match_criteria.match_criteria_id ="
-       "            scap2.cpe_matches.match_criteria_id"
-       "      AND scap2.cpe_matches.cpe_name_id = scap2.cpes.cpe_name_id;");
+  init_iterator(&cves_iter,
+                "SELECT DISTINCT cve_id FROM scap2.cpe_match_nodes");
+
+  int count = 0;
+
+  cve_ids_buffer = g_string_new("");
+  while (next (&cves_iter))
+    {
+      resource_t cve_id;
+      cve_id = iterator_int64 (&cves_iter, 0);
+      g_string_append_printf (cve_ids_buffer, "%s%llu",
+                              cve_ids_buffer->len ? ", " : "",
+                              cve_id);
+      count ++;
+
+      if (count % affected_products_query_size == 0)
+        {
+          exec_affected_products_sql (cve_ids_buffer->str);
+          g_string_truncate (cve_ids_buffer, 0);
+          g_message("%s: Products of %d CVEs processed", __func__, count);
+        }
+    }
+
+  if (cve_ids_buffer->len)
+    {
+      exec_affected_products_sql (cve_ids_buffer->str);
+      g_string_truncate (cve_ids_buffer, 0);
+      g_message("%s: Products of %d CVEs processed", __func__, count);
+    }
+
 }
 
 /**
@@ -5785,6 +5831,20 @@ manage_rebuild_scap (GSList *log_config, const db_conn_info_t *database)
   return -1;
 }
 
+/**
+ * @brief Set the affected products query size.
+ *
+ * @param new_size The new affected products query size.
+ */
+void
+set_affected_products_query_size (int new_size)
+{
+  if (new_size <= 0)
+    affected_products_query_size = AFFECTED_PRODUCTS_QUERY_SIZE_DEFAULT;
+  else
+    secinfo_commit_size = new_size;
+}
+
 /**
  * @brief Set the SecInfo update commit size.
  *

src/manage_sql_secinfo.h

@@ -165,6 +165,11 @@
    { NULL, NULL, KEYWORD_TYPE_UNKNOWN }                          \
  }
 
+/**
+ * @brief Default for affected_products_query_size.
+ */
+#define AFFECTED_PRODUCTS_QUERY_SIZE_DEFAULT 1000
+
 /**
  * @brief Default for secinfo_commit_size.
  */
@@ -191,6 +196,9 @@ check_cert_db_version ();
 int
 get_secinfo_commit_size ();
 
+void
+set_affected_products_query_size (int);
+
 void
 set_secinfo_commit_size (int);