Skip to content

Commit 5162a75

Browse files
timopollmeiertimopollmeier
committed
Change: Use COPY to load CPE match strings
The CPE match strings are now loaded into a new SCAP schema using COPY statements, which is faster than using INSERT.
1 parent a69f5e1 commit 5162a75

File tree

1 file changed

+192
-69
lines changed

1 file changed

+192
-69
lines changed

‎src/manage_sql_secinfo.c

+192-69
Original file line numberDiff line numberDiff line change
@@ -4420,15 +4420,20 @@ update_scap_affected_products ()
44204420
* @return 0 success, -1 error.
44214421
*/
44224422
static int
4423-
handle_json_cpe_match_string (inserts_t *inserts, inserts_t *matches_inserts,
4423+
handle_json_cpe_match_string (inserts_t *inserts,
4424+
inserts_t *matches_inserts,
4425+
db_copy_buffer_t *copy_buffer,
4426+
db_copy_buffer_t *matches_copy_buffer,
4427+
gboolean use_copy,
44244428
cJSON *match_string_item)
44254429
{
44264430
cJSON *match_string, *matches_array;
4427-
char *criteria, *match_criteria_id, *status, *ver_se;
4431+
char *criteria, *match_criteria_id, *status, *ver_str;
44284432
gchar *quoted_version_start_incl, *quoted_version_start_excl;
44294433
gchar *quoted_version_end_incl, *quoted_version_end_excl;
4430-
gchar *quoted_criteria, *quoted_match_criteria_id;
4434+
gchar *quoted_criteria, *quoted_match_criteria_id, *quoted_status;
44314435
int first;
4436+
gchar* (*quote_func)(const char*) = use_copy ? sql_copy_escape : sql_quote;
44324437

44334438
assert (inserts);
44344439
assert (matches_inserts);
@@ -4465,53 +4470,89 @@ handle_json_cpe_match_string (inserts_t *inserts, inserts_t *matches_inserts,
44654470
return -1;
44664471
}
44674472

4468-
ver_se = json_object_item_string (match_string, "versionStartIncluding");
4469-
if (ver_se == NULL)
4470-
quoted_version_start_incl = g_strdup ("NULL");
4473+
ver_str = json_object_item_string (match_string, "versionStartIncluding");
4474+
if (use_copy)
4475+
quoted_version_start_incl = ver_str
4476+
? sql_copy_escape (ver_str)
4477+
: g_strdup ("\\N");
44714478
else
4472-
quoted_version_start_incl = g_strdup_printf ("'%s'", ver_se);
4479+
quoted_version_start_incl = sql_insert (ver_str);
4480+
44734481

4474-
ver_se = json_object_item_string (match_string, "versionStartExcluding");
4475-
if (ver_se == NULL)
4476-
quoted_version_start_excl = g_strdup ("NULL");
4482+
ver_str = json_object_item_string (match_string, "versionStartExcluding");
4483+
if (use_copy)
4484+
quoted_version_start_excl = ver_str
4485+
? sql_copy_escape (ver_str)
4486+
: g_strdup ("\\N");
44774487
else
4478-
quoted_version_start_excl = g_strdup_printf ("'%s'", ver_se);
4488+
quoted_version_start_excl = sql_insert (ver_str);
44794489

4480-
ver_se = json_object_item_string (match_string, "versionEndIncluding");
4481-
if (ver_se == NULL)
4482-
quoted_version_end_incl = g_strdup ("NULL");
4490+
ver_str = json_object_item_string (match_string, "versionEndIncluding");
4491+
if (use_copy)
4492+
quoted_version_end_incl = ver_str
4493+
? sql_copy_escape (ver_str)
4494+
: g_strdup ("\\N");
44834495
else
4484-
quoted_version_end_incl = g_strdup_printf ("'%s'", ver_se);
4496+
quoted_version_end_incl = sql_insert (ver_str);
44854497

4486-
ver_se = json_object_item_string (match_string, "versionEndExcluding");
4487-
if (ver_se == NULL)
4488-
quoted_version_end_excl = g_strdup ("NULL");
4498+
ver_str = json_object_item_string (match_string, "versionEndExcluding");
4499+
if (use_copy)
4500+
quoted_version_end_excl = ver_str
4501+
? sql_copy_escape (ver_str)
4502+
: g_strdup ("\\N");
44894503
else
4490-
quoted_version_end_excl = g_strdup_printf ("'%s'", ver_se);
4504+
quoted_version_end_excl = sql_insert (ver_str);
44914505

4492-
quoted_match_criteria_id = sql_quote (match_criteria_id);
4493-
quoted_criteria = fs_to_uri_convert_and_quote_cpe_name (criteria, sql_quote);
4506+
quoted_match_criteria_id = quote_func (match_criteria_id);
4507+
quoted_criteria = fs_to_uri_convert_and_quote_cpe_name (criteria, quote_func);
4508+
quoted_status = quote_func (status);
44944509

4495-
first = inserts_check_size (inserts);
4510+
if (use_copy)
4511+
{
4512+
if (db_copy_buffer_append_printf (copy_buffer,
4513+
"%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
4514+
quoted_match_criteria_id,
4515+
quoted_criteria,
4516+
quoted_version_start_incl,
4517+
quoted_version_start_excl,
4518+
quoted_version_end_incl,
4519+
quoted_version_end_excl,
4520+
quoted_status))
4521+
{
4522+
g_free (quoted_match_criteria_id);
4523+
g_free (quoted_criteria);
4524+
g_free (quoted_version_start_incl);
4525+
g_free (quoted_version_start_excl);
4526+
g_free (quoted_version_end_incl);
4527+
g_free (quoted_version_end_excl);
4528+
g_free (quoted_status);
4529+
return -1;
4530+
}
4531+
}
4532+
else
4533+
{
4534+
first = inserts_check_size (inserts);
44964535

4497-
g_string_append_printf (inserts->statement,
4498-
"%s ('%s', '%s', %s, %s, %s, %s, '%s')",
4499-
first ? "" : ",",
4500-
quoted_match_criteria_id,
4501-
quoted_criteria,
4502-
quoted_version_start_incl,
4503-
quoted_version_start_excl,
4504-
quoted_version_end_incl,
4505-
quoted_version_end_excl,
4506-
status);
4536+
g_string_append_printf (inserts->statement,
4537+
"%s ('%s', '%s', %s, %s, %s, %s, '%s')",
4538+
first ? "" : ",",
4539+
quoted_match_criteria_id,
4540+
quoted_criteria,
4541+
quoted_version_start_incl,
4542+
quoted_version_start_excl,
4543+
quoted_version_end_incl,
4544+
quoted_version_end_excl,
4545+
quoted_status);
45074546

4508-
inserts->current_chunk_size++;
4547+
inserts->current_chunk_size++;
4548+
}
45094549

45104550
g_free (quoted_criteria);
45114551
g_free (quoted_version_start_incl);
45124552
g_free (quoted_version_start_excl);
45134553
g_free (quoted_version_end_incl);
45144554
g_free (quoted_version_end_excl);
4555+
g_free (quoted_status);
45154556

45164557
matches_array = cJSON_GetObjectItemCaseSensitive (match_string, "matches");
45174558

@@ -4541,20 +4582,38 @@ handle_json_cpe_match_string (inserts_t *inserts, inserts_t *matches_inserts,
45414582
return -1;
45424583
}
45434584

4544-
quoted_cpe_name_id = sql_quote (cpe_name_id);
4585+
quoted_cpe_name_id = quote_func (cpe_name_id);
45454586
quoted_cpe_name = fs_to_uri_convert_and_quote_cpe_name (cpe_name,
4546-
sql_quote);
4587+
quote_func);
45474588

4548-
first = inserts_check_size (matches_inserts);
4589+
if (use_copy)
4590+
{
4591+
if (db_copy_buffer_append_printf (matches_copy_buffer,
4592+
"%s\t%s\t%s\n",
4593+
quoted_match_criteria_id,
4594+
quoted_cpe_name_id,
4595+
quoted_cpe_name))
4596+
{
4597+
g_free (quoted_match_criteria_id);
4598+
g_free (quoted_cpe_name_id);
4599+
g_free (quoted_cpe_name);
4600+
return -1;
4601+
}
4602+
4603+
}
4604+
else
4605+
{
4606+
first = inserts_check_size (matches_inserts);
45494607

4550-
g_string_append_printf (matches_inserts->statement,
4551-
"%s ('%s', '%s', '%s')",
4552-
first ? "" : ",",
4553-
quoted_match_criteria_id,
4554-
quoted_cpe_name_id,
4555-
quoted_cpe_name);
4608+
g_string_append_printf (matches_inserts->statement,
4609+
"%s ('%s', '%s', '%s')",
4610+
first ? "" : ",",
4611+
quoted_match_criteria_id,
4612+
quoted_cpe_name_id,
4613+
quoted_cpe_name);
45564614

4557-
matches_inserts->current_chunk_size++;
4615+
matches_inserts->current_chunk_size++;
4616+
}
45584617

45594618
g_free (quoted_cpe_name_id);
45604619
g_free (quoted_cpe_name);
@@ -4578,6 +4637,8 @@ update_scap_cpe_match_strings ()
45784637
gvm_json_pull_event_t event;
45794638
gvm_json_pull_parser_t parser;
45804639
inserts_t inserts, matches_inserts;
4640+
db_copy_buffer_t copy_buffer, matches_copy_buffer;
4641+
gboolean use_copy = TRUE;
45814642

45824643
current_json_path = g_build_filename (GVM_SCAP_DATA_DIR,
45834644
"nvd-cpe-matches.json.gz",
@@ -4665,28 +4726,52 @@ update_scap_cpe_match_strings ()
46654726
}
46664727

46674728
sql_begin_immediate ();
4668-
inserts_init (&inserts,
4669-
CPE_MAX_CHUNK_SIZE,
4670-
setting_secinfo_sql_buffer_threshold_bytes (),
4671-
"INSERT INTO scap2.cpe_match_strings"
4672-
" (match_criteria_id, criteria, version_start_incl,"
4673-
" version_start_excl, version_end_incl, version_end_excl,"
4674-
" status)"
4675-
" VALUES ",
4676-
" ON CONFLICT (match_criteria_id) DO UPDATE"
4677-
" SET criteria = EXCLUDED.criteria,"
4678-
" version_start_incl = EXCLUDED.version_start_incl,"
4679-
" version_start_excl = EXCLUDED.version_start_excl,"
4680-
" version_end_incl = EXCLUDED.version_end_incl,"
4681-
" version_end_excl = EXCLUDED.version_end_excl,"
4682-
" status = EXCLUDED.status");
4683-
4684-
inserts_init (&matches_inserts, 10,
4685-
setting_secinfo_sql_buffer_threshold_bytes (),
4686-
"INSERT INTO scap2.cpe_matches"
4687-
" (match_criteria_id, cpe_name_id, cpe_name)"
4688-
" VALUES ",
4689-
"");
4729+
4730+
if (use_copy)
4731+
{
4732+
db_copy_buffer_init
4733+
(&copy_buffer,
4734+
setting_secinfo_sql_buffer_threshold_bytes () / 2,
4735+
"COPY scap2.cpe_match_strings"
4736+
" (match_criteria_id, criteria, version_start_incl,"
4737+
" version_start_excl, version_end_incl, version_end_excl,"
4738+
" status)"
4739+
" FROM STDIN");
4740+
4741+
db_copy_buffer_init
4742+
(&matches_copy_buffer,
4743+
setting_secinfo_sql_buffer_threshold_bytes () / 2,
4744+
"COPY scap2.cpe_matches"
4745+
" (match_criteria_id, cpe_name_id, cpe_name)"
4746+
" FROM STDIN");
4747+
}
4748+
else
4749+
{
4750+
inserts_init
4751+
(&inserts,
4752+
CPE_MAX_CHUNK_SIZE,
4753+
setting_secinfo_sql_buffer_threshold_bytes (),
4754+
"INSERT INTO scap2.cpe_match_strings"
4755+
" (match_criteria_id, criteria, version_start_incl,"
4756+
" version_start_excl, version_end_incl, version_end_excl,"
4757+
" status)"
4758+
" VALUES ",
4759+
" ON CONFLICT (match_criteria_id) DO UPDATE"
4760+
" SET criteria = EXCLUDED.criteria,"
4761+
" version_start_incl = EXCLUDED.version_start_incl,"
4762+
" version_start_excl = EXCLUDED.version_start_excl,"
4763+
" version_end_incl = EXCLUDED.version_end_incl,"
4764+
" version_end_excl = EXCLUDED.version_end_excl,"
4765+
" status = EXCLUDED.status");
4766+
4767+
inserts_init
4768+
(&matches_inserts, 10,
4769+
setting_secinfo_sql_buffer_threshold_bytes (),
4770+
"INSERT INTO scap2.cpe_matches"
4771+
" (match_criteria_id, cpe_name_id, cpe_name)"
4772+
" VALUES ",
4773+
"");
4774+
}
46904775

46914776
gvm_json_pull_parser_next (&parser, &event);
46924777
while (event.type == GVM_JSON_PULL_EVENT_OBJECT_START)
@@ -4699,8 +4784,16 @@ update_scap_cpe_match_strings ()
46994784
g_warning ("%s: Error expanding match string item: %s",
47004785
__func__, error_message);
47014786
cJSON_Delete (cpe_match_string_item);
4702-
inserts_free (&inserts);
4703-
inserts_free (&matches_inserts);
4787+
if (use_copy)
4788+
{
4789+
db_copy_buffer_cleanup (&copy_buffer);
4790+
db_copy_buffer_cleanup (&matches_copy_buffer);
4791+
}
4792+
else
4793+
{
4794+
inserts_free (&inserts);
4795+
inserts_free (&matches_inserts);
4796+
}
47044797
sql_commit ();
47054798
g_warning ("Update of CPE match strings failed");
47064799
gvm_json_pull_event_cleanup (&event);
@@ -4710,11 +4803,24 @@ update_scap_cpe_match_strings ()
47104803
}
47114804
if (handle_json_cpe_match_string (&inserts,
47124805
&matches_inserts,
4806+
&copy_buffer,
4807+
&matches_copy_buffer,
4808+
use_copy,
47134809
cpe_match_string_item))
47144810
{
47154811
cJSON_Delete (cpe_match_string_item);
47164812
inserts_free (&inserts);
47174813
inserts_free (&matches_inserts);
4814+
if (use_copy)
4815+
{
4816+
db_copy_buffer_cleanup (&copy_buffer);
4817+
db_copy_buffer_cleanup (&matches_copy_buffer);
4818+
}
4819+
else
4820+
{
4821+
inserts_free (&inserts);
4822+
inserts_free (&matches_inserts);
4823+
}
47184824
sql_commit ();
47194825
g_warning ("Update of CPE match strings failed");
47204826
gvm_json_pull_event_cleanup (&event);
@@ -4744,8 +4850,25 @@ update_scap_cpe_match_strings ()
47444850
return -1;
47454851
}
47464852

4747-
inserts_run (&inserts, TRUE);
4748-
inserts_run (&matches_inserts, TRUE);
4853+
if (use_copy)
4854+
{
4855+
if (db_copy_buffer_commit (&copy_buffer, TRUE)
4856+
|| db_copy_buffer_commit (&matches_copy_buffer, TRUE))
4857+
{
4858+
db_copy_buffer_cleanup (&copy_buffer);
4859+
db_copy_buffer_cleanup (&matches_copy_buffer);
4860+
return -1;
4861+
}
4862+
sql ("SELECT setval('scap2.cpe_match_strings_id_seq', max(id))"
4863+
" FROM scap2.cpe_match_strings;");
4864+
sql ("SELECT setval('scap2.cpe_matches_id_seq', max(id))"
4865+
" FROM scap2.cpe_matches;");
4866+
}
4867+
else
4868+
{
4869+
inserts_run (&inserts, TRUE);
4870+
inserts_run (&matches_inserts, TRUE);
4871+
}
47494872
sql_commit ();
47504873
gvm_json_pull_event_cleanup (&event);
47514874
gvm_json_pull_parser_cleanup (&parser);

0 commit comments

Comments
 (0)