Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Chunk CPE INSERTs, to reduce memory usage (9.0) #890

Merged
merged 6 commits into from
Dec 4, 2019
Merged

Chunk CPE INSERTs, to reduce memory usage (9.0) #890

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b7dc609
Chunk CPE INSERTs, to reduce memory usage
mattmundell Dec 4, 2019
4d9a293
Update changelog
mattmundell Dec 4, 2019
27ea7d3
Make closing SQL an arg of inserts_init
mattmundell Dec 4, 2019
b513e96
Make opening SQL an arg of inserts_init
mattmundell Dec 4, 2019
9e35b8d
Move buffered insert API up, now that it is generic
mattmundell Dec 4, 2019
3846a65
Merge branch 'gvmd-9.0' into chunk-cpe-inserts-9.0
mattmundell Dec 4, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Try importing private keys with libssh if GnuTLS fails [#841](https://github.com/greenbone/gvmd/pull/841)
- Allow resuming OSPd-based OpenVAS tasks [#869](https://github.com/greenbone/gvmd/pull/869)
- Require Postgres 9.6 as a minimum [#872](https://github.com/greenbone/gvmd/pull/872)
- Speed up the SCAP sync [#875](https://github.com/greenbone/gvmd/pull/875) [#877](https://github.com/greenbone/gvmd/pull/877) [#879](https://github.com/greenbone/gvmd/pull/879) [#881](https://github.com/greenbone/gvmd/pull/881) [#883](https://github.com/greenbone/gvmd/pull/883) [#887](https://github.com/greenbone/gvmd/pull/887) [#889](https://github.com/greenbone/gvmd/pull/889) [#891](https://github.com/greenbone/gvmd/pull/891)
- Speed up the SCAP sync [#875](https://github.com/greenbone/gvmd/pull/875) [#877](https://github.com/greenbone/gvmd/pull/877) [#879](https://github.com/greenbone/gvmd/pull/879) [#881](https://github.com/greenbone/gvmd/pull/881) [#883](https://github.com/greenbone/gvmd/pull/883) [#887](https://github.com/greenbone/gvmd/pull/887) [#889](https://github.com/greenbone/gvmd/pull/889) [#890](https://github.com/greenbone/gvmd/pull/890) [#891](https://github.com/greenbone/gvmd/pull/891)

### Fixed
- Consider results_trash when deleting users [#799](https://github.com/greenbone/gvmd/pull/799)
Expand Down
191 changes: 162 additions & 29 deletions src/manage_sql_secinfo.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@

/* Static variables. */

/**
* @brief Maximum number of rows in an INSERT.
*/
#define CPE_MAX_CHUNK_SIZE 10000

/**
* @brief Commit size for updates.
*/
Expand Down Expand Up @@ -166,6 +171,140 @@ increment_transaction_size (int* current_size)
}
}


/* Helper: buffer structure for INSERTs. */

/**
* @brief Buffer for INSERT statements.
*/
typedef struct
{
array_t *statements; ///< Buffered statements.
GString *statement; ///< Current statemet.
int current_chunk_size; ///< Number of rows in current statement.
int max_chunk_size; ///< Max number of rows per INSERT.
gchar *open_sql; ///< SQL to open each statement.
gchar *close_sql; ///< SQL to close each statement.
} inserts_t;

/**
* @brief Check size of current statement.
*
* @param[in] inserts Insert buffer.
* @param[in] max_chunk_size Max chunk size.
* @param[in] open_sql SQL to to start each statement.
* @param[in] close_sql SQL to append to the end of each statement.
*
* @return Whether this is the first value in the statement.
*/
static void
inserts_init (inserts_t *inserts, int max_chunk_size, const gchar *open_sql,
const gchar *close_sql)
{
inserts->statements = make_array ();
inserts->statement = NULL;
inserts->current_chunk_size = 0;
inserts->max_chunk_size = max_chunk_size;
inserts->open_sql = open_sql ? g_strdup (open_sql) : NULL;
inserts->close_sql = close_sql ? g_strdup (close_sql) : NULL;
}

/**
* @brief Close the current statement.
*
* @param[in] inserts Insert buffer.
*/
static void
inserts_statement_close (inserts_t *inserts)
{
if (inserts->statement)
{
if (inserts->close_sql)
g_string_append (inserts->statement, inserts->close_sql);
g_string_append (inserts->statement, ";");
}
}

/**
* @brief Check size of current statement.
*
* @param[in] inserts Insert buffer.
*
* @return Whether this is the first value in the statement.
*/
static int
inserts_check_size (inserts_t *inserts)
{
int first;

first = 0;

if (inserts->statement
&& inserts->current_chunk_size >= inserts->max_chunk_size)
{
inserts_statement_close (inserts);
array_add (inserts->statements, inserts->statement);
inserts->statement = NULL;
inserts->current_chunk_size = 0;
}

if (inserts->statement == NULL)
{
inserts->statement
= g_string_new (inserts->open_sql ? inserts->open_sql : "");
first = 1;
}

return first;
}

/**
* @brief Free everything.
*
* @param[in] inserts Insert buffer.
*/
static void
inserts_free (inserts_t *inserts)
{
int index;

for (index = 0; index < inserts->statements->len; index++)
g_string_free (g_ptr_array_index (inserts->statements, index), TRUE);
g_ptr_array_free (inserts->statements, TRUE);
g_free (inserts->open_sql);
g_free (inserts->close_sql);
bzero (inserts, sizeof (*inserts));
}

/**
* @brief Run the INSERT SQL, freeing the buffers.
*
* @param[in] inserts Insert buffer.
*/
static void
inserts_run (inserts_t *inserts)
{
guint index;

if (inserts->statement)
{
inserts_statement_close (inserts);
array_add (inserts->statements, inserts->statement);
inserts->statement = NULL;
inserts->current_chunk_size = 0;
}

for (index = 0; index < inserts->statements->len; index++)
{
GString *statement;

statement = g_ptr_array_index (inserts->statements, index);
sql ("%s", statement->str);
}

inserts_free (inserts);
}


/* CPE data. */

Expand Down Expand Up @@ -2006,7 +2145,7 @@ update_cert_bund_advisories (int last_cert_update)
* @return 0 success, -1 error.
*/
static int
insert_scap_cpe (GString **inserts, element_t cpe_item, element_t item_metadata,
insert_scap_cpe (inserts_t *inserts, element_t cpe_item, element_t item_metadata,
int modification_time)
{
gchar *name, *status, *deprecated, *nvd_id;
Expand Down Expand Up @@ -2094,15 +2233,9 @@ insert_scap_cpe (GString **inserts, element_t cpe_item, element_t item_metadata,
quoted_nvd_id = sql_quote (nvd_id);
g_free (nvd_id);

first = (*inserts == NULL);
if (first)
*inserts = g_string_new ("INSERT INTO scap.cpes"
" (uuid, name, title, creation_time,"
" modification_time, status, deprecated_by_id,"
" nvd_id)"
" VALUES");
first = inserts_check_size (inserts);

g_string_append_printf (*inserts,
g_string_append_printf (inserts->statement,
"%s ('%s', '%s', '%s', %i, %i, '%s', %s, '%s')",
first ? "" : ",",
quoted_name,
Expand All @@ -2114,6 +2247,8 @@ insert_scap_cpe (GString **inserts, element_t cpe_item, element_t item_metadata,
deprecated ? deprecated : "NULL",
quoted_nvd_id);

inserts->current_chunk_size++;

g_free (quoted_title);
g_free (quoted_name);
g_free (quoted_status);
Expand All @@ -2139,7 +2274,7 @@ update_scap_cpes (int last_scap_update)
gsize xml_len;
GStatBuf state;
int updated_scap_cpes, last_cve_update;
GString *inserts;
inserts_t inserts;

updated_scap_cpes = 0;
full_path = g_build_filename (GVM_SCAP_DATA_DIR,
Expand Down Expand Up @@ -2200,7 +2335,21 @@ update_scap_cpes (int last_scap_update)

sql_begin_immediate ();

inserts = NULL;
inserts_init (&inserts,
CPE_MAX_CHUNK_SIZE,
"INSERT INTO scap.cpes"
" (uuid, name, title, creation_time,"
" modification_time, status, deprecated_by_id,"
" nvd_id)"
" VALUES",
" ON CONFLICT (uuid) DO UPDATE"
" SET name = EXCLUDED.name,"
" title = EXCLUDED.title,"
" creation_time = EXCLUDED.creation_time,"
" modification_time = EXCLUDED.modification_time,"
" status = EXCLUDED.status,"
" deprecated_by_id = EXCLUDED.deprecated_by_id,"
" nvd_id = EXCLUDED.nvd_id");
cpe_item = element_first_child (cpe_list);
while (cpe_item)
{
Expand Down Expand Up @@ -2245,29 +2394,13 @@ update_scap_cpes (int last_scap_update)

element_free (element);

assert (updated_scap_cpes ? (inserts != NULL) : (inserts == NULL));

if (updated_scap_cpes)
{
sql ("%s"
" ON CONFLICT (uuid) DO UPDATE"
" SET name = EXCLUDED.name,"
" title = EXCLUDED.title,"
" creation_time = EXCLUDED.creation_time,"
" modification_time = EXCLUDED.modification_time,"
" status = EXCLUDED.status,"
" deprecated_by_id = EXCLUDED.deprecated_by_id,"
" nvd_id = EXCLUDED.nvd_id",
inserts->str);
g_string_free (inserts, TRUE);
}
inserts_run (&inserts);
timopollmeier marked this conversation as resolved.
Show resolved Hide resolved

sql_commit ();
return updated_scap_cpes;

fail:
if (inserts)
g_string_free (inserts, TRUE);
inserts_free (&inserts);
element_free (element);
g_warning ("Update of CPEs failed");
sql_commit ();
Expand Down