From 0f7e7f389148c544a4b967f88df7108d27bc062c Mon Sep 17 00:00:00 2001 From: Yashwant Sahu Date: Mon, 11 Nov 2024 20:15:02 +0530 Subject: [PATCH] Previously mysql_hostgroup_attributes was not read from configuration file. Now mysql_hostgroup_attributes will be read from configuration file. backporting of #4704 --- lib/ProxySQL_Config.cpp | 127 +++++++++ ...sql_hostgroup_attributes_config_file-t.cpp | 249 ++++++++++++++++++ 2 files changed, 376 insertions(+) create mode 100644 test/tap/tests/mysql_hostgroup_attributes_config_file-t.cpp diff --git a/lib/ProxySQL_Config.cpp b/lib/ProxySQL_Config.cpp index dc7360471a..c84673df62 100644 --- a/lib/ProxySQL_Config.cpp +++ b/lib/ProxySQL_Config.cpp @@ -972,6 +972,42 @@ int ProxySQL_Config::Write_MySQL_Servers_to_configfile(std::string& data) { } } + if (sqlite_resultset) + delete sqlite_resultset; + + query = (char *)"SELECT * FROM mysql_hostgroup_attributes"; + admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); + if (error) { + proxy_error("Error on read from mysql_hostgroup_attributes: %s\n", error); + return -1; + } else { + if (sqlite_resultset) { + data += "mysql_hostgroup_attributes:\n(\n"; + bool isNext = false; + for (auto r : sqlite_resultset->rows) { + if (isNext) + data += ",\n"; + data += "\t{\n"; + addField(data, "hostgroup_id", r->fields[0], ""); + addField(data, "max_num_online_servers", r->fields[1], ""); + addField(data, "autocommit", r->fields[2], ""); + addField(data, "free_connections_pct", r->fields[3], ""); + addField(data, "init_connect", r->fields[4]); + addField(data, "multiplex", r->fields[5], ""); + addField(data, "connection_warming", r->fields[6], ""); + addField(data, "throttle_connections_per_sec", r->fields[7], ""); + addField(data, "ignore_session_variables", r->fields[8]); + addField(data, "hostgroup_settings", r->fields[9]); + addField(data, "servers_defaults", r->fields[10]); + addField(data, "comment", r->fields[11]); + + data += "\t}"; + isNext = true; + } + data += "\n)\n"; + } + } + if (sqlite_resultset) delete sqlite_resultset; @@ -1289,6 +1325,97 @@ int ProxySQL_Config::Read_MySQL_Servers_from_configfile() { rows++; } } + if (root.exists("mysql_hostgroup_attributes") == true) { + const Setting &mysql_hostgroup_attributes = root["mysql_hostgroup_attributes"]; + int count = mysql_hostgroup_attributes.getLength(); + + for (i = 0; i < count; i++) { + const Setting &hostgroup_attributes = mysql_hostgroup_attributes[i]; + bool is_first_field = true; + int integer_val = 0; + std::string string_val = ""; + std::string fields = ""; + std::string values = ""; + + auto process_field = [&](const std::string &field_name, const std::string &field_value, int is_int) { + if (!is_first_field) { + fields += ", "; + values += ", "; + } + else { + is_first_field = false; + } + fields += field_name; + + if (is_int) { + values += field_value; + } + else { + char *cs = strdup(field_value.c_str()); + char *ecs = escape_string_single_quotes(cs, false); + values += std::string("'") + ecs + "'"; + if (cs != ecs) free(cs); + free(ecs); + } + }; + + // Only inserting/updating fields which are in configuration file. + // Fields default will be from table schema. + + // Parsing integer field + if (hostgroup_attributes.lookupValue("hostgroup_id", integer_val) ) { + process_field("hostgroup_id", to_string(integer_val), true); + } + else { + proxy_error("Admin: detected a mysql_hostgroup_attributes in config file without a mandatory hostgroup_id.\n"); + continue; + } + if (hostgroup_attributes.lookupValue("max_num_online_servers", integer_val)) { + process_field("max_num_online_servers", to_string(integer_val), true); + } + if (hostgroup_attributes.lookupValue("autocommit", integer_val)) { + process_field("autocommit", to_string(integer_val), true); + } + if (hostgroup_attributes.lookupValue("free_connections_pct", integer_val)) { + process_field("free_connections_pct", to_string(integer_val), true); + } + if (hostgroup_attributes.lookupValue("multiplex", integer_val)) { + process_field("multiplex", to_string(integer_val), true); + } + if (hostgroup_attributes.lookupValue("connection_warming", integer_val)) { + process_field("connection_warming", to_string(integer_val), true); + } + if (hostgroup_attributes.lookupValue("throttle_connections_per_sec", integer_val)) { + process_field("throttle_connections_per_sec", to_string(integer_val), true); + } + // Parsing string field + if (hostgroup_attributes.lookupValue("init_connect", string_val)) { + process_field("init_connect", string_val, false); + } + if (hostgroup_attributes.lookupValue("ignore_session_variables", string_val)) { + process_field("ignore_session_variables", string_val, false); + } + if (hostgroup_attributes.lookupValue("hostgroup_settings", string_val)) { + process_field("hostgroup_settings", string_val, false); + } + if (hostgroup_attributes.lookupValue("servers_defaults", string_val)) { + process_field("servers_defaults", string_val, false); + } + if (hostgroup_attributes.lookupValue("comment", string_val)) { + process_field("comment", string_val, false); + } + + std::string s_query = "INSERT OR REPLACE INTO mysql_hostgroup_attributes ("; + s_query += fields + ") VALUES (" + values + ")"; + + //fprintf(stderr, "%s\n", s_query.c_str()); + if (admindb->execute(s_query.c_str()) == false) { + proxy_error("Admin: detected a mysql_hostgroup_attributes invalid value. Failed to insert in the table.\n"); + continue; + } + rows++; + } + } admindb->execute("PRAGMA foreign_keys = ON"); return rows; } diff --git a/test/tap/tests/mysql_hostgroup_attributes_config_file-t.cpp b/test/tap/tests/mysql_hostgroup_attributes_config_file-t.cpp new file mode 100644 index 0000000000..0e5bd59a9a --- /dev/null +++ b/test/tap/tests/mysql_hostgroup_attributes_config_file-t.cpp @@ -0,0 +1,249 @@ +/** + * @file mysql_hostgroup_attributes_config_file-t.cpp + * @brief Reading and saving of 'mysql_hostgroup_attributes' table from configuration file: + */ + +#include +#include +#include +#include + +#include "mysql.h" +#include "mysqld_error.h" + +#include "json.hpp" + +#include "tap.h" +#include "utils.h" +#include "command_line.h" + +using nlohmann::json; +using std::string; +using std::fstream; + +int validate_mysql_hostgroup_attributes_from_config(MYSQL* admin) { + string hostgroup_attributes_values[5][12] = { + {"900000", "900000", "-1", "11", "ic1", "0", "1", "9001", "{\"isv\":100}", "{\"hs\":200}", "{\"weight\":100,\"max_connections\":500}", "attributes test hostgroup 900000"}, + {"900001", "900001", "0", "12", "ic2", "1", "0", "9002", "{\"isv\":101}", "{\"hs\":201}", "{\"weight\":101,\"max_connections\":501}", "attributes test hostgroup 900001"}, + {"900002", "900002", "1", "13", "ic3", "0", "1", "9003", "{\"isv\":102}", "{\"hs\":202}", "{\"weight\":102,\"max_connections\":502}", "attributes test hostgroup 900002"}, + {"900003", "900003", "-1", "14", "ic4", "1", "0", "9004", "{\"isv\":103}", "{\"hs\":203}", "{\"weight\":103,\"max_connections\":503}", "attributes test hostgroup 900003"}, + {"900004", "900004", "0", "15", "ic5", "0", "1", "9005", "{\"isv\":104}", "{\"hs\":204}", "{\"weight\":104,\"max_connections\":504}", "attributes test hostgroup 900004"}}; + + auto check_result = [&] () { + MYSQL_RES* myres = mysql_store_result(admin); + MYSQL_ROW myrow; + + int row_num {0}; + while ((myrow = mysql_fetch_row(myres))) { + + int field_num = {0}; + for (field_num = 0; field_num < 12; field_num++) { + if(!myrow[field_num]) { + diag("ERROR: hostgroup_attributes_values field: %d is null", field_num); + mysql_free_result(myres); + return false; + } + if(strncmp(myrow[field_num], hostgroup_attributes_values[row_num][field_num].c_str(), + sizeof(hostgroup_attributes_values[row_num][field_num]))) { + diag("INSERTED 'field' should match with config value - Exp: `%s`, Act: `%s`", + hostgroup_attributes_values[row_num][field_num].c_str(), myrow[field_num]); + mysql_free_result(myres); + return false; + } + } + row_num++; + } + mysql_free_result(myres); + return true; + }; + + diag("Checking loading of mysql_hostgroup_attributes from config file"); + MYSQL_QUERY_T(admin, "SELECT * FROM mysql_hostgroup_attributes"); + + auto b_config_parsing = check_result(); + + ok( + b_config_parsing == true, + "Parsed hostgroup_attributes_values values are correct" + ); + + MYSQL_QUERY_T(admin, "DELETE FROM mysql_hostgroup_attributes"); + MYSQL_QUERY_T(admin, "LOAD MYSQL SERVERS FROM CONFIG;"); + diag("Checking loading of mysql_hostgroup_attributes from config file after deleting and reloading config"); + MYSQL_QUERY_T(admin, "SELECT * FROM mysql_hostgroup_attributes"); + + b_config_parsing = check_result(); + + ok( + b_config_parsing == true, + "Parsed hostgroup_attributes_values values after reload config are correct" + ); + + return EXIT_SUCCESS; +} + +void make_hostgroup_attributes_config_lines(std::vector& config_lines){ + config_lines = { + "mysql_hostgroup_attributes:", + "(", + " {", + " hostgroup_id=900000", + " max_num_online_servers=900000", + " autocommit=-1", + " free_connections_pct=11", + " init_connect=\"ic1\"", + " multiplex=0", + " connection_warming=1", + " throttle_connections_per_sec=9001", + " ignore_session_variables=\"{\"isv\":100}\"", + " hostgroup_settings=\"{\"hs\":200}\"", + " servers_defaults=\"{\"weight\":100,\"max_connections\":500}\"", + " comment=\"attributes test hostgroup 900000\"", + " },", + " {", + " hostgroup_id=900001", + " max_num_online_servers=900001", + " autocommit=0", + " free_connections_pct=12", + " init_connect=\"ic2\"", + " multiplex=1", + " connection_warming=0", + " throttle_connections_per_sec=9002", + " ignore_session_variables=\"{\"isv\":101}\"", + " hostgroup_settings=\"{\"hs\":201}\"", + " servers_defaults=\"{\"weight\":101,\"max_connections\":501}\"", + " comment=\"attributes test hostgroup 900001\"", + " },", + " {", + " hostgroup_id=900002", + " max_num_online_servers=900002", + " autocommit=1", + " free_connections_pct=13", + " init_connect=\"ic3\"", + " multiplex=0", + " connection_warming=1", + " throttle_connections_per_sec=9003", + " ignore_session_variables=\"{\"isv\":102}\"", + " hostgroup_settings=\"{\"hs\":202}\"", + " servers_defaults=\"{\"weight\":102,\"max_connections\":502}\"", + " comment=\"attributes test hostgroup 900002\"", + " },", + " {", + " hostgroup_id=900003", + " max_num_online_servers=900003", + " autocommit=-1", + " free_connections_pct=14", + " init_connect=\"ic4\"", + " multiplex=1", + " connection_warming=0", + " throttle_connections_per_sec=9004", + " ignore_session_variables=\"{\"isv\":103}\"", + " hostgroup_settings=\"{\"hs\":203}\"", + " servers_defaults=\"{\"weight\":103,\"max_connections\":503}\"", + " comment=\"attributes test hostgroup 900003\"", + " },", + " {", + " hostgroup_id=900004", + " max_num_online_servers=900004", + " autocommit=0", + " free_connections_pct=15", + " init_connect=\"ic5\"", + " multiplex=0", + " connection_warming=1", + " throttle_connections_per_sec=9005", + " ignore_session_variables=\"{\"isv\":104}\"", + " hostgroup_settings=\"{\"hs\":204}\"", + " servers_defaults=\"{\"weight\":104,\"max_connections\":504}\"", + " comment=\"attributes test hostgroup 900004\"", + " }", + ")" + }; +} + +int write_mysql_hostgroup_attributes_to_config(MYSQL* admin) { + std::vector config_lines; + string config_file_path {"myproxysql.cnf"}; + string save_config_query = "SAVE CONFIG TO FILE " + config_file_path; + fstream f_stream; + + make_hostgroup_attributes_config_lines(config_lines); + MYSQL_QUERY_T(admin, save_config_query.c_str()); + diag("Checking correctness of config file. "); + + auto check_config_file = [&] () { + int cur_line {0}; + string next_line {""}; + bool first_matched {false}; + const char* c_f_path { config_file_path.c_str() }; + f_stream.open(config_file_path.c_str(), std::fstream::out | std::fstream::in | std::fstream::trunc); + + if (!f_stream.is_open() || !f_stream.good()) { + diag("Failed to open '%s' file: { path: %s, error: %d }", basename(c_f_path), c_f_path, errno); + return false;; + } + while (getline(f_stream, next_line)) { + if (next_line == config_lines[0]) { + first_matched = true; + } + + if (first_matched) { + if (cur_line >= config_lines.size()) { + return true; + } + next_line.erase(remove(next_line.begin(), next_line.end(), '\\'), next_line.end()); + if (next_line == config_lines[cur_line]) { + cur_line++; + } + else { + diag("Confige file line didn't match, config line %s, expected line %s", next_line.c_str(), config_lines[cur_line].c_str()); + return false; + } + } + + next_line = ""; + } + return true; + }; + + auto b_config_parsing = check_config_file(); + ok( + b_config_parsing == true, + "mysql_hostgroup_attributes values are correctly written in config file." + ); + f_stream.close(); + remove(config_file_path.c_str()); + return EXIT_SUCCESS; +} + +int main(int, char**) { + plan(3); + + CommandLine cl; + + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return EXIT_FAILURE; + } + + MYSQL* admin = mysql_init(NULL); + + if (!mysql_real_connect(admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(admin)); + return EXIT_FAILURE; + } + + // For cleanup + MYSQL_QUERY_T(admin, "DROP TABLE IF EXISTS mysql_hostgroup_attributes_0508"); + MYSQL_QUERY_T(admin, "CREATE TABLE mysql_hostgroup_attributes_0508 AS SELECT * FROM mysql_hostgroup_attributes"); + + validate_mysql_hostgroup_attributes_from_config(admin); + write_mysql_hostgroup_attributes_to_config(admin); + +cleanup: + + MYSQL_QUERY_T(admin, "DELETE FROM mysql_hostgroup_attributes"); + MYSQL_QUERY_T(admin, "INSERT INTO mysql_hostgroup_attributes SELECT * FROM mysql_hostgroup_attributes_0508"); + mysql_close(admin); + + return exit_status(); +} \ No newline at end of file