MXS-4742: Add server variables to maxctrl report

The server global variables, global status and InnoDB status are now
included in the report generated by MaxCtrl. This will store all the
required information in one place in a consistent format that'll make
analyzing problems faster.

Since querying the server variables can be slow, the retrieval of the
information must be done in two steps. By first collecting the target
servers to query, the MainWorker is blocked only for the duration of the
iteration over the monitor and server lists. The querying itself is done
asynchronously by using the callback mechanism in the HttpResponse which
executes it in the mxs::thread_pool().

The information returned by the endpoint is not something that is
expected to be monitored often, at least for now, which is why it is
prefixed in the /maxscale/debug path.

Added a system test for report generation. In practice the report
generation has already been used by the system tests as it is called
when a test ends up in a timeout. Explicitly testing that it succeeds in
the maxctrl_basic will catch any case where an error will occur.

include/maxscale/json_api.hh

@@ -41,6 +41,7 @@
 #define MXS_JSON_API_QC          "/maxscale/query_classifier/"
 #define MXS_JSON_API_QC_CLASSIFY "/maxscale/query_classifier/classify"
 #define MXS_JSON_API_QC_CACHE    "/maxscale/query_classifier/cache"
+#define MXS_JSON_API_SERVER_DIAG "/maxscale/debug/server_diagnostics"
 #define MXS_JSON_API_USERS       "/users/"
 
 /**

include/maxscale/monitor.hh

@@ -529,6 +529,8 @@ public:
     const std::string m_name;           /**< Monitor instance name. */
     const std::string m_module;         /**< Name of the monitor module */
 
+    const MonitorServer::ConnectionSettings& conn_settings() const;
+
 protected:
     /**
      * Stop the monitor. If the monitor uses a polling thread, the thread should be stopped.
@@ -656,7 +658,6 @@ protected:
     };
 
     const Settings&                          settings() const;
-    const MonitorServer::ConnectionSettings& conn_settings() const;
 
     /**< Number of monitor ticks ran. Derived classes should increment this whenever completing a tick. */
     std::atomic_long m_ticks {0};

maxctrl/lib/create.js

@@ -617,6 +617,7 @@ exports.builder = function (yargs) {
             { endpoint: "maxscale/query_classifier", name: "query_classifier" },
             { endpoint: "maxscale/threads", name: "threads" },
             { endpoint: "maxscale/logs/data", name: "logs", options: "?page[size]=" + argv.lines },
+            { endpoint: "maxscale/debug/server_diagnostics", name: "server_diagnostics" },
           ];
 
           var data = {};

server/core/internal/monitormanager.hh

@@ -181,4 +181,30 @@ public:
      * @return True on success
      */
     static bool wait_one_tick();
+
+    using ConnDetails = std::vector<std::pair<SERVER*, mxs::MonitorServer::ConnectionSettings>>;
+
+    /**
+     * Get connection settings for each server
+     *
+     * With the help of MonitorServer::ping_or_connect_to_db(), the settings can be used to execute queries
+     * without blocking the monitors or the MainWorker.
+     *
+     * @return The connection settings for all monitored servers
+     */
+    static ConnDetails get_connection_settings();
+
+    /**
+     * Connect to the servers and get JSON diagnostics from them
+     *
+     * This function connects to the servers and converts the results of diagnostic queries
+     * (e.g. SHOW GLOBAL VARIABLES) into JSON. Since this function can block for a long time, it should be
+     * executed asynchronously by the REST-API.
+     *
+     * @param servers The connection details from get_connection_settings()
+     * @param host    The hostname of this MaxScale instance
+     *
+     * @return The results as JSON
+     */
+    static json_t* server_diagnostics(const ConnDetails& servers, const char* host);
 };

server/core/monitormanager.cc

@@ -18,6 +18,7 @@
 #include <maxbase/format.hh>
 #include <maxscale/json_api.hh>
 #include <maxscale/paths.hh>
+#include <maxscale/mysql_utils.hh>
 
 #include "internal/config.hh"
 #include "internal/monitor.hh"
@@ -83,6 +84,22 @@ class ThisUnit
         m_deact_monitors.push_back(monitor);
     }
 
+    MonitorManager::ConnDetails get_connection_settings()
+    {
+        MonitorManager::ConnDetails servers;
+        Guard guard(m_all_monitors_lock);
+
+        for (auto* m : m_all_monitors)
+        {
+            for (auto* s : m->servers())
+            {
+                servers.emplace_back(s->server, m->conn_settings());
+            }
+        }
+
+        return servers;
+    }
+
 private:
     std::mutex            m_all_monitors_lock;  /**< Protects access to arrays */
     std::vector<Monitor*> m_all_monitors;       /**< Global list of monitors, in configuration file order */
@@ -519,3 +536,64 @@ bool MonitorManager::remove_server_from_monitor(mxs::Monitor* mon, SERVER* serve
 
     return success;
 }
+
+// static
+MonitorManager::ConnDetails MonitorManager::get_connection_settings()
+{
+    return this_unit.get_connection_settings();
+}
+
+// static
+json_t* MonitorManager::server_diagnostics(const MonitorManager::ConnDetails& servers, const char* host)
+{
+    json_t* attr = json_object();
+
+    for (const auto& kv : servers)
+    {
+        MYSQL* conn = nullptr;
+        std::string err;
+        auto result = MonitorServer::ping_or_connect_to_db(kv.second, *kv.first, &conn, &err);
+
+        if (result == MonitorServer::ConnectResult::NEWCONN_OK)
+        {
+            auto json_query = [&](auto sql, int name_idx, int val_idx){
+                unsigned int errnum;
+
+                if (auto r = mxs::execute_query(conn, sql, &err, &errnum))
+                {
+                    json_t* var = json_object();
+
+                    while (r->next_row())
+                    {
+                        json_object_set_new(var, r->get_string(name_idx).c_str(),
+                                            json_string(r->get_string(val_idx).c_str()));
+                    }
+
+                    return var;
+                }
+                else
+                {
+                    return json_pack("{s: s}", "error", err.c_str());
+                }
+            };
+
+            json_t* obj = json_object();
+            json_object_set_new(obj, "global_variables", json_query("SHOW GLOBAL VARIABLES", 0, 1));
+            json_object_set_new(obj, "global_status", json_query("SHOW GLOBAL STATUS", 0, 1));
+            json_object_set_new(obj, "engine_status", json_query("SHOW ENGINE INNODB STATUS", 0, 2));
+            json_object_set_new(attr, kv.first->name(), obj);
+
+            mysql_close(conn);
+        }
+        else
+        {
+            json_object_set_new(attr, kv.first->name(), json_pack("{s: s}", "error", err.c_str()));
+        }
+    }
+
+    json_t* rval = json_object();
+    json_object_set_new(rval, CN_ID, json_string("server_diagnostics"));
+    json_object_set_new(rval, CN_TYPE, json_string("server_diagnostics"));
+    json_object_set_new(rval, CN_ATTRIBUTES, attr);
+    return mxs_json_resource(host, MXS_JSON_API_SERVER_DIAG, rval);
+}

server/core/resource.cc

@@ -1041,6 +1041,19 @@ HttpResponse cb_monitor_wait(const HttpRequest& request)
     return HttpResponse(MHD_HTTP_OK);
 }
 
+HttpResponse cb_debug_server_diagnostics(const HttpRequest& request)
+{
+    auto servers = MonitorManager::get_connection_settings();
+    std::string host = request.host();
+
+    // The server diagnostics requires blocking communication with the databases. To prevent it from blocking
+    // the REST-API, the MainWorker and the monitors, they need to be executed asynchronously in the thread
+    // pool.
+    return HttpResponse([servers, host](){
+        return HttpResponse(MHD_HTTP_OK, MonitorManager::server_diagnostics(servers, host.c_str()));
+    });
+}
+
 HttpResponse cb_create_user(const HttpRequest& request)
 {
     mxb_assert(request.get_json());
@@ -1392,6 +1405,7 @@ class RootResource
 
         /** Debug utility endpoints */
         m_get.emplace_back(cb_monitor_wait, "maxscale", "debug", "monitor_wait");
+        m_get.emplace_back(cb_debug_server_diagnostics, "maxscale", "debug", "server_diagnostics");
 
         /** Create new resources */
         m_post.emplace_back(REQ_BODY | REQ_SYNC, cb_create_server, "servers");

system-test/maxctrl_basic.cc

@@ -109,7 +109,8 @@ int main(int argc, char** argv)
                               "maxctrl rotate logs;"
                               "maxctrl call command mariadbmon reset-replication MySQL-Monitor;"
                               "maxctrl api get servers;"
-                              "maxctrl classify 'select 1';");
+                              "maxctrl classify 'select 1';"
+                              "maxctrl --timeout 30s create report test-report.txt");
 
     test.tprintf("MXS-3697: MaxCtrl fails with \"ENOENT: no such file or directory, stat '/~/.maxctrl.cnf'\" "
                  "when running commands from the root directory.");