add generation statistics report

Author: Lana243

server/CMakeLists.txt

@@ -114,6 +114,7 @@ file(GLOB ALL_SOURCES
         "${PROJECT_SOURCE_DIR}/src/streams/tests/*"
         "${PROJECT_SOURCE_DIR}/src/testgens/*"
         "${PROJECT_SOURCE_DIR}/src/utils/*"
+        "${PROJECT_SOURCE_DIR}/src/utils/stats/*"
         "${PROJECT_SOURCE_DIR}/src/stubs/*"
         "${PROJECT_SOURCE_DIR}/src/visitors/*"
         "${PROJECT_SOURCE_DIR}/src/commands/*"

server/src/KleeRunner.cpp

@@ -9,14 +9,54 @@
 #include "utils/FileSystemUtils.h"
 #include "utils/KleeUtils.h"
 #include "utils/LogUtils.h"
+#include "utils/stats/CSVReader.h"
 
 #include "loguru.h"
 
 #include <fstream>
 #include <utility>
+#include <utils/stats/TestsGenerationStats.h>
 
 using namespace tests;
 
+namespace {
+    void clearUnusedData(const fs::path &kleeDir) {
+        fs::remove(kleeDir / "assembly.ll");
+        fs::remove(kleeDir / "run.istats");
+    }
+
+    StatsUtils::KleeStats parseKleeStatsReport(const std::string &kleeStatsReport) {
+        std::stringstream ss(kleeStatsReport);
+        StatsUtils::CSVTable parsedCSV = StatsUtils::readCSV(ss, ',');
+        std::map<std::string, std::chrono::milliseconds> timeStats;
+        std::vector<std::string> keys = {"Time(s)", "TSolver(s)", "TResolve(s)"};
+        std::vector<std::chrono::milliseconds> timeValues;
+        for (const auto &key: keys) {
+            if (!CollectionUtils::containsKey(parsedCSV, key)) {
+                LOG_S(WARNING) << StringUtils::stringFormat("Key %s not found in klee-stats report", key);
+            }
+            std::chrono::milliseconds totalTime((int)(1000 * std::stof(parsedCSV[key].back())));
+            timeValues.emplace_back(totalTime);
+        }
+        return StatsUtils::KleeStats(timeValues[0], timeValues[1], timeValues[2]);
+    }
+
+    StatsUtils::KleeStats writeKleeStats(const fs::path &kleeOut) {
+        ShellExecTask::ExecutionParameters kleeStatsParams("klee-stats",
+                                                           {"--utbot-config", kleeOut.string(),
+                                                            "--table-format=readable-csv"});
+        auto[out, status, _] = ShellExecTask::runShellCommandTask(kleeStatsParams);
+        if (status != 0) {
+            LOG_S(ERROR) << "klee-stats call failed:";
+            LOG_S(ERROR) << out;
+        } else {
+            LOG_S(DEBUG) << "klee-stats report:";
+            LOG_S(DEBUG) << '\n' << out;
+        }
+        return parseKleeStatsReport(out);
+    }
+}
+
 KleeRunner::KleeRunner(utbot::ProjectContext projectContext,
                        utbot::SettingsContext settingsContext,
                        fs::path serverBuildDir)
@@ -31,7 +71,8 @@ void KleeRunner::runKlee(const std::vector<tests::TestMethod> &testMethods,
                          const std::shared_ptr<LineInfo> &lineInfo,
                          TestsWriter *testsWriter,
                          bool isBatched,
-                         bool interactiveMode) {
+                         bool interactiveMode,
+                         StatsUtils::TestsGenerationStatsFileMap &generationStats) {
     LOG_SCOPE_FUNCTION(DEBUG);
 
     fs::path kleeOutDir = Paths::getKleeOutDir(projectTmpPath);
@@ -74,120 +115,97 @@ void KleeRunner::runKlee(const std::vector<tests::TestMethod> &testMethods,
             LOG_S(MAX) << logStream.str();
         }
         if (interactiveMode) {
-            if (!batch.empty()) {
-                processBatchWithInteractive(batch, tests, ktests);
-            }
+            processBatchWithInteractive(batch, tests, ktests);
         } else {
-          for (auto const &testMethod : batch) {
-              MethodKtests ktestChunk;
-              processBatchWithoutInteractive(ktestChunk, testMethod, tests);
-              ExecUtils::throwIfCancelled();
-              ktests.push_back(ktestChunk);
-          }
+            processBatchWithoutInteractive(batch, tests, ktests);
         }
+        auto kleeStats = writeKleeStats(Paths::kleeOutDirForFilePath(projectContext, projectTmpPath, filePath));
         generator->parseKTestsToFinalCode(tests, methodNameToReturnTypeMap, ktests, lineInfo,
                                           settingsContext.verbose);
+        generationStats.addFileStats(kleeStats, tests);
     };
 
     testsWriter->writeTestsWithProgress(testsMap, "Running klee", projectContext.testDirPath,
                                         std::move(writeFunctor));
 }
 
-namespace {
-    void clearUnusedData(const fs::path &kleeDir) {
-        fs::remove(kleeDir / "assembly.ll");
-        fs::remove(kleeDir / "run.istats");
-    }
-
-    void writeKleeStats(const fs::path &kleeOut) {
-        ShellExecTask::ExecutionParameters kleeStatsParams("klee-stats",
-                                                           { "--utbot-config", kleeOut.string() });
-        auto [out, status, _] = ShellExecTask::runShellCommandTask(kleeStatsParams);
-        if (status != 0) {
-            LOG_S(ERROR) << "klee-stats call failed:";
-            LOG_S(ERROR) << out;
-        } else {
-            LOG_S(DEBUG) << "klee-stats report:";
-            LOG_S(DEBUG) << '\n' << out;
-        }
-    }
-}
-
 static void processMethod(MethodKtests &ktestChunk,
                           tests::Tests &tests,
                           const fs::path &kleeOut,
                           const tests::TestMethod &method) {
-    if (fs::exists(kleeOut)) {
-        clearUnusedData(kleeOut);
-        bool hasTimeout = false;
-        bool hasError = false;
-        for (auto const &entry : fs::directory_iterator(kleeOut)) {
-            auto const &path = entry.path();
-            if (Paths::isKtestJson(path)) {
-                if (Paths::hasEarly(path)) {
-                    hasTimeout = true;
-                } else if (Paths::hasInternalError(path)) {
-                    hasError = true;
-                } else {
-                    std::unique_ptr<TestCase, decltype(&TestCase_free)> ktestData{
-                        TC_fromFile(path.c_str()), TestCase_free
-                    };
-                    if (ktestData == nullptr) {
-                        LOG_S(WARNING) << "Unable to open .ktestjson file";
-                        continue;
-                    }
-                    UTBotKTest::Status status = Paths::hasError(path) ? UTBotKTest::Status::FAILED
-                                                                      : UTBotKTest::Status::SUCCESS;
-                    std::vector<ConcretizedObject> kTestObjects(
-                        ktestData->objects, ktestData->objects + ktestData->n_objects);
-
-                    std::vector<UTBotKTestObject> objects = CollectionUtils::transform(
-                        kTestObjects, [](const ConcretizedObject &kTestObject) {
-                            return UTBotKTestObject{ kTestObject };
-                        });
-
-                    ktestChunk[method].emplace_back(objects, status);
+    if (!fs::exists(kleeOut)) {
+        return;
+    }
+
+    clearUnusedData(kleeOut);
+    bool hasTimeout = false;
+    bool hasError = false;
+    for (auto const &entry : fs::directory_iterator(kleeOut)) {
+        auto const &path = entry.path();
+        if (Paths::isKtestJson(path)) {
+            if (Paths::hasEarly(path)) {
+                hasTimeout = true;
+            } else if (Paths::hasInternalError(path)) {
+                hasError = true;
+            } else {
+                std::unique_ptr<TestCase, decltype(&TestCase_free)> ktestData{
+                    TC_fromFile(path.c_str()), TestCase_free
+                };
+                if (ktestData == nullptr) {
+                    LOG_S(WARNING) << "Unable to open .ktestjson file";
+                    continue;
                 }
-            }
-        }
-        if (hasTimeout) {
-            std::string message = StringUtils::stringFormat(
-                "Some tests for function '%s' were skipped, as execution of function is "
-                "out of timeout.",
-                method.methodName);
-            tests.commentBlocks.emplace_back(std::move(message));
-        }
-        if (hasError) {
-            std::string message = StringUtils::stringFormat(
-                "Some tests for function '%s' were skipped, as execution of function leads "
-                "KLEE to the internal error. See console log for more details.",
-                method.methodName);
-            tests.commentBlocks.emplace_back(std::move(message));
-        }
+                UTBotKTest::Status status = Paths::hasError(path) ? UTBotKTest::Status::FAILED
+                                                                  : UTBotKTest::Status::SUCCESS;
+                std::vector<ConcretizedObject> kTestObjects(
+                    ktestData->objects, ktestData->objects + ktestData->n_objects);
 
-        writeKleeStats(kleeOut);
+                std::vector<UTBotKTestObject> objects = CollectionUtils::transform(
+                    kTestObjects, [](const ConcretizedObject &kTestObject) {
+                        return UTBotKTestObject{ kTestObject };
+                    });
 
-        if (!CollectionUtils::containsKey(ktestChunk, method) || ktestChunk.at(method).empty()) {
-            tests.commentBlocks.emplace_back(StringUtils::stringFormat(
-                "Tests for %s were not generated. Maybe the function is too complex.",
-                method.methodName));
+                ktestChunk[method].emplace_back(objects, status);
+            }
         }
     }
+    if (hasTimeout) {
+        std::string message = StringUtils::stringFormat(
+            "Some tests for function '%s' were skipped, as execution of function is "
+            "out of timeout.",
+            method.methodName);
+        tests.commentBlocks.emplace_back(std::move(message));
+    }
+    if (hasError) {
+        std::string message = StringUtils::stringFormat(
+            "Some tests for function '%s' were skipped, as execution of function leads "
+            "KLEE to the internal error. See console log for more details.",
+            method.methodName);
+        tests.commentBlocks.emplace_back(std::move(message));
+    }
+
+    if (!CollectionUtils::containsKey(ktestChunk, method) || ktestChunk.at(method).empty()) {
+        tests.commentBlocks.emplace_back(StringUtils::stringFormat(
+            "Tests for %s were not generated. Maybe the function is too complex.",
+            method.methodName));
+    }
 }
 
-void KleeRunner::processBatchWithoutInteractive(MethodKtests &ktestChunk,
-                                                const TestMethod &testMethod,
-                                                Tests &tests) {
-    if (!tests.isFilePresentedInArtifact) {
+void KleeRunner::processBatchWithoutInteractive(const std::vector<tests::TestMethod> &testMethods,
+                                                tests::Tests &tests,
+                                                std::vector<tests::MethodKtests> &ktests) {
+    if (!tests.isFilePresentedInArtifact || testMethods.empty()) {
         return;
     }
-    if (testMethod.sourceFilePath != tests.sourceFilePath) {
-        std::string message = StringUtils::stringFormat(
-                "While generating tests for source file: %s tried to generate tests for method %s "
-                "from another source file: %s. This can cause invalid generation.\n",
-                tests.sourceFilePath, testMethod.methodName, testMethod.sourceFilePath);
-        LOG_S(WARNING) << message;
-    }
+
+    for (const auto &testMethod : testMethods) {
+        if (testMethod.sourceFilePath != tests.sourceFilePath) {
+            std::string message = StringUtils::stringFormat(
+                    "While generating tests for source file: %s tried to generate tests for method %s "
+                    "from another source file: %s. This can cause invalid generation.\n",
+                    tests.sourceFilePath, testMethod.methodName, testMethod.sourceFilePath);
+            LOG_S(WARNING) << message;
+        }
 
     std::string entryPoint = KleeUtils::entryPointFunction(tests, testMethod.methodName, true);
     std::string entryPointFlag = StringUtils::stringFormat("--entry-point=%s", entryPoint);
@@ -219,25 +237,28 @@ void KleeRunner::processBatchWithoutInteractive(MethodKtests &ktestChunk,
     argvData.emplace_back("--sym-stdin");
     argvData.emplace_back(std::to_string(types::Type::symStdinSize));
 
-    {
-        std::vector<char *> cargv, cenvp;
-        std::vector<std::string> tmp;
-        ExecUtils::toCArgumentsPtr(argvData, tmp, cargv, cenvp, false);
-        LOG_S(DEBUG) << "Klee command :: " + StringUtils::joinWith(argvData, " ");
-        MEASURE_FUNCTION_EXECUTION_TIME
+        {
+            std::vector<char *> cargv, cenvp;
+            std::vector<std::string> tmp;
+            ExecUtils::toCArgumentsPtr(argvData, tmp, cargv, cenvp, false);
+            LOG_S(DEBUG) << "Klee command :: " + StringUtils::joinWith(argvData, " ");
+            MEASURE_FUNCTION_EXECUTION_TIME
 
-        RunKleeTask task(cargv.size(), cargv.data(), settingsContext.timeoutPerFunction);
-        ExecUtils::ExecutionResult result __attribute__((unused)) = task.run();
-        ExecUtils::throwIfCancelled();
+            RunKleeTask task(cargv.size(), cargv.data(), settingsContext.timeoutPerFunction);
+            ExecUtils::ExecutionResult result __attribute__((unused)) = task.run();
+            ExecUtils::throwIfCancelled();
 
-        processMethod(ktestChunk, tests, kleeOut, testMethod);
+            MethodKtests ktestChunk;
+            processMethod(ktestChunk, tests, kleeOut, testMethod);
+            ktests.push_back(ktestChunk);
+        }
     }
 }
 
 void KleeRunner::processBatchWithInteractive(const std::vector<tests::TestMethod> &testMethods,
                                              tests::Tests &tests,
                                              std::vector<tests::MethodKtests> &ktests) {
-    if (!tests.isFilePresentedInArtifact) {
+    if (!tests.isFilePresentedInArtifact || testMethods.empty()) {
         return;
     }
 

server/src/KleeRunner.h

@@ -6,10 +6,12 @@
 #include "SettingsContext.h"
 #include "Tests.h"
 #include "streams/tests/TestsWriter.h"
+#include "utils/stats/KleeStats.h"
 
 #include <grpcpp/grpcpp.h>
 
 #include <vector>
+#include <utils/stats/TestsGenerationStats.h>
 
 class KleeRunner {
 public:
@@ -27,24 +29,21 @@ class KleeRunner {
      * generated unit tests for each batch.
      * @throws ExecutionProcessException if a Clang call returns non-zero code.
      */
-    void runKlee(const std::vector<tests::TestMethod> &testMethods,
-                 tests::TestsMap &testsMap,
-                 const std::shared_ptr<KleeGenerator>& generator,
+    void runKlee(const std::vector<tests::TestMethod> &testMethods, tests::TestsMap &testsMap,
+                 const std::shared_ptr<KleeGenerator> &generator,
                  const std::unordered_map<std::string, types::Type> &methodNameToReturnTypeMap,
-                 const std::shared_ptr<LineInfo> &lineInfo,
-                 TestsWriter *testsWriter,
-                 bool isBatched,
-                 bool interactiveMode);
+                 const std::shared_ptr<LineInfo> &lineInfo, TestsWriter *testsWriter, bool isBatched,
+                 bool interactiveMode,
+                 StatsUtils::TestsGenerationStatsFileMap &generationStats);
 
 private:
     const utbot::ProjectContext projectContext;
     const utbot::SettingsContext settingsContext;
     fs::path projectTmpPath;
 
-
-    void processBatchWithoutInteractive(tests::MethodKtests &ktestChunk,
-                                        const tests::TestMethod &testMethod,
-                                        tests::Tests &tests);
+    void processBatchWithoutInteractive(const std::vector<tests::TestMethod> &testMethods,
+                                        tests::Tests &tests,
+                                        std::vector<tests::MethodKtests> &ktests);
 
     void processBatchWithInteractive(const std::vector<tests::TestMethod> &testMethods,
                                      tests::Tests &tests,

server/src/Paths.cpp

@@ -153,15 +153,20 @@ namespace Paths {
                            [&path](auto const &suffix) { return errorFileExists(path, suffix); });
     }
 
+    fs::path kleeOutDirForFilePath(const utbot::ProjectContext &projectContext, const fs::path &projectTmpPath,
+                                   const fs::path &filePath) {
+        fs::path kleeOutDir = getKleeOutDir(projectTmpPath);
+        fs::path relative = (fs::relative(addOrigExtensionAsSuffixAndAddNew(filePath, ""), projectContext.projectPath));
+        return kleeOutDir / relative;
+    }
+
     fs::path kleeOutDirForEntrypoints(const utbot::ProjectContext &projectContext, const fs::path &projectTmpPath,
                                       const fs::path &srcFilePath, const std::string &methodName) {
-        fs::path kleeOutDir = getKleeOutDir(projectTmpPath);
-        fs::path relative = (fs::relative(addOrigExtensionAsSuffixAndAddNew(srcFilePath, ""),
-                                          projectContext.projectPath));
         if (!methodName.empty()) {
-            return kleeOutDir / relative / ("klee_out_" + methodName);
+            return kleeOutDirForFilePath(projectContext, projectTmpPath, srcFilePath) / ("klee_out_" + methodName);
         }
-        return kleeOutDir / relative / ("klee_out_" + srcFilePath.filename().stem().string());
+        return kleeOutDirForFilePath(projectContext, projectTmpPath, srcFilePath) /
+               ("klee_out_" + srcFilePath.filename().stem().string());
     }
 
     //endregion

server/src/Paths.h

@@ -238,6 +238,9 @@ namespace Paths {
 
     bool hasError(fs::path const &path);
 
+    fs::path kleeOutDirForFilePath(const utbot::ProjectContext &projectContext, const fs::path &projectTmpPath,
+                                   const fs::path &filePath);
+
     fs::path kleeOutDirForEntrypoints(const utbot::ProjectContext &projectContext, const fs::path &projectTmpPath,
                                       const fs::path &srcFilePath, const std::string &methodName = "");
 
@@ -411,6 +414,15 @@ namespace Paths {
 
     //endregion
 
+    //region stats
+    inline fs::path getGenerationStatsCSVPath(const utbot::ProjectContext &projectContext) {
+        return projectContext.resultsDirPath / "generation-stats.csv";
+    }
+    inline fs::path getExecutionStatsCSVPath(const utbot::ProjectContext &projectContext) {
+        return projectContext.resultsDirPath / "execution-stats.csv";
+    }
+    //endregion
+
     bool isHeadersEqual(const fs::path &srcPath, const fs::path &headerPath);
 }