add statistics report for coverage and results

Lana243 · Lana243 · commit bb7af9000bcc · 2022-06-06T12:41:23.000+03:00
diff --git a/server/proto/testgen.proto b/server/proto/testgen.proto
@@ -188,6 +188,7 @@ message TestResultObject {
     string testname = 2;
     TestStatus status = 3;
     string output = 4;
+    uint32 executionTimeMs = 5;
 }
 
 message CoverageAndResultsResponse {
diff --git a/server/src/coverage/Coverage.cpp b/server/src/coverage/Coverage.cpp
@@ -4,10 +4,10 @@
 
 #include "Coverage.h"
 
-int Coverage::TestStatusMap::getNumberOfTests() {
+int Coverage::TestResultMap::getNumberOfTests() {
     int cnt = 0;
-    for (auto const &[fileName, testsStatus] : *this) {
-        cnt += testsStatus.size();
+    for (auto const &[fileName, testsResult] : *this) {
+        cnt += testsResult.size();
     }
     return cnt;
 }
diff --git a/server/src/coverage/Coverage.h b/server/src/coverage/Coverage.h
@@ -41,11 +41,11 @@ namespace Coverage {
     class FileTestsStatus : public std::unordered_map<std::string, testsgen::TestStatus> {
     };
 
-    class TestStatusMap : public CollectionUtils::MapFileTo<FileTestsStatus> {
+    using FileTestsResult = std::unordered_map<std::string, testsgen::TestResultObject>;
+    class TestResultMap : public CollectionUtils::MapFileTo<FileTestsResult> {
     public:
         int getNumberOfTests();
     };
-
 }
 
 #endif //UNITTESTBOT_COVERAGE_H
diff --git a/server/src/coverage/CoverageAndResultsGenerator.cpp b/server/src/coverage/CoverageAndResultsGenerator.cpp
@@ -69,7 +69,7 @@ void CoverageAndResultsGenerator::showErrors() const {
         errorMessage = message;
     }
 
-    coverageAndResultsWriter->writeResponse(testStatusMap, coverageMap, totals, errorMessage);
+    coverageAndResultsWriter->writeResponse(projectContext, testResultMap, coverageMap, totals, errorMessage);
 }
 
 Coverage::CoverageMap const &CoverageAndResultsGenerator::getCoverageMap() {
@@ -87,7 +87,7 @@ void CoverageAndResultsGenerator::collectCoverage() {
     }
     std::vector<ShellExecTask> coverageCommands = coverageTool->getCoverageCommands(
         CollectionUtils::filterToVector(testsToLaunch, [this](const UnitTest &testToLaunch) {
-            return testStatusMap[testToLaunch.testFilePath][testToLaunch.testname] !=
+            return testResultMap[testToLaunch.testFilePath][testToLaunch.testname].status() !=
                    testsgen::TEST_INTERRUPTED;
         }));
     if (coverageCommands.empty()) {
diff --git a/server/src/coverage/TestRunner.cpp b/server/src/coverage/TestRunner.cpp
@@ -11,6 +11,7 @@
 #include "utils/StringUtils.h"
 
 #include "loguru.h"
+#include <regex>
 
 using grpc::ServerWriter;
 using grpc::Status;
@@ -139,11 +140,15 @@ grpc::Status TestRunner::runTests(bool withCoverage, const std::optional<std::ch
                                   auto const &[unitTest, buildCommand, runCommand] =
                                       buildRunCommand;
                                   try {
-                                      auto status = runTest(runCommand, testTimeout);
-                                      testStatusMap[unitTest.testFilePath][unitTest.testname] = status;
+                                      auto status = runTest(buildRunCommand, testTimeout);
+                                      testResultMap[unitTest.testFilePath][unitTest.testname] = status;
                                       ExecUtils::throwIfCancelled();
                                   } catch (ExecutionProcessException const &e) {
-                                      testStatusMap[unitTest.testFilePath][unitTest.testname] = testsgen::TEST_FAILED;
+                                      testsgen::TestResultObject testRes;
+                                      testRes.set_testfilepath(unitTest.testFilePath);
+                                      testRes.set_testname(unitTest.testname);
+                                      testRes.set_status(testsgen::TEST_FAILED);
+                                      testResultMap[unitTest.testFilePath][unitTest.testname] = testRes;
                                       exceptions.emplace_back(e);
                                   }
                               });
@@ -191,23 +196,42 @@ size_t TestRunner::buildTests(const utbot::ProjectContext& projectContext, const
     return fail_count;
 }
 
-testsgen::TestStatus TestRunner::runTest(const MakefileUtils::MakefileCommand &command, const std::optional <std::chrono::seconds> &testTimeout) {
-    auto res = command.run(projectContext.buildDir, true, true, testTimeout);
+testsgen::TestResultObject TestRunner::runTest(const BuildRunCommand &command, const std::optional <std::chrono::seconds> &testTimeout) {
+    auto res = command.runCommand.run(projectContext.buildDir, true, true, testTimeout);
     GTestLogger::log(res.output);
+
+    std::smatch match;
+    std::regex expr(R"(\[==========\] 1 test from 1 test suite ran. \(\d+ ms total\))");
+    std::regex_search(res.output, match, expr);
+    uint32_t executionTimeMs = 0;
+    if (!match.empty()) {
+        std::string strMatch = res.output.substr(match.position(0), match.length(0));
+        std::string strExecutionTime = StringUtils::splitByWhitespaces(strMatch)[8];
+        executionTimeMs = std::stoul(strExecutionTime.substr(1));
+    }
+
+    testsgen::TestResultObject testRes;
+    testRes.set_testfilepath(command.unitTest.testFilePath);
+    testRes.set_testname(command.unitTest.testname);
+    testRes.set_executiontimems(executionTimeMs);
     if (StringUtils::contains(res.output, "[  PASSED  ] 1 test")) {
-        return testsgen::TEST_PASSED;
+        testRes.set_status(testsgen::TEST_PASSED);
+        return testRes;
     }
     if (StringUtils::contains(res.output, "[  FAILED  ] 1 test")) {
-        return testsgen::TEST_FAILED;
+        testRes.set_status(testsgen::TEST_FAILED);
+        return testRes;
     }
     if (BaseForkTask::wasInterrupted(res.status)) {
-        return testsgen::TEST_INTERRUPTED;
+        testRes.set_status(testsgen::TEST_INTERRUPTED);
+        return testRes;
     }
-    return testsgen::TEST_DEATH;
+    testRes.set_status(testsgen::TEST_DEATH);
+    return testRes;
 }
 
-const Coverage::TestStatusMap &TestRunner::getTestStatusMap() const {
-    return testStatusMap;
+const Coverage::TestResultMap &TestRunner::getTestResultMap() const {
+    return testResultMap;
 }
 
 bool TestRunner::hasExceptions() const {
diff --git a/server/src/coverage/TestRunner.h b/server/src/coverage/TestRunner.h
@@ -28,7 +28,7 @@ class TestRunner {
 
     std::unique_ptr<CoverageTool> coverageTool{};
     std::vector<UnitTest> testsToLaunch{};
-    Coverage::TestStatusMap testStatusMap{};
+    Coverage::TestResultMap testResultMap{};
 
     std::vector<ExecutionProcessException> exceptions;
 
@@ -52,7 +52,7 @@ class TestRunner {
 
     std::vector<UnitTest> getTestsToLaunch();
 
-    const Coverage::TestStatusMap &getTestStatusMap() const;
+    const Coverage::TestResultMap &getTestResultMap() const;
 
     bool hasExceptions() const;
 
@@ -72,8 +72,8 @@ class TestRunner {
     std::vector<UnitTest> getTestsFromMakefile(const fs::path &makefile,
                                                const fs::path &testFilePath);
 
-    testsgen::TestStatus runTest(const MakefileUtils::MakefileCommand &command,
-                                 const std::optional<std::chrono::seconds> &testTimeout);
+    testsgen::TestResultObject runTest(const BuildRunCommand &command,
+                                       const std::optional<std::chrono::seconds> &testTimeout);
 
     ServerCoverageAndResultsWriter writer{ nullptr };
 
diff --git a/server/src/printers/CoverageAndResultsStatisticsPrinter.cpp b/server/src/printers/CoverageAndResultsStatisticsPrinter.cpp
@@ -0,0 +1,64 @@
+#include <utils/FileSystemUtils.h>
+
+#include "CoverageAndResultsStatisticsPrinter.h"
+#include "utils/StringUtils.h"
+#include "utils/CollectionUtils.h"
+#include "Paths.h"
+
+void printer::CoverageAndResultsStatisticsPrinter::write(const utbot::ProjectContext &projectContext,
+                                                         const Coverage::TestResultMap &testsResultMap,
+                                                         const Coverage::CoverageMap &coverageMap) {
+    for (auto const &[testPath, testsResult]: testsResultMap) {
+        fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testPath);
+        Coverage::FileCoverage fileCoverage = CollectionUtils::getOrDefault(coverageMap,
+                                                                            sourcePath,
+                                                                            Coverage::FileCoverage());
+        insert({sourcePath, FileCoverageAndResultsStatistics(testsResult, fileCoverage)});
+    }
+    std::vector<std::string> metricNames = {
+            "filename", "executionTime (ms)",
+            "totalTestsNumber", "passedTestsNumber", "failedTestsNumber", "deathTestsNumber", "interruptedTestsNumber",
+            "totalLinesNumber", "coveredLinesNumber", "lineCoverageRatio (%)"
+    };
+    std::string header = StringUtils::joinWith(metricNames, ",");
+    std::stringstream ss;
+    ss << header << '\n';
+    for (auto const &[sourcePath, statistics]: *this) {
+        ss << fs::relative(sourcePath, projectContext.projectPath) << ",";
+        ss << statistics.totalExecutionTimeMs << ',';
+        ss << statistics.totalTestsNum << ",";
+        ss << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_PASSED, 0u)
+           << ',';
+        ss << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_FAILED, 0u)
+           << ',';
+        ss << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_DEATH, 0u) << ',';
+        ss << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_INTERRUPTED, 0u)
+           << ',';
+
+        uint32_t coveredLinesNum =
+                statistics.coverage.fullCoverageLines.size() + statistics.coverage.partialCoverageLines.size();
+        uint32_t totalLinesNum = coveredLinesNum + statistics.coverage.noCoverageLines.size();
+        double lineCoverageRatio = 0;
+        if (totalLinesNum != 0) {
+            lineCoverageRatio = 100.0 * coveredLinesNum / totalLinesNum;
+        }
+        ss << totalLinesNum << ',' << coveredLinesNum << ',';
+        ss << std::fixed << lineCoverageRatio << '\n';
+    }
+    fs::path resultsFilePath = resultsDirectory / (TimeUtils::getDate() + "-stats.csv");
+    FileSystemUtils::writeToFile(resultsFilePath, ss.str());
+    LOG_S(INFO) << StringUtils::stringFormat("See statistics info here: %s", resultsFilePath);
+}
+
+printer::FileCoverageAndResultsStatistics::FileCoverageAndResultsStatistics(
+        const Coverage::FileTestsResult &testsResult,
+        Coverage::FileCoverage fileCoverage) {
+    coverage = std::move(fileCoverage);
+    totalTestsNum = 0;
+    totalExecutionTimeMs = 0;
+    for (const auto &[_, testResult]: testsResult) {
+        totalTestsNum++;
+        testsWithStatusNum[testResult.status()]++;
+        totalExecutionTimeMs += testResult.executiontimems();
+    }
+}
diff --git a/server/src/printers/CoverageAndResultsStatisticsPrinter.h b/server/src/printers/CoverageAndResultsStatisticsPrinter.h
@@ -0,0 +1,39 @@
+#ifndef UNITTESTBOT_COVERAGEANDRESULTSSTATISTICSPRINTER_H
+#define UNITTESTBOT_COVERAGEANDRESULTSSTATISTICSPRINTER_H
+
+#include <chrono>
+#include <unordered_map>
+#include <utility>
+#include <protobuf/testgen.grpc.pb.h>
+#include <ProjectContext.h>
+#include "coverage/Coverage.h"
+#include "loguru.h"
+
+namespace printer {
+    class FileCoverageAndResultsStatistics {
+    public:
+        FileCoverageAndResultsStatistics(const Coverage::FileTestsResult &testsResult, Coverage::FileCoverage coverage);
+
+        // Time statistics
+        uint32_t totalExecutionTimeMs;
+
+        // Test runs statistics
+        uint32_t totalTestsNum;
+        std::unordered_map<testsgen::TestStatus, uint32_t> testsWithStatusNum;
+
+        // Coverage
+        Coverage::FileCoverage coverage;
+    };
+
+    class CoverageAndResultsStatisticsPrinter : CollectionUtils::MapFileTo<FileCoverageAndResultsStatistics> {
+    public:
+        explicit CoverageAndResultsStatisticsPrinter(fs::path resultsDirectory) : resultsDirectory(
+                std::move(resultsDirectory)) {};
+        void write(const utbot::ProjectContext &, const Coverage::TestResultMap &, const Coverage::CoverageMap &);
+
+    private:
+        fs::path resultsDirectory;
+    };
+}
+
+#endif //UNITTESTBOT_COVERAGEANDRESULTSSTATISTICSPRINTER_H
diff --git a/server/src/streams/coverage/CLICoverageAndResultsWriter.cpp b/server/src/streams/coverage/CLICoverageAndResultsWriter.cpp
@@ -7,6 +7,7 @@
 
 #include "utils/FileSystemUtils.h"
 #include "utils/TimeUtils.h"
+#include <printers/CoverageAndResultsStatisticsPrinter.h>
 
 #include "loguru.h"
 
@@ -27,17 +28,18 @@ std::string statusToString(testsgen::TestStatus status) {
     return it == description.end() ? "UNKNOWN" : it->second;
 }
 
-void CLICoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap &testsStatusMap,
+void CLICoverageAndResultsWriter::writeResponse(const utbot::ProjectContext &projectContext,
+                                                const Coverage::TestResultMap &testsResultMap,
                                                 const Coverage::CoverageMap &coverageMap,
                                                 const nlohmann::json &totals,
                                                 std::optional<std::string> errorMessage) {
     std::stringstream ss;
 
     ss << "Test results summary." << std::endl;
-    for (const auto &[filepath, fileTestsStatus] : testsStatusMap) {
+    for (const auto &[filepath, fileTestsResult] : testsResultMap) {
         ss << "==== Tests in " << filepath << std::endl;
-        for (const auto &[testName, status] : fileTestsStatus) {
-            ss << "======== " << testName << " -> " << statusToString(status) << std::endl;
+        for (const auto &[testName, result] : fileTestsResult) {
+            ss << "======== " << testName << " -> " << statusToString(result.status()) << std::endl;
         }
     }
 
@@ -60,4 +62,7 @@ void CLICoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap &t
     fs::path resultsFilePath = resultsDirectory / (TimeUtils::getDate() + ".log");
     FileSystemUtils::writeToFile(resultsFilePath, ss.str());
     LOG_S(INFO) << ss.str();
+    printer::CoverageAndResultsStatisticsPrinter statsPrinter = printer::CoverageAndResultsStatisticsPrinter(
+            resultsDirectory);
+    statsPrinter.write(projectContext, testsResultMap, coverageMap);
 }
diff --git a/server/src/streams/coverage/CLICoverageAndResultsWriter.h b/server/src/streams/coverage/CLICoverageAndResultsWriter.h
@@ -12,7 +12,8 @@ class CLICoverageAndResultsWriter : public CoverageAndResultsWriter {
 public:
     explicit CLICoverageAndResultsWriter(const fs::path &resultsDirectory);
 
-    virtual void writeResponse(const Coverage::TestStatusMap &testsStatusMap,
+    virtual void writeResponse(const utbot::ProjectContext &projectContext,
+                               const Coverage::TestResultMap &testsResultMap,
                                const Coverage::CoverageMap &coverageMap,
                                const nlohmann::json &totals,
                                std::optional<std::string> errorMessage) override;
diff --git a/server/src/streams/coverage/CoverageAndResultsWriter.h b/server/src/streams/coverage/CoverageAndResultsWriter.h
@@ -19,7 +19,8 @@ class CoverageAndResultsWriter : public utbot::ServerWriter<testsgen::CoverageAn
 
     explicit CoverageAndResultsWriter(grpc::ServerWriter<testsgen::CoverageAndResultsResponse> *writer);
 
-    virtual void writeResponse(const Coverage::TestStatusMap &testsStatusMap,
+    virtual void writeResponse(const utbot::ProjectContext &projectContext,
+                               const Coverage::TestResultMap &testsResultMap,
                                const Coverage::CoverageMap &coverageMap,
                                const nlohmann::json &totals,
                                std::optional<std::string> errorMessage) = 0;
diff --git a/server/src/streams/coverage/ServerCoverageAndResultsWriter.cpp b/server/src/streams/coverage/ServerCoverageAndResultsWriter.cpp
@@ -14,9 +14,10 @@ ServerCoverageAndResultsWriter::ServerCoverageAndResultsWriter(
     : CoverageAndResultsWriter(writer) {
 }
 
-void ServerCoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap &testsStatusMap,
+void ServerCoverageAndResultsWriter::writeResponse(const utbot::ProjectContext &projectContext,
+                                                   const Coverage::TestResultMap &testsResultMap,
                                                    const Coverage::CoverageMap &coverageMap,
-                                                  const nlohmann::json &totals,
+                                                   const nlohmann::json &totals,
                                                    std::optional<std::string> errorMessage) {
     if (!hasStream()) {
         return;
@@ -25,13 +26,10 @@ void ServerCoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap
 
     testsgen::CoverageAndResultsResponse response;
 
-    for (const auto &[filepath, fileTestsStatus] : testsStatusMap) {
-        for (const auto &[testname, status] : fileTestsStatus) {
+    for (const auto &[filepath, fileTestsResult] : testsResultMap) {
+        for (const auto &[testname, result] : fileTestsResult) {
             auto testResultsGrpc = response.add_testrunresults();
-            testResultsGrpc->set_testfilepath(filepath);
-            testResultsGrpc->set_testname(testname);
-            testResultsGrpc->set_status(status);
-            testResultsGrpc->set_output("");
+            *testResultsGrpc = result;
         }
     }
 
diff --git a/server/src/streams/coverage/ServerCoverageAndResultsWriter.h b/server/src/streams/coverage/ServerCoverageAndResultsWriter.h
@@ -20,7 +20,8 @@ class ServerCoverageAndResultsWriter : public CoverageAndResultsWriter {
     explicit ServerCoverageAndResultsWriter(
         grpc::ServerWriter<testsgen::CoverageAndResultsResponse> *writer);
 
-    virtual void writeResponse(const Coverage::TestStatusMap &testsStatusMap,
+    virtual void writeResponse(const utbot::ProjectContext &projectContext,
+                               const Coverage::TestResultMap &testResultMap,
                                const Coverage::CoverageMap &coverageMap,
                                const nlohmann::json &totals,
                                std::optional<std::string> errorMessage) override;
diff --git a/server/test/framework/Server_Tests.cpp b/server/test/framework/Server_Tests.cpp
diff --git a/server/test/framework/Syntax_Tests.cpp b/server/test/framework/Syntax_Tests.cpp
diff --git a/server/test/framework/TestUtils.cpp b/server/test/framework/TestUtils.cpp
diff --git a/server/test/framework/TestUtils.h b/server/test/framework/TestUtils.h

Original file line number	Diff line number	Diff line change
`@@ -188,6 +188,7 @@ message TestResultObject {`
`188`	`188`	`string testname = 2;`
`189`	`189`	`TestStatus status = 3;`
`190`	`190`	`string output = 4;`
	`191`	`+ uint32 executionTimeMs = 5;`
`191`	`192`	`}`
`192`	`193`
`193`	`194`	`message CoverageAndResultsResponse {`
Original file line number	Diff line number	Diff line change
`@@ -4,10 +4,10 @@`
`4`	`4`
`5`	`5`	`#include "Coverage.h"`
`6`	`6`
`7`		`-int Coverage::TestStatusMap::getNumberOfTests() {`
	`7`	`+int Coverage::TestResultMap::getNumberOfTests() {`
`8`	`8`	`int cnt = 0;`
`9`		`- for (auto const &[fileName, testsStatus] : *this) {`
`10`		`- cnt += testsStatus.size();`
	`9`	`+ for (auto const &[fileName, testsResult] : *this) {`
	`10`	`+ cnt += testsResult.size();`
`11`	`11`	`}`
`12`	`12`	`return cnt;`
`13`	`13`	`}`
Original file line number	Diff line number	Diff line change
`@@ -69,7 +69,7 @@ void CoverageAndResultsGenerator::showErrors() const {`
`69`	`69`	`errorMessage = message;`
`70`	`70`	`}`
`71`	`71`
`72`		`- coverageAndResultsWriter->writeResponse(testStatusMap, coverageMap, totals, errorMessage);`
	`72`	`+ coverageAndResultsWriter->writeResponse(projectContext, testResultMap, coverageMap, totals, errorMessage);`
`73`	`73`	`}`
`74`	`74`
`75`	`75`	`Coverage::CoverageMap const &CoverageAndResultsGenerator::getCoverageMap() {`
`@@ -87,7 +87,7 @@ void CoverageAndResultsGenerator::collectCoverage() {`
`87`	`87`	`}`
`88`	`88`	`std::vector<ShellExecTask> coverageCommands = coverageTool->getCoverageCommands(`
`89`	`89`	`CollectionUtils::filterToVector(testsToLaunch, [this](const UnitTest &testToLaunch) {`
`90`		`- return testStatusMap[testToLaunch.testFilePath][testToLaunch.testname] !=`
	`90`	`+ return testResultMap[testToLaunch.testFilePath][testToLaunch.testname].status() !=`
`91`	`91`	`testsgen::TEST_INTERRUPTED;`
`92`	`92`	`}));`
`93`	`93`	`if (coverageCommands.empty()) {`