Add coverage and results statistics report (#261)

 add coverage and results statistics report

Author: Lana243

server/proto/testgen.proto

@@ -3,6 +3,7 @@ syntax = "proto3";
 package testsgen;
 
 import "util.proto";
+import "google/protobuf/duration.proto";
 
 service TestsGenService {
     rpc Handshake(DummyRequest) returns(DummyResponse) {}
@@ -188,6 +189,7 @@ message TestResultObject {
     string testname = 2;
     TestStatus status = 3;
     string output = 4;
+    google.protobuf.Duration executionTime = 5;
 }
 
 message CoverageAndResultsResponse {

server/src/KleeGenerator.cpp

@@ -57,7 +57,7 @@ KleeGenerator::buildByCDb(const CollectionUtils::MapFileTo<fs::path> &filesToBui
     fs::path makefile = projectTmpPath / "GenerationCompileMakefile.mk";
     FileSystemUtils::writeToFile(makefile, makefilePrinter.ss.str());
 
-    auto command = MakefileUtils::makefileCommand(projectContext, makefile, "all");
+    auto command = MakefileUtils::MakefileCommand(projectContext, makefile, "all");
     ExecUtils::ExecutionResult res = command.run();
     if (res.status != 0) {
         LOG_S(ERROR) << StringUtils::stringFormat("Make for \"%s\" failed.\nCommand: \"%s\"\n%s\n",
@@ -205,7 +205,7 @@ Result<fs::path> KleeGenerator::defaultBuild(const fs::path &hintPath,
     fs::path makefile = projectTmpPath / "BCForKLEE.mk";
     FileSystemUtils::writeToFile(makefile, makefilePrinter.ss.str());
 
-    auto makefileCommand = MakefileUtils::makefileCommand(projectContext, makefile, "build");
+    auto makefileCommand = MakefileUtils::MakefileCommand(projectContext, makefile, "build");
     auto [out, status, _] = makefileCommand.run();
     if (status != 0) {
         LOG_S(ERROR) << "Compilation for " << sourceFilePath << " failed.\n"

server/src/Paths.cpp

@@ -183,6 +183,9 @@ namespace Paths {
     fs::path getArtifactsRootDir(const utbot::ProjectContext &projectContext) {
         return projectContext.buildDir / "utbot";
     }
+    fs::path getGTestResultsJsonPath(const utbot::ProjectContext &projectContext) {
+        return getArtifactsRootDir(projectContext) / "gtest-results.json";
+    }
     fs::path getFlagsDir(const utbot::ProjectContext &projectContext) {
         return getArtifactsRootDir(projectContext) / "flags";
     }

server/src/Paths.h

@@ -273,6 +273,8 @@ namespace Paths {
 
     fs::path getArtifactsRootDir(const utbot::ProjectContext &projectContext);
 
+    fs::path getGTestResultsJsonPath(const utbot::ProjectContext &projectContext);
+
     fs::path getFlagsDir(const utbot::ProjectContext &projectContext);
 
     fs::path getTestExecDir(const utbot::ProjectContext &projectContext);

server/src/building/Linker.cpp

@@ -364,7 +364,7 @@ Result<Linker::LinkResult> Linker::link(const CollectionUtils::MapFileTo<fs::pat
     FileSystemUtils::writeToFile(linkMakefile, bitcodeLinkMakefilePrinter.ss.str());
 
     auto command =
-        MakefileUtils::makefileCommand(testGen.projectContext, linkMakefile, targetBitcode);
+        MakefileUtils::MakefileCommand(testGen.projectContext, linkMakefile, targetBitcode);
     auto [out, status, logFilePath] = command.run(testGen.serverBuildDir);
     if (status != 0) {
         std::string errorMessage =
@@ -477,7 +477,7 @@ Result<utbot::Void> Linker::linkWithStubsIfNeeded(const fs::path &linkMakefile,
         return errorMessage;
     }
 
-    auto command = MakefileUtils::makefileCommand(testGen.projectContext, linkMakefile, "all");
+    auto command = MakefileUtils::MakefileCommand(testGen.projectContext, linkMakefile, "all");
     auto [out, status, _] = command.run(testGen.serverBuildDir);
     if (status != 0) {
         std::string errorMessage =

server/src/coverage/Coverage.cpp

@@ -1,9 +1,9 @@
 #include "Coverage.h"
 
-int Coverage::TestStatusMap::getNumberOfTests() {
+int Coverage::TestResultMap::getNumberOfTests() {
     int cnt = 0;
-    for (auto const &[fileName, testsStatus] : *this) {
-        cnt += testsStatus.size();
+    for (auto const &[fileName, testsResult] : *this) {
+        cnt += testsResult.size();
     }
     return cnt;
 }

server/src/coverage/Coverage.h

@@ -4,6 +4,7 @@
 #include "utils/CollectionUtils.h"
 
 #include <protobuf/testgen.grpc.pb.h>
+#include <google/protobuf/util/time_util.h>
 
 #include <unordered_map>
 #include <vector>
@@ -37,11 +38,11 @@ namespace Coverage {
     class FileTestsStatus : public std::unordered_map<std::string, testsgen::TestStatus> {
     };
 
-    class TestStatusMap : public CollectionUtils::MapFileTo<FileTestsStatus> {
+    using FileTestsResult = std::unordered_map<std::string, testsgen::TestResultObject>;
+    class TestResultMap : public CollectionUtils::MapFileTo<FileTestsResult> {
     public:
         int getNumberOfTests();
     };
-
 }
 
 #endif //UNITTESTBOT_COVERAGE_H

server/src/coverage/CoverageAndResultsGenerator.cpp

@@ -37,15 +37,17 @@ grpc::Status CoverageAndResultsGenerator::generate(bool withCoverage,
         }
     } catch (CoverageGenerationException &e) {
         showErrors();
+        fs::remove(Paths::getGTestResultsJsonPath(projectContext));
         return Status(StatusCode::FAILED_PRECONDITION, e.what());
     } catch (ExecutionProcessException &e) {
         exceptions.emplace_back(e);
         showErrors();
+        fs::remove(Paths::getGTestResultsJsonPath(projectContext));
         return Status(StatusCode::FAILED_PRECONDITION, e.what());
     } catch (CancellationException &e) {
+        fs::remove(Paths::getGTestResultsJsonPath(projectContext));
         return Status::CANCELLED;
     }
-
     showErrors();
     return Status::OK;
 }
@@ -65,7 +67,7 @@ void CoverageAndResultsGenerator::showErrors() const {
         errorMessage = message;
     }
 
-    coverageAndResultsWriter->writeResponse(testStatusMap, coverageMap, totals, errorMessage);
+    coverageAndResultsWriter->writeResponse(projectContext, testResultMap, coverageMap, totals, errorMessage);
 }
 
 Coverage::CoverageMap const &CoverageAndResultsGenerator::getCoverageMap() {
@@ -83,7 +85,7 @@ void CoverageAndResultsGenerator::collectCoverage() {
     }
     std::vector<ShellExecTask> coverageCommands = coverageTool->getCoverageCommands(
         CollectionUtils::filterToVector(testsToLaunch, [this](const UnitTest &testToLaunch) {
-            return testStatusMap[testToLaunch.testFilePath][testToLaunch.testname] !=
+            return testResultMap[testToLaunch.testFilePath][testToLaunch.testname].status() !=
                    testsgen::TEST_INTERRUPTED;
         }));
     if (coverageCommands.empty()) {

server/src/coverage/CoverageTool.cpp

@@ -8,7 +8,8 @@
 
 using namespace CompilationUtils;
 
-CoverageTool::CoverageTool(ProgressWriter const *progressWriter) : progressWriter(progressWriter) {
+CoverageTool::CoverageTool(utbot::ProjectContext projectContext, ProgressWriter const *progressWriter) :
+        projectContext(std::move(projectContext)), progressWriter(progressWriter) {
 }
 
 std::unique_ptr<CoverageTool> getCoverageTool(const std::string &compileCommandsJsonPath,
@@ -29,6 +30,10 @@ std::unique_ptr<CoverageTool> getCoverageTool(const std::string &compileCommands
     }
 }
 
-std::string CoverageTool::getTestFilter(const UnitTest &unitTest) const {
-    return StringUtils::stringFormat("--gtest_filter=*.%s", unitTest.testname);
+std::string CoverageTool::getGTestFlags(const UnitTest &unitTest) const {
+    std::string gtestFilterFlag = StringUtils::stringFormat("\"--gtest_filter=*.%s\"", unitTest.testname);
+    std::string gtestOutputFlag = StringUtils::stringFormat("\"--gtest_output=json:%s\"",
+                                                            Paths::getGTestResultsJsonPath(projectContext));
+    std::vector<std::string> gtestFlagsList = { gtestFilterFlag, gtestOutputFlag };
+    return StringUtils::joinWith(gtestFlagsList, " ");
 }

server/src/coverage/CoverageTool.h

@@ -20,11 +20,12 @@ struct BuildRunCommand {
 class CoverageTool {
 protected:
     ProgressWriter const *progressWriter;
+    const utbot::ProjectContext projectContext;
 
-    [[nodiscard]] std::string getTestFilter(UnitTest const &unitTest) const;
+    [[nodiscard]] std::string getGTestFlags(const UnitTest &unitTest) const;
 
 public:
-    explicit CoverageTool(ProgressWriter const *progressWriter);
+    CoverageTool(utbot::ProjectContext projectContext, ProgressWriter const *progressWriter);
 
     [[nodiscard]] virtual std::vector<BuildRunCommand>
     getBuildRunCommands(const std::vector<UnitTest> &testsToLaunch, bool withCoverage) = 0;