Skip to content

Add generation statistics report #329

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Aug 2, 2022
Merged

Add generation statistics report #329

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e84c7f1
add generation statistics report
Lana243 Jul 18, 2022
3f3fa83
add tests
Lana243 Jul 19, 2022
3a8f934
small fix
Lana243 Jul 20, 2022
ed0dce6
fix code review comments
Lana243 Jul 27, 2022
47e18a3
[BUG] SARIF report is not recognized by github #352 (#353)
alexey-utkin Jul 27, 2022
9a0f202
Merge branch 'main' of https://github.com/UnitTestBot/UTBotCpp
Lana243 Jul 28, 2022
cfb6a32
fixed codestyle
Lana243 Aug 1, 2022
4a6700d
small fix
Lana243 Aug 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions server/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ file(GLOB ALL_SOURCES
"${PROJECT_SOURCE_DIR}/src/streams/tests/*"
"${PROJECT_SOURCE_DIR}/src/testgens/*"
"${PROJECT_SOURCE_DIR}/src/utils/*"
"${PROJECT_SOURCE_DIR}/src/utils/stats/*"
"${PROJECT_SOURCE_DIR}/src/stubs/*"
"${PROJECT_SOURCE_DIR}/src/visitors/*"
"${PROJECT_SOURCE_DIR}/src/commands/*"
Expand Down
1 change: 0 additions & 1 deletion server/proto/testgen.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@ message ProjectContext {
string projectPath = 2;
string testDirPath = 3;
string buildDirRelativePath = 4;
string resultsDirRelativePath = 5;
}

message SettingsContext {
Expand Down
228 changes: 115 additions & 113 deletions server/src/KleeRunner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
#include "utils/FileSystemUtils.h"
#include "utils/KleeUtils.h"
#include "utils/LogUtils.h"
#include "utils/stats/CSVReader.h"
#include "utils/stats/TestsGenerationStats.h"

#include "loguru.h"

Expand All @@ -18,6 +20,27 @@

using namespace tests;

namespace {
void clearUnusedData(const fs::path &kleeDir) {
fs::remove(kleeDir / "assembly.ll");
fs::remove(kleeDir / "run.istats");
}

StatsUtils::KleeStats writeKleeStats(const fs::path &kleeOut) {
ShellExecTask::ExecutionParameters kleeStatsParams("klee-stats",
{"--utbot-config", kleeOut.string(),
"--table-format=readable-csv"});
auto[out, status, _] = ShellExecTask::runShellCommandTask(kleeStatsParams);
if (status != 0) {
LOG_S(ERROR) << "klee-stats call failed:" << "\n" << out;
return {};
}
LOG_S(DEBUG) << "klee-stats report:" << '\n' << out;
std::stringstream ss(out);
return StatsUtils::KleeStats(ss);
}
}

KleeRunner::KleeRunner(utbot::ProjectContext projectContext,
utbot::SettingsContext settingsContext,
fs::path serverBuildDir)
Expand All @@ -32,7 +55,8 @@ void KleeRunner::runKlee(const std::vector<tests::TestMethod> &testMethods,
const std::shared_ptr<LineInfo> &lineInfo,
TestsWriter *testsWriter,
bool isBatched,
bool interactiveMode) {
bool interactiveMode,
StatsUtils::TestsGenerationStatsFileMap &generationStats) {
LOG_SCOPE_FUNCTION(DEBUG);

fs::path kleeOutDir = Paths::getKleeOutDir(projectTmpPath);
Expand Down Expand Up @@ -77,19 +101,14 @@ void KleeRunner::runKlee(const std::vector<tests::TestMethod> &testMethods,
LOG_S(MAX) << logStream.str();
}
if (interactiveMode) {
if (!batch.empty()) {
processBatchWithInteractive(batch, tests, ktests);
}
processBatchWithInteractive(batch, tests, ktests);
} else {
for (auto const &testMethod : batch) {
MethodKtests ktestChunk;
processBatchWithoutInteractive(ktestChunk, testMethod, tests);
ExecUtils::throwIfCancelled();
ktests.push_back(ktestChunk);
}
processBatchWithoutInteractive(batch, tests, ktests);
}
auto kleeStats = writeKleeStats(Paths::kleeOutDirForFilePath(projectContext, projectTmpPath, filePath));
generator->parseKTestsToFinalCode(tests, methodNameToReturnTypeMap, ktests, lineInfo,
settingsContext.verbose);
generationStats.addFileStats(kleeStats, tests);

sarif::sarifAddTestsToResults(projectContext, tests, sarifResults);
};
Expand All @@ -108,121 +127,101 @@ void KleeRunner::runKlee(const std::vector<tests::TestMethod> &testMethods,
std::move(prepareTotal));
}

namespace {
void clearUnusedData(const fs::path &kleeDir) {
fs::remove(kleeDir / "assembly.ll");
fs::remove(kleeDir / "run.istats");
}

void writeKleeStats(const fs::path &kleeOut) {
ShellExecTask::ExecutionParameters kleeStatsParams("klee-stats",
{ "--utbot-config", kleeOut.string() });
auto [out, status, _] = ShellExecTask::runShellCommandTask(kleeStatsParams);
if (status != 0) {
LOG_S(ERROR) << "klee-stats call failed:";
LOG_S(ERROR) << out;
} else {
LOG_S(DEBUG) << "klee-stats report:";
LOG_S(DEBUG) << '\n' << out;
}
}
}

static void processMethod(MethodKtests &ktestChunk,
tests::Tests &tests,
const fs::path &kleeOut,
const tests::TestMethod &method) {
if (fs::exists(kleeOut)) {
clearUnusedData(kleeOut);
bool hasTimeout = false;
bool hasError = false;
for (auto const &entry : fs::directory_iterator(kleeOut)) {
auto const &path = entry.path();
if (Paths::isKtestJson(path)) {
if (Paths::hasEarly(path)) {
hasTimeout = true;
} else if (Paths::hasInternalError(path)) {
hasError = true;
} else {
std::unique_ptr<TestCase, decltype(&TestCase_free)> ktestData{
TC_fromFile(path.c_str()), TestCase_free
};
if (ktestData == nullptr) {
LOG_S(WARNING) << "Unable to open .ktestjson file";
continue;
}

const std::vector<fs::path> &errorDescriptorFiles =
Paths::getErrorDescriptors(path);
if (!fs::exists(kleeOut)) {
return;
}

UTBotKTest::Status status = errorDescriptorFiles.empty()
? UTBotKTest::Status::SUCCESS
: UTBotKTest::Status::FAILED;
std::vector<ConcretizedObject> kTestObjects(
ktestData->objects, ktestData->objects + ktestData->n_objects);

std::vector<UTBotKTestObject> objects = CollectionUtils::transform(
kTestObjects, [](const ConcretizedObject &kTestObject) {
return UTBotKTestObject{ kTestObject };
});

std::vector<std::string> errorDescriptors = CollectionUtils::transform(
errorDescriptorFiles, [](const fs::path &errorFile) {
std::ifstream fileWithError(errorFile.c_str(), std::ios_base::in);
std::string content((std::istreambuf_iterator<char>(fileWithError)),
std::istreambuf_iterator<char>());

const std::string &errorId = errorFile.stem().extension().string();
if (!errorId.empty()) {
// skip leading dot
content += "\n" + sarif::ERROR_ID_KEY + ":" + errorId.substr(1);
}
return content;
});


ktestChunk[method].emplace_back(objects, status, errorDescriptors);
clearUnusedData(kleeOut);
bool hasTimeout = false;
bool hasError = false;
for (auto const &entry : fs::directory_iterator(kleeOut)) {
auto const &path = entry.path();
if (Paths::isKtestJson(path)) {
if (Paths::hasEarly(path)) {
hasTimeout = true;
} else if (Paths::hasInternalError(path)) {
hasError = true;
} else {
std::unique_ptr<TestCase, decltype(&TestCase_free)> ktestData{
TC_fromFile(path.c_str()), TestCase_free
};
if (ktestData == nullptr) {
LOG_S(WARNING) << "Unable to open .ktestjson file";
continue;
}
const std::vector<fs::path> &errorDescriptorFiles =
Paths::getErrorDescriptors(path);

UTBotKTest::Status status = errorDescriptorFiles.empty()
? UTBotKTest::Status::SUCCESS
: UTBotKTest::Status::FAILED;
std::vector<ConcretizedObject> kTestObjects(
ktestData->objects, ktestData->objects + ktestData->n_objects);

std::vector<UTBotKTestObject> objects = CollectionUtils::transform(
kTestObjects, [](const ConcretizedObject &kTestObject) {
return UTBotKTestObject{ kTestObject };
});

std::vector<std::string> errorDescriptors = CollectionUtils::transform(
errorDescriptorFiles, [](const fs::path &errorFile) {
std::ifstream fileWithError(errorFile.c_str(), std::ios_base::in);
std::string content((std::istreambuf_iterator<char>(fileWithError)),
std::istreambuf_iterator<char>());

const std::string &errorId = errorFile.stem().extension().string();
if (!errorId.empty()) {
// skip leading dot
content += "\n" + sarif::ERROR_ID_KEY + ":" + errorId.substr(1);
}
return content;
});

ktestChunk[method].emplace_back(objects, status, errorDescriptors);
}
}
if (hasTimeout) {
std::string message = StringUtils::stringFormat(
"Some tests for function '%s' were skipped, as execution of function is "
"out of timeout.",
method.methodName);
tests.commentBlocks.emplace_back(std::move(message));
}
if (hasError) {
std::string message = StringUtils::stringFormat(
"Some tests for function '%s' were skipped, as execution of function leads "
"KLEE to the internal error. See console log for more details.",
method.methodName);
tests.commentBlocks.emplace_back(std::move(message));
}

writeKleeStats(kleeOut);
}
if (hasTimeout) {
std::string message = StringUtils::stringFormat(
"Some tests for function '%s' were skipped, as execution of function is "
"out of timeout.",
method.methodName);
tests.commentBlocks.emplace_back(std::move(message));
}
if (hasError) {
std::string message = StringUtils::stringFormat(
"Some tests for function '%s' were skipped, as execution of function leads "
"KLEE to the internal error. See console log for more details.",
method.methodName);
tests.commentBlocks.emplace_back(std::move(message));
}

if (!CollectionUtils::containsKey(ktestChunk, method) || ktestChunk.at(method).empty()) {
tests.commentBlocks.emplace_back(StringUtils::stringFormat(
"Tests for %s were not generated. Maybe the function is too complex.",
method.methodName));
}
if (!CollectionUtils::containsKey(ktestChunk, method) || ktestChunk.at(method).empty()) {
tests.commentBlocks.emplace_back(StringUtils::stringFormat(
"Tests for %s were not generated. Maybe the function is too complex.",
method.methodName));
}
}

void KleeRunner::processBatchWithoutInteractive(MethodKtests &ktestChunk,
const TestMethod &testMethod,
Tests &tests) {
if (!tests.isFilePresentedInArtifact) {
void KleeRunner::processBatchWithoutInteractive(const std::vector<tests::TestMethod> &testMethods,
tests::Tests &tests,
std::vector<tests::MethodKtests> &ktests) {
if (!tests.isFilePresentedInArtifact || testMethods.empty()) {
return;
}
if (testMethod.sourceFilePath != tests.sourceFilePath) {
std::string message = StringUtils::stringFormat(

for (const auto &testMethod : testMethods) {
if (testMethod.sourceFilePath != tests.sourceFilePath) {
std::string message = StringUtils::stringFormat(
"While generating tests for source file: %s tried to generate tests for method %s "
"from another source file: %s. This can cause invalid generation.\n",
tests.sourceFilePath, testMethod.methodName, testMethod.sourceFilePath);
LOG_S(WARNING) << message;
}
LOG_S(WARNING) << message;
}

std::string entryPoint = KleeUtils::entryPointFunction(tests, testMethod.methodName, true);
std::string entryPointFlag = StringUtils::stringFormat("--entry-point=%s", entryPoint);
Expand Down Expand Up @@ -261,18 +260,21 @@ void KleeRunner::processBatchWithoutInteractive(MethodKtests &ktestChunk,
LOG_S(DEBUG) << "Klee command :: " + StringUtils::joinWith(argvData, " ");
MEASURE_FUNCTION_EXECUTION_TIME

RunKleeTask task(cargv.size(), cargv.data(), settingsContext.timeoutPerFunction);
ExecUtils::ExecutionResult result __attribute__((unused)) = task.run();
ExecUtils::throwIfCancelled();
RunKleeTask task(cargv.size(), cargv.data(), settingsContext.timeoutPerFunction);
ExecUtils::ExecutionResult result __attribute__((unused)) = task.run();
ExecUtils::throwIfCancelled();

processMethod(ktestChunk, tests, kleeOut, testMethod);
MethodKtests ktestChunk;
processMethod(ktestChunk, tests, kleeOut, testMethod);
ktests.push_back(ktestChunk);
}
}
}

void KleeRunner::processBatchWithInteractive(const std::vector<tests::TestMethod> &testMethods,
tests::Tests &tests,
std::vector<tests::MethodKtests> &ktests) {
if (!tests.isFilePresentedInArtifact) {
if (!tests.isFilePresentedInArtifact || testMethods.empty()) {
return;
}

Expand Down
21 changes: 10 additions & 11 deletions server/src/KleeRunner.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
#include "SettingsContext.h"
#include "Tests.h"
#include "streams/tests/TestsWriter.h"
#include "utils/stats/KleeStats.h"
#include "utils/stats/TestsGenerationStats.h"

#include <grpcpp/grpcpp.h>

Expand All @@ -27,24 +29,21 @@ class KleeRunner {
* generated unit tests for each batch.
* @throws ExecutionProcessException if a Clang call returns non-zero code.
*/
void runKlee(const std::vector<tests::TestMethod> &testMethods,
tests::TestsMap &testsMap,
const std::shared_ptr<KleeGenerator>& generator,
void runKlee(const std::vector<tests::TestMethod> &testMethods, tests::TestsMap &testsMap,
const std::shared_ptr<KleeGenerator> &generator,
const std::unordered_map<std::string, types::Type> &methodNameToReturnTypeMap,
const std::shared_ptr<LineInfo> &lineInfo,
TestsWriter *testsWriter,
bool isBatched,
bool interactiveMode);
const std::shared_ptr<LineInfo> &lineInfo, TestsWriter *testsWriter, bool isBatched,
bool interactiveMode,
StatsUtils::TestsGenerationStatsFileMap &generationStats);

private:
const utbot::ProjectContext projectContext;
const utbot::SettingsContext settingsContext;
fs::path projectTmpPath;


void processBatchWithoutInteractive(tests::MethodKtests &ktestChunk,
const tests::TestMethod &testMethod,
tests::Tests &tests);
void processBatchWithoutInteractive(const std::vector<tests::TestMethod> &testMethods,
tests::Tests &tests,
std::vector<tests::MethodKtests> &ktests);

void processBatchWithInteractive(const std::vector<tests::TestMethod> &testMethods,
tests::Tests &tests,
Expand Down
Loading