Skip to content

Commit 1d626b8

Browse files
jsingletjsinglet
committed
update workflow
1 parent 5381b83 commit 1d626b8

File tree

2 files changed

+180
-7
lines changed

2 files changed

+180
-7
lines changed

‎.github/workflows/internal-pr-bundle-integration-test-cpp.yml

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -106,10 +106,4 @@ jobs:
106106
shell: bash
107107
run: |
108108
# Compare the expected vs the actual
109-
cat integration-tests/cpp/expected.sarif | jq '.runs' > integration-tests/cpp/expected
110-
cat ${{ steps.analysis.outputs.sarif-output }}/cpp.sarif | jq '.runs' > integration-tests/cpp/actual
111-
112-
if ! diff integration-tests/cpp/expected integration-tests/cpp/actual ; then
113-
echo "Expected file does not match actual. Please check the SARIF file for differences."
114-
exit 1
115-
fi
109+
qlt bundle run validate-integration-tests --expected integration-tests/cpp/expected.sarif --actual ${{ steps.analysis.outputs.sarif-output }}/cpp.sarif

‎src/CodeQLToolkit.Features/Bundle/Commands/Targets/ValidateIntegrationTestResults.cs

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
using CodeQLToolkit.Features.CodeQL.Commands.Targets;
2+
using CodeQLToolkit.Features.Validation;
3+
using Newtonsoft.Json;
4+
using NLog;
5+
using System;
6+
using System.Collections.Generic;
7+
using System.Linq;
8+
using System.Text;
9+
using System.Threading.Tasks;
10+
11+
namespace CodeQLToolkit.Features.Bundle.Commands.Targets
12+
{
13+
public class ValidateIntegrationTestResults : CommandTarget
14+
{
15+
public string Expected { get; set; }
16+
public string Actual { get; set; }
17+
18+
public override void Run()
19+
{
20+
Log<ValidateIntegrationTestResults>.G().LogInformation($"Running validate-integration-tests command");
21+
22+
// First load up the expected and the actuals.
23+
if (!File.Exists(Expected))
24+
{
25+
DieWithError($"Expected file {Expected} does not exist.");
26+
}
27+
28+
if(!File.Exists(Actual))
29+
{
30+
DieWithError($"Actual file {Actual} does not exist.");
31+
}
32+
33+
try
34+
{
35+
var expectedSARIF = JsonConvert.DeserializeObject<SARIFResult>(File.ReadAllText(Expected));
36+
var actualSARIF = JsonConvert.DeserializeObject<SARIFResult>(File.ReadAllText(Actual));
37+
38+
39+
// Check 1: Ensure they contain the same number of results. This is a pretty signifigant error so we don't report
40+
// past this point.
41+
if (expectedSARIF.runs.Length != actualSARIF.runs.Length)
42+
{
43+
DieWithError($"Number of runs does not match. Expected: {expectedSARIF.runs.Length}, Actual: {actualSARIF.runs.Length}");
44+
}
45+
46+
// Check 2: For each run, check if the number of results match.
47+
for (var i = 0; i < expectedSARIF.runs.Length; i++)
48+
{
49+
if (expectedSARIF.runs[i].results.Length != actualSARIF.runs[i].results.Length)
50+
{
51+
Log<ValidateIntegrationTestResults>.G().LogWarning($"Number of results does not match. Expected: {expectedSARIF.runs[i].results.Length}, Actual: {actualSARIF.runs[i].results.Length}");
52+
}
53+
}
54+
55+
// Check 3: Build up some mappings to compare exactly what is missing.
56+
var actualDict = ExtractResultMap(actualSARIF);
57+
var expectedDict = ExtractResultMap(expectedSARIF);
58+
59+
60+
// Populate the differences.
61+
var resultsInExpectedNotInActual = FindMissingResults(expectedSARIF, actualDict);
62+
var resultsInActualNotInExpected = FindMissingResults(actualSARIF, expectedDict);
63+
64+
// Report results.
65+
if(resultsInExpectedNotInActual.Count == 0 && resultsInActualNotInExpected.Count == 0)
66+
{
67+
Log<ValidateIntegrationTestResults>.G().LogInformation($"SARIF results identical.");
68+
}
69+
else
70+
{
71+
Log<ValidateIntegrationTestResults>.G().LogInformation($"SARIF results not identical. Please see below for full report.");
72+
73+
Log<ValidateIntegrationTestResults>.G().LogInformation($"SARIF Results in EXPECTED not contained in ACTUAL");
74+
Log<ValidateIntegrationTestResults>.G().LogInformation($"------------------------------------------------------");
75+
76+
foreach (var r in resultsInExpectedNotInActual)
77+
{
78+
79+
Log<ValidateIntegrationTestResults>.G().LogInformation($"Rule: {r.ruleId} @ Location {r.LocationsString()} with Message \"{r.message.text}\" exists in EXPECTED but is missing from ACTUAL.");
80+
81+
}
82+
83+
Log<ValidateIntegrationTestResults>.G().LogInformation($"SARIF Results in ACTUAL not contained in EXPECTED");
84+
Log<ValidateIntegrationTestResults>.G().LogInformation($"------------------------------------------------------");
85+
86+
foreach (var r in resultsInActualNotInExpected)
87+
{
88+
Log<ValidateIntegrationTestResults>.G().LogInformation($"Rule: {r.ruleId} @ Location {r.LocationsString()} with Message \"{r.message.text}\" exists in ACTUAL but is missing from EXPECTED.");
89+
}
90+
91+
DieWithError("SARIF results NOT identical. Results reported, above.");
92+
}
93+
94+
}
95+
catch (Exception e)
96+
{
97+
Log<ValidateIntegrationTestResults>.G().LogError(e.Message);
98+
DieWithError("Error decoding SARIF files. Please check the format and try again.");
99+
}
100+
101+
List<Result> FindMissingResults(SARIFResult? expectedSARIF, Dictionary<string, List<Result>> actualDict)
102+
{
103+
var resultsNotFound = new List<Result>();
104+
105+
for (var i = 0; i < expectedSARIF.runs.Length; i++)
106+
{
107+
for (var j = 0; j < expectedSARIF.runs[i].results.Length; j++)
108+
{
109+
var result = expectedSARIF.runs[i].results[j];
110+
111+
if (!actualDict.ContainsKey(result.ruleId))
112+
{
113+
resultsNotFound.Add(result);
114+
}
115+
else
116+
{
117+
if (!ContainsResult(result, actualDict[result.ruleId]))
118+
{
119+
resultsNotFound.Add(result);
120+
}
121+
}
122+
}
123+
124+
}
125+
126+
return resultsNotFound;
127+
}
128+
}
129+
130+
private bool ContainsResult(Result searchFor, List<Result> inResults)
131+
{
132+
foreach(var result in inResults)
133+
{
134+
if(ResultMatches(result, searchFor))
135+
{
136+
return true;
137+
}
138+
}
139+
140+
return false;
141+
}
142+
143+
private bool ResultMatches(Result a, Result b)
144+
{
145+
if(a.ruleId == b.ruleId && a.message.text == b.message.text && a.LocationsString() == b.LocationsString())
146+
{
147+
return true;
148+
}
149+
150+
return false;
151+
}
152+
153+
private Dictionary<string, List<Result>> ExtractResultMap(SARIFResult? sarif)
154+
{
155+
Dictionary<string, List<Result>> dict = new Dictionary<string, List<Result>>();
156+
157+
for (var i = 0; i < sarif.runs.Length; i++)
158+
{
159+
for (var j = 0; j < sarif.runs[i].results.Length; j++)
160+
{
161+
var result = sarif.runs[i].results[j];
162+
163+
if (dict.ContainsKey(result.ruleId))
164+
{
165+
dict[result.ruleId].Add(result);
166+
}
167+
else
168+
{
169+
dict[result.ruleId] = new List<Result>()
170+
{
171+
result
172+
};
173+
}
174+
}
175+
}
176+
return dict;
177+
}
178+
}
179+
}

0 commit comments

Comments
 (0)