Skip to content

Commit

Permalink
[test] Add several more cmdline tests for tag diffs
Browse files Browse the repository at this point in the history
My previous commit that added cmdline tests targeted local-local,
remote-local and local-remote diffs, and remote-remote diffs to a lesser
extent. It also added 2 cases related to tags.

This PR adds cases from this issue: Ericsson#3884, specifically demonstrating
the table found in section "Tag diffs" -> "Remote-Remote" _before_ the
addition of a third tag.
  • Loading branch information
Szelethus committed Aug 15, 2023
1 parent 38f9315 commit deab57c
Showing 1 changed file with 156 additions and 12 deletions.
168 changes: 156 additions & 12 deletions web/tests/functional/diff_cmdline/test_diff_cmdline.py
Original file line number Diff line number Diff line change
Expand Up @@ -576,9 +576,99 @@ def get_run_diff_count_reverse(diff_type: DiffType):
# Ideally, diffing tags should work the same as diffing two remote runs or
# local directory.

# TODO: remote-remote diffs concerning tags
def test_remoteTag_remoteTag_identical(self):
dir1 = os.path.join(self.test_workspace, "dir1")
dir2 = os.path.join(self.test_workspace, "dir2")

src_div_by_zero = """
void a() {
int i = 0;
(void)(10 / i);
}
"""
# Note that we're storing under the same run.
self.__analyze_and_store(dir1, "run1", src_div_by_zero, "tag1")
self.__analyze_and_store(dir2, "run1", src_div_by_zero, "tag2")

report_filter = ReportFilter()
report_filter.reviewStatus = []
report_filter.detection_status = []

def get_run_diff_count(diff_type: DiffType):
reports, _, _ = get_diff_remote_runs(
self._cc_client, report_filter, diff_type, [],
["run1:tag1"], ["run1:tag2"])
return len(reports)

self.assertEqual(get_run_diff_count(DiffType.NEW), 0)
self.assertEqual(get_run_diff_count(DiffType.RESOLVED), 0)
self.assertEqual(get_run_diff_count(DiffType.UNRESOLVED), 1)

def get_run_diff_count_reverse(diff_type: DiffType):
reports, _, _ = get_diff_remote_runs(
self._cc_client, report_filter, diff_type, [],
["run1:tag2"], ["run1:tag1"])
return len(reports)

self.assertEqual(get_run_diff_count_reverse(DiffType.NEW), 0)
self.assertEqual(get_run_diff_count_reverse(DiffType.RESOLVED), 0)
self.assertEqual(get_run_diff_count_reverse(DiffType.UNRESOLVED), 1)

def test_remoteTag_remoteTag_different(self):
dir1 = os.path.join(self.test_workspace, "dir1")
dir2 = os.path.join(self.test_workspace, "dir2")

src_div_by_zero = """
void a() {
int i = 0;
(void)(10 / i);
}
"""
src_nullptr_deref = """
void b() {
int *i = 0;
*i = 5;
}
"""
# Note that we're storing under the same run.
self.__analyze_and_store(dir1, "run1", src_div_by_zero, "tag1")
self.__analyze_and_store(dir2, "run1", src_nullptr_deref, "tag2")

report_filter = ReportFilter()
report_filter.reviewStatus = []
report_filter.detection_status = []

def get_run_diff_count(diff_type: DiffType):
reports, _, _ = get_diff_remote_runs(
self._cc_client, report_filter, diff_type, [],
["run1:tag1"], ["run1:tag2"])
return len(reports)

self.assertEqual(get_run_diff_count(DiffType.NEW), 1)
# FIXME: The division by zero disappeared from tag1 to tag2, so the
# RESOLVED set should have a report.
self.assertEqual(get_run_diff_count(DiffType.RESOLVED), 0)
self.assertEqual(get_run_diff_count(DiffType.UNRESOLVED), 0)

def test_remoteFPAnnotated_remote_tag(self):
def get_run_diff_count_reverse(diff_type: DiffType):
reports, _, _ = get_diff_remote_runs(
self._cc_client, report_filter, diff_type, [],
["run1:tag2"], ["run1:tag1"])
return len(reports)

# FIXME: The division by zero disappeared from tag1 to tag2, so the
# NEW set should have a report when we reverse the diff.
self.assertEqual(get_run_diff_count_reverse(DiffType.NEW), 0)
self.assertEqual(get_run_diff_count_reverse(DiffType.RESOLVED), 1)
self.assertEqual(get_run_diff_count_reverse(DiffType.UNRESOLVED), 0)

def test_remoteTagFPAnnotated_remoteTag_identical(self):
"""
Test source code suppression changes -- in tag1, a FP suppression is
present, and in tag2, it disappears. Internally, as of writing, this
will irreversibly remove the fixed_at attribute, so we are not really
able to get this right.
"""
dir1 = os.path.join(self.test_workspace, "dir1")
dir2 = os.path.join(self.test_workspace, "dir2")

Expand Down Expand Up @@ -618,7 +708,7 @@ def get_run_diff_count(diff_type: DiffType):
def get_run_diff_count_reverse(diff_type: DiffType):
reports, _, _ = get_diff_remote_runs(
self._cc_client, report_filter, diff_type, [],
["run1:tag1"], ["run1:tag2"])
["run1:tag2"], ["run1:tag1"])
return len(reports)

# FIXME: The FP suppression disappeared from one tag to the next, so it
Expand All @@ -627,7 +717,13 @@ def get_run_diff_count_reverse(diff_type: DiffType):
self.assertEqual(get_run_diff_count_reverse(DiffType.RESOLVED), 0)
self.assertEqual(get_run_diff_count_reverse(DiffType.UNRESOLVED), 1)

def test_remote_remoteFPAnnotated_tag(self):
def test_remoteTag_remoteTagFPAnnotated(self):
"""
Test source code suppression changes -- in tag1, there is no
suppression, and in tag2, there is an FP suppression. This should be
doable, as first, the fixed_at attribute is NULL, then it'll be set --
there is no permanent loss of information.
"""
# Diff two different, local runs.
dir1 = os.path.join(self.test_workspace, "dir1")
dir2 = os.path.join(self.test_workspace, "dir2")
Expand Down Expand Up @@ -668,7 +764,7 @@ def get_run_diff_count(diff_type: DiffType):
def get_run_diff_count_reverse(diff_type: DiffType):
reports, _, _ = get_diff_remote_runs(
self._cc_client, report_filter, diff_type, [],
["run1:tag1"], ["run1:tag2"])
["run1:tag2"], ["run1:tag1"])
return len(reports)

# FIXME: The FP suppression appeared from one tag to the next, so it
Expand All @@ -677,7 +773,56 @@ def get_run_diff_count_reverse(diff_type: DiffType):
self.assertEqual(get_run_diff_count_reverse(DiffType.RESOLVED), 0)
self.assertEqual(get_run_diff_count_reverse(DiffType.UNRESOLVED), 0)

def test_remote_remote_tag_FixedAtDate(self):
def test_remoteTagReviewStatusRule_remoteTag_identical(self):
"""
You can find more context for why this is the expected result in the
docs of test_local_remoteReviewStatusRule_identical.
"""
dir1 = os.path.join(self.test_workspace, "dir1")
dir2 = os.path.join(self.test_workspace, "dir2")

src_div_by_zero = """
void a() {
int i = 0;
(void)(10 / i);
}
"""
# Note that we're storing under the same run.
self.__analyze_and_store(dir1, "run1", src_div_by_zero, "tag1")

# Add a "false positive" review status rule on the stored report.
results = get_all_run_results(self._cc_client)
self.assertEqual(len(results), 1)
self._cc_client.addReviewStatusRule(
results[0].bugHash, ReviewStatus.FALSE_POSITIVE, "")

self.__analyze_and_store(dir2, "run1", src_div_by_zero, "tag2")

report_filter = ReportFilter()
report_filter.reviewStatus = []
report_filter.detection_status = []

def get_run_diff_count(diff_type: DiffType):
reports, _, _ = get_diff_remote_runs(
self._cc_client, report_filter, diff_type, [],
["run1:tag1"], ["run1:tag2"])
return len(reports)

self.assertEqual(get_run_diff_count(DiffType.NEW), 0)
self.assertEqual(get_run_diff_count(DiffType.RESOLVED), 0)
self.assertEqual(get_run_diff_count(DiffType.UNRESOLVED), 0)

def get_run_diff_count_reverse(diff_type: DiffType):
reports, _, _ = get_diff_remote_runs(
self._cc_client, report_filter, diff_type, [],
["run1:tag2"], ["run1:tag1"])
return len(reports)

self.assertEqual(get_run_diff_count_reverse(DiffType.NEW), 0)
self.assertEqual(get_run_diff_count_reverse(DiffType.RESOLVED), 0)
self.assertEqual(get_run_diff_count_reverse(DiffType.UNRESOLVED), 0)

def test_remoteTag_remoteTag_FixedAtDate(self):
"""
When a run disappears from one tag to the next, we regard it as fixed,
and set it fixed_at date. Test whether just because the fixed_at date
Expand All @@ -690,10 +835,9 @@ def test_remote_remote_tag_FixedAtDate(self):
dir1 = os.path.join(self.test_workspace, "dir1")
dir2 = os.path.join(self.test_workspace, "dir2")

src_div_by_zero = """
src_no_warnings = """
void a() {
int i = 0;
(void)(10 / i);
// No faults.
}
"""

Expand All @@ -704,7 +848,7 @@ def test_remote_remote_tag_FixedAtDate(self):
}
"""
# Note that we're storing under the same run.
self.__analyze_and_store(dir1, "run1", src_div_by_zero, "tag1")
self.__analyze_and_store(dir1, "run1", src_no_warnings, "tag1")
self.__analyze_and_store(dir2, "run1", src_nullptr_deref, "tag2")

report_filter = ReportFilter()
Expand All @@ -724,9 +868,9 @@ def get_run_diff_count(diff_type: DiffType):
# NOTE that we store the first report dir again. The report in tag2 is
# abscent, so its fixed_at date will be set. Still, the diff in between
# tag1 and tag2 shouldn't change.
self.__analyze_and_store(dir1, "run1", src_div_by_zero, "tag3")
self.__analyze_and_store(dir1, "run1", src_no_warnings, "tag3")

# FIXME: This report should not have disappeared!
self.assertEqual(get_run_diff_count(DiffType.NEW), 0)
self.assertEqual(get_run_diff_count(DiffType.RESOLVED), 0)
self.assertEqual(get_run_diff_count(DiffType.UNRESOLVED), 1)
self.assertEqual(get_run_diff_count(DiffType.UNRESOLVED), 0)

0 comments on commit deab57c

Please sign in to comment.