diff --git a/tools/perf/cli_tools/tbmv3/validators/simple_validator.py b/tools/perf/cli_tools/tbmv3/validators/simple_validator.py index f0030391a40344..0cf3ee0511f5b2 100644 --- a/tools/perf/cli_tools/tbmv3/validators/simple_validator.py +++ b/tools/perf/cli_tools/tbmv3/validators/simple_validator.py @@ -2,7 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """ -Validates the rendering/frame_times metric. +Implementation for simple validators defined in simple_configs.pyl. """ import sys @@ -25,13 +25,10 @@ def CheckConfig(simple_config): (name, CONFIG_FORMAT)) -def GetHistogram(histogram_set, name, metric, version): +def OptionalGetHistogram(histogram_set, name, metric, version): hists = histogram_set.GetHistogramsNamed(name) if len(hists) == 0: - msg = ('List of histograms produced by TBM%s metric %s:\n' % - (version, metric)) - msg += '\n'.join([h.name for h in histogram_set]) - raise Exception('Histogram %s not found.\n%s' % (name, msg)) + return None if len(hists) > 1: raise Exception('Multiple histograms named %s found for TBM%s metric %s' % (name, version, metric)) @@ -60,8 +57,21 @@ def CompareHistograms(test_ctx): raise Exception('v3_histogram must be either string of v3_histogram ' ' name of (v3_hist_name, precision) tuple.') - v2_hist = GetHistogram(v2_histograms, v2_hist_name, v2_metric, 'v2') - v3_hist = GetHistogram(v3_histograms, v3_hist_name, v3_metric, 'v3') + v2_hist = OptionalGetHistogram(v2_histograms, v2_hist_name, v2_metric, 'v2') + v3_hist = OptionalGetHistogram(v3_histograms, v3_hist_name, v3_metric, 'v3') + + if (v2_hist is None) or (v2_hist.num_values == 0): + if (v3_hist is not None) and (v3_hist.num_values > 0): + raise Exception('v3 metric produced non-empty histogram %s, but ' + 'equivalent histogram %s is not present or empty ' + 'in v2 metric' % (v3_hist_name, v2_hist_name)) + continue + + if v3_hist is None: + msg = ('List of histograms produced by v3 metric %s:\n' % (v3_metric)) + msg += '\n'.join([h.name for h in v3_histograms]) + raise Exception('Histogram %s not produced by v3 metric\n%s' % + (v3_hist_name, msg)) try: utils.AssertHistogramStatsAlmostEqual(test_ctx, v2_hist, v3_hist, diff --git a/tools/perf/cli_tools/tbmv3/validators/utils.py b/tools/perf/cli_tools/tbmv3/validators/utils.py index beb0324f901614..d5d925c8f1bfee 100644 --- a/tools/perf/cli_tools/tbmv3/validators/utils.py +++ b/tools/perf/cli_tools/tbmv3/validators/utils.py @@ -2,7 +2,6 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - def AssertHistogramStatsAlmostEqual(test_ctx, v2_hist, v3_hist, precision=1e-3): """Asserts major histogram statistics are close enough. @@ -21,8 +20,9 @@ def AssertHistogramSamplesAlmostEqual(test_ctx, v2_hist, v3_hist, precision=1e-3): - v2_samples = v2_hist.sample_values - v3_samples = v3_hist.sample_values + v2_samples = [s for s in v2_hist.sample_values if s is not None] + v3_samples = [s for s in v3_hist.sample_values if s is not None] + test_ctx.assertEqual(len(v2_samples), len(v3_samples)) v2_samples.sort() v3_samples.sort()