-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'tmp-branch-move-files-to-framework' into move-files-int…
…o-framework
- Loading branch information
Showing
3 changed files
with
616 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,87 @@ | ||
#!/usr/bin/env python3 | ||
|
||
"""Sanity checks for test data. | ||
This program contains a class for traversing test cases that can be used | ||
independently of the checks. | ||
""" | ||
|
||
# Copyright The Mbed TLS Contributors | ||
# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later | ||
|
||
import argparse | ||
import re | ||
import sys | ||
|
||
import scripts_path # pylint: disable=unused-import | ||
from mbedtls_framework import collect_test_cases | ||
|
||
|
||
class DescriptionChecker(collect_test_cases.TestDescriptionExplorer): | ||
"""Check all test case descriptions. | ||
* Check that each description is valid (length, allowed character set, etc.). | ||
* Check that there is no duplicated description inside of one test suite. | ||
""" | ||
|
||
def __init__(self, results): | ||
self.results = results | ||
|
||
def new_per_file_state(self): | ||
"""Dictionary mapping descriptions to their line number.""" | ||
return {} | ||
|
||
def process_test_case(self, per_file_state, | ||
file_name, line_number, description): | ||
"""Check test case descriptions for errors.""" | ||
results = self.results | ||
seen = per_file_state | ||
if description in seen: | ||
results.error(file_name, line_number, | ||
'Duplicate description (also line {})', | ||
seen[description]) | ||
return | ||
if re.search(br'[\t;]', description): | ||
results.error(file_name, line_number, | ||
'Forbidden character \'{}\' in description', | ||
re.search(br'[\t;]', description).group(0).decode('ascii')) | ||
if re.search(br'[^ -~]', description): | ||
results.error(file_name, line_number, | ||
'Non-ASCII character in description') | ||
if len(description) > 66: | ||
results.warning(file_name, line_number, | ||
'Test description too long ({} > 66)', | ||
len(description)) | ||
seen[description] = line_number | ||
|
||
def main(): | ||
parser = argparse.ArgumentParser(description=__doc__) | ||
parser.add_argument('--list-all', | ||
action='store_true', | ||
help='List all test cases, without doing checks') | ||
parser.add_argument('--quiet', '-q', | ||
action='store_true', | ||
help='Hide warnings') | ||
parser.add_argument('--verbose', '-v', | ||
action='store_false', dest='quiet', | ||
help='Show warnings (default: on; undoes --quiet)') | ||
options = parser.parse_args() | ||
if options.list_all: | ||
descriptions = collect_test_cases.collect_available_test_cases() | ||
sys.stdout.write('\n'.join(descriptions + [''])) | ||
return | ||
results = collect_test_cases.Results(options) | ||
checker = DescriptionChecker(results) | ||
try: | ||
checker.walk_all() | ||
except collect_test_cases.ScriptOutputError as e: | ||
results.error(e.script_name, e.idx, | ||
'"{}" should be listed as "<suite_name>;<description>"', | ||
e.line) | ||
if (results.warnings or results.errors) and not options.quiet: | ||
sys.stderr.write('{}: {} errors, {} warnings\n' | ||
.format(sys.argv[0], results.errors, results.warnings)) | ||
sys.exit(1 if results.errors else 0) | ||
|
||
if __name__ == '__main__': | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,167 @@ | ||
"""Discover all the test cases (unit tests and SSL tests).""" | ||
|
||
# Copyright The Mbed TLS Contributors | ||
# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later | ||
|
||
import glob | ||
import os | ||
import re | ||
import subprocess | ||
import sys | ||
|
||
import scripts_path # pylint: disable=unused-import | ||
from mbedtls_framework import build_tree | ||
|
||
|
||
class ScriptOutputError(ValueError): | ||
"""A kind of ValueError that indicates we found | ||
the script doesn't list test cases in an expected | ||
pattern. | ||
""" | ||
|
||
@property | ||
def script_name(self): | ||
return super().args[0] | ||
|
||
@property | ||
def idx(self): | ||
return super().args[1] | ||
|
||
@property | ||
def line(self): | ||
return super().args[2] | ||
|
||
class Results: | ||
"""Store file and line information about errors or warnings in test suites.""" | ||
|
||
def __init__(self, options): | ||
self.errors = 0 | ||
self.warnings = 0 | ||
self.ignore_warnings = options.quiet | ||
|
||
def error(self, file_name, line_number, fmt, *args): | ||
sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n'). | ||
format(file_name, line_number, *args)) | ||
self.errors += 1 | ||
|
||
def warning(self, file_name, line_number, fmt, *args): | ||
if not self.ignore_warnings: | ||
sys.stderr.write(('{}:{}:Warning:' + fmt + '\n') | ||
.format(file_name, line_number, *args)) | ||
self.warnings += 1 | ||
|
||
class TestDescriptionExplorer: | ||
"""An iterator over test cases with descriptions. | ||
The test cases that have descriptions are: | ||
* Individual unit tests (entries in a .data file) in test suites. | ||
* Individual test cases in ssl-opt.sh. | ||
This is an abstract class. To use it, derive a class that implements | ||
the process_test_case method, and call walk_all(). | ||
""" | ||
|
||
def process_test_case(self, per_file_state, | ||
file_name, line_number, description): | ||
"""Process a test case. | ||
per_file_state: an object created by new_per_file_state() at the beginning | ||
of each file. | ||
file_name: a relative path to the file containing the test case. | ||
line_number: the line number in the given file. | ||
description: the test case description as a byte string. | ||
""" | ||
raise NotImplementedError | ||
|
||
def new_per_file_state(self): | ||
"""Return a new per-file state object. | ||
The default per-file state object is None. Child classes that require per-file | ||
state may override this method. | ||
""" | ||
#pylint: disable=no-self-use | ||
return None | ||
|
||
def walk_test_suite(self, data_file_name): | ||
"""Iterate over the test cases in the given unit test data file.""" | ||
in_paragraph = False | ||
descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none | ||
with open(data_file_name, 'rb') as data_file: | ||
for line_number, line in enumerate(data_file, 1): | ||
line = line.rstrip(b'\r\n') | ||
if not line: | ||
in_paragraph = False | ||
continue | ||
if line.startswith(b'#'): | ||
continue | ||
if not in_paragraph: | ||
# This is a test case description line. | ||
self.process_test_case(descriptions, | ||
data_file_name, line_number, line) | ||
in_paragraph = True | ||
|
||
def collect_from_script(self, script_name): | ||
"""Collect the test cases in a script by calling its listing test cases | ||
option""" | ||
descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none | ||
listed = subprocess.check_output(['sh', script_name, '--list-test-cases']) | ||
# Assume test file is responsible for printing identical format of | ||
# test case description between --list-test-cases and its OUTCOME.CSV | ||
# | ||
# idx indicates the number of test case since there is no line number | ||
# in the script for each test case. | ||
for idx, line in enumerate(listed.splitlines()): | ||
# We are expecting the script to list the test cases in | ||
# `<suite_name>;<description>` pattern. | ||
script_outputs = line.split(b';', 1) | ||
if len(script_outputs) == 2: | ||
suite_name, description = script_outputs | ||
else: | ||
raise ScriptOutputError(script_name, idx, line.decode("utf-8")) | ||
|
||
self.process_test_case(descriptions, | ||
suite_name.decode('utf-8'), | ||
idx, | ||
description.rstrip()) | ||
|
||
@staticmethod | ||
def collect_test_directories(): | ||
"""Get the relative path for the TLS and Crypto test directories.""" | ||
mbedtls_root = build_tree.guess_mbedtls_root() | ||
directories = [os.path.join(mbedtls_root, 'tests'), | ||
os.path.join(mbedtls_root, 'tf-psa-crypto', 'tests')] | ||
directories = [os.path.relpath(p) for p in directories] | ||
return directories | ||
|
||
def walk_all(self): | ||
"""Iterate over all named test cases.""" | ||
test_directories = self.collect_test_directories() | ||
for directory in test_directories: | ||
for data_file_name in glob.glob(os.path.join(directory, 'suites', | ||
'*.data')): | ||
self.walk_test_suite(data_file_name) | ||
|
||
for sh_file in ['ssl-opt.sh', 'compat.sh']: | ||
sh_file = os.path.join(directory, sh_file) | ||
if os.path.isfile(sh_file): | ||
self.collect_from_script(sh_file) | ||
|
||
class TestDescriptions(TestDescriptionExplorer): | ||
"""Collect the available test cases.""" | ||
|
||
def __init__(self): | ||
super().__init__() | ||
self.descriptions = set() | ||
|
||
def process_test_case(self, _per_file_state, | ||
file_name, _line_number, description): | ||
"""Record an available test case.""" | ||
base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) | ||
key = ';'.join([base_name, description.decode('utf-8')]) | ||
self.descriptions.add(key) | ||
|
||
def collect_available_test_cases(): | ||
"""Collect the available test cases.""" | ||
explorer = TestDescriptions() | ||
explorer.walk_all() | ||
return sorted(explorer.descriptions) |
Oops, something went wrong.