Skip to content

Commit 78229f5

Browse files
vvbaefruffy
vvbae
authored and
fruffy
committed
add testing to CI
1 parent e254fa2 commit 78229f5

File tree

4 files changed

+128
-7
lines changed

4 files changed

+128
-7
lines changed

‎targets/bmv2/CMakeLists.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
if(ENABLE_TESTING)
2+
# Include the test subdirectory.
3+
message("-- Adding Rtsmith v1model test suite.")
4+
include(test/P4Tests.cmake)
5+
endif()
6+
17
# Source files for the main P4RtSmith.
28
set(RTSMITH_SOURCES
39
${RTSMITH_SOURCES}

‎targets/bmv2/test/P4Tests.cmake‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
include(${CMAKE_CURRENT_LIST_DIR}/TestTemplate.cmake)
2+
# ##################################################################################################
3+
# TEST PROGRAMS
4+
# ##################################################################################################
5+
set(PYTHON_TEST_SCRIPT "${CMAKE_CURRENT_LIST_DIR}/run_test_batch.py")
6+
7+
set(V1_SEARCH_PATTERNS "include.*v1model.p4" "main|common_v1_test")
8+
# General BMv2 tests supplied by the compiler.
9+
set(P4TESTS_FOR_BMV2
10+
"${P4C_SOURCE_DIR}/testdata/p4_16_samples/*.p4"
11+
"${P4C_SOURCE_DIR}/testdata/p4_16_samples/dash/*.p4"
12+
"${P4C_SOURCE_DIR}/testdata/p4_16_samples/fabric_*/fabric.p4"
13+
"${P4C_SOURCE_DIR}/testdata/p4_16_samples/omec/*.p4"
14+
"${P4C_SOURCE_DIR}/testdata/p4_16_samples/pins/*.p4"
15+
# Custom tests
16+
"${CMAKE_CURRENT_LIST_DIR}/programs/*.p4"
17+
)
18+
19+
p4c_find_tests("${P4TESTS_FOR_BMV2}" P4_16_V1_TESTS INCLUDE "${V1_SEARCH_PATTERNS}" EXCLUDE "")
20+
21+
# Filter some programs because they have issues that are not captured with Xfails.
22+
list(
23+
REMOVE_ITEM
24+
P4_16_V1_TESTS
25+
# These tests time out and require fixing.
26+
)
27+
28+
set (EXTRA_OPTS "")
29+
30+
p4tools_add_tests(
31+
TESTS
32+
"${P4_16_V1_TESTS}"
33+
TAG
34+
"rtsmith-bmv2-v1model"
35+
DRIVER
36+
${PYTHON_TEST_SCRIPT}
37+
TARGET
38+
"bmv2"
39+
ARCH
40+
"v1model"
41+
TEST_ARGS
42+
"${EXTRA_OPTS}"
43+
)

‎targets/bmv2/test/TestTemplate.cmake‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# This file defines how a test should be written for a particular target. This is used by testutils
2+
3+
# Add a single test to the testsuite. Arguments: - TAG is a label for the set of test suite where
4+
# this test belongs (for example, p4ctest) - DRIVER is the script that is used to run the test and
5+
# compare the results - ALIAS is a possibly different name for the test such that the same p4
6+
# program can be used in different test configurations. Must be unique across the test suite. -
7+
# P4TEST is the name of the p4 program to test (path relative to the p4c directory) - TARGET is the
8+
# target to test against - ARCH is the p4 architecture - TEST_ARGS is a list of arguments to pass to
9+
# the test - CMAKE_ARGS are additional arguments to pass to the test
10+
#
11+
# It generates a ${p4test}.test file invoking ${driver} on the p4 program with command line
12+
# arguments ${args} Sets the timeout on tests at 300s. For the slow CI machines.
13+
function(p4tools_add_test_with_args)
14+
# Parse arguments.
15+
set(options)
16+
set(oneValueArgs TAG DRIVER ALIAS P4TEST TARGET ARCH)
17+
set(multiValueArgs TEST_ARGS CMAKE_ARGS)
18+
cmake_parse_arguments(
19+
TOOLS_RTSMITH_V1MODEL_TESTS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}
20+
)
21+
# Set some lowercase variables for convenience.
22+
set(tag ${TOOLS_RTSMITH_V1MODEL_TESTS_TAG})
23+
set(driver ${TOOLS_RTSMITH_V1MODEL_TESTS_DRIVER})
24+
set(alias ${TOOLS_RTSMITH_V1MODEL_TESTS_ALIAS})
25+
set(p4test ${TOOLS_RTSMITH_V1MODEL_TESTS_P4TEST})
26+
set(target ${TOOLS_RTSMITH_V1MODEL_TESTS_TARGET})
27+
set(arch ${TOOLS_RTSMITH_V1MODEL_TESTS_ARCH})
28+
set(test_args ${TOOLS_RTSMITH_V1MODEL_TESTS_TEST_ARGS})
29+
set(cmake_args ${TOOLS_RTSMITH_V1MODEL_TESTS_CMAKE_ARGS})
30+
31+
# This is the actual test processing.
32+
p4c_test_set_name(__testname ${tag} ${alias})
33+
string(REGEX REPLACE ".p4" "" aliasname ${alias})
34+
set(__testfile "${RTSMITH_DIR}/${tag}/${alias}.test")
35+
get_filename_component(__testdir ${p4test} DIRECTORY)
36+
file(WRITE ${__testfile} "#! /usr/bin/env bash\n")
37+
file(APPEND ${__testfile} "# Generated file, modify with care\n\n")
38+
file(APPEND ${__testfile} "set -e\n")
39+
file(APPEND ${__testfile} "cd ${P4C_BINARY_DIR}\n")
40+
41+
file(APPEND ${__testfile} "${driver} ${p4test} ${RTSMITH_DRIVER} \n"
42+
)
43+
44+
execute_process(COMMAND chmod +x ${__testfile})
45+
separate_arguments(__args UNIX_COMMAND ${cmake_args})
46+
add_test(NAME ${__testname} COMMAND ${tag}/${alias}.test ${__args} WORKING_DIRECTORY ${RTSMITH_DIR})
47+
if(NOT DEFINED ${tag}_timeout)
48+
set(${tag}_timeout 420)
49+
endif()
50+
set_tests_properties(${__testname} PROPERTIES LABELS ${tag} TIMEOUT ${${tag}_timeout})
51+
endfunction(p4tools_add_test_with_args)

‎targets/bmv2/test/run_test_batch.py‎

100644100755
Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#!/usr/bin/env python3
12
import argparse
23
import logging
34
import os
@@ -26,6 +27,12 @@
2627
dest="seed",
2728
help="Random seed for generating configs.",
2829
)
30+
PARSER.add_argument(
31+
"--num_init_config",
32+
default=1,
33+
dest="num_init_config",
34+
help="Number of initial configs to generate for the given P4 program.",
35+
)
2936

3037

3138
# Parse options and process argv
@@ -46,16 +53,20 @@ class Options:
4653
testdir: Path = Path(".")
4754
# Random seed for generating configs.
4855
seed: int = 1
56+
# Number of initial configs to generate for the given P4 program.
57+
num_init_config: int = 1
4958

5059

5160
def generate_config(p4rtsmith_path, seed, p4_program_path, testdir, config_file_path):
5261
command = f"{p4rtsmith_path} --target bmv2 --arch v1model --seed {seed} --output-dir {testdir} --generate-config {config_file_path} {p4_program_path}"
53-
subprocess.run(command, shell=True)
62+
returncode = subprocess.run(command, shell=True)
63+
return returncode.returncode
5464

5565

5666
def run_test(run_test_script, p4_program_path, config_file_path):
5767
command = f"sudo -E {run_test_script} .. {p4_program_path} -tf {config_file_path}"
58-
subprocess.run(command, shell=True)
68+
returncode = subprocess.run(command, shell=True)
69+
return returncode.returncode
5970

6071

6172
def find_p4c_dir():
@@ -73,16 +84,24 @@ def find_p4c_dir():
7384

7485

7586
def run_tests(options: Options) -> int:
76-
config_file_path = "initial_config.txtpb"
77-
7887
seed = options.seed
7988
testdir = options.testdir
8089
p4rtsmith_path = options.p4rtsmith
8190
run_test_script = FILE_DIR / "run-bmv2-proto-test.py"
8291
p4_program_path = options.p4_file
92+
filename = os.path.splitext(os.path.basename(p4_program_path))[0]
8393

84-
generate_config(p4rtsmith_path, seed, p4_program_path, testdir, config_file_path)
85-
run_test(run_test_script, p4_program_path, testdir / config_file_path)
94+
for i in range(options.num_init_config):
95+
config_file_path = f"initial_config_{filename}_{i}.txtpb"
96+
result = generate_config(
97+
p4rtsmith_path, seed, p4_program_path, testdir, config_file_path
98+
)
99+
if result != 0:
100+
return result
101+
result = run_test(run_test_script, p4_program_path, testdir / config_file_path)
102+
if result != 0:
103+
return result
104+
return 0
86105

87106

88107
def create_options(test_args: Any) -> Optional[Options]:
@@ -101,6 +120,7 @@ def create_options(test_args: Any) -> Optional[Options]:
101120
os.chmod(testdir, 0o755)
102121
options.testdir = Path(testdir)
103122
options.seed = test_args.seed
123+
options.num_init_config = test_args.num_init_config
104124

105125
# Configure logging.
106126
logging.basicConfig(
@@ -120,4 +140,5 @@ def create_options(test_args: Any) -> Optional[Options]:
120140
if not test_options:
121141
sys.exit()
122142

123-
run_tests(test_options)
143+
test_result = run_tests(test_options)
144+
sys.exit(test_result)

0 commit comments

Comments
 (0)