Skip to content

Commit c398434

Browse files
WangJialei-AWangJialei-A
committed
benchgc: update ci case
1 parent ede109a commit c398434

File tree

4 files changed

+62
-60
lines changed

4 files changed

+62
-60
lines changed

‎.github/workflows/build.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,5 +56,9 @@ jobs:
5656
pip install test/benchgc/dist/benchgc-*.whl
5757
5858
- name: Correctness Test
59+
env:
60+
MLIR_RUNNER_UTILS: ../install/llvm/lib/libmlir_runner_utils.so
61+
MLIR_C_RUNNER_UTILS: ../install/llvm/lib/libmlir_c_runner_utils.so
62+
LD_PRELOAD: /lib/x86_64-linux-gnu/libomp5.so
5963
run: |
6064
scripts/correctness.sh

‎scripts/correctness.sh

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,44 @@
11
#! /bin/bash
22

3-
43
# need to import tools as a package
54
export PYTHONPATH=$(pwd)
6-
export MLIR_RUNNER_UTILS=../install/llvm/lib/libmlir_runner_utils.so
7-
export MLIR_C_RUNNER_UTILS=../install/llvm/lib/libmlir_c_runner_utils.so
8-
export LD_PRELOAD=/lib/x86_64-linux-gnu/libomp5.so
5+
export CASE_DIR=$(pwd)/test/benchgc/cases
6+
7+
8+
FAIL=0
9+
set -e
10+
11+
# misc
12+
python3 -m benchgc --verbose 0 --driver linalg --case fill -i f32:U:1:128 -o 32x4096xf32:P:0:0 || FAIL=1
13+
python3 -m benchgc --verbose 0 --driver linalg --case copy -i 1024x1024xf32:U:-128:127 -o 1024x1024xbf16:P:0.0078125:30.0 || FAIL=1
14+
python3 -m benchgc --verbose 0 --driver linalg --case broadcast -i 1024xf32:U:-128:127 -o 2x32x1024xf32:P:0:30.0 --dimensions=0 --dimensions=1 || FAIL=1
15+
16+
# matmul
17+
python3 -m benchgc --verbose 0 --driver linalg --case matmul_transpose_b -i 1024x512xf32:D -i 1024x512xf32:D -o 1024x1024xf32:D --cast cast_signed || FAIL=1
18+
19+
# binary
20+
python3 -m benchgc --verbose 0 --driver linalg --case add -i 1x32x4096xf32:D -i 1x32x4096xf32:D -o 1x32x4096xf32:D || FAIL=1
21+
python3 -m benchgc --verbose 0 --driver linalg --case mul -i 1x32x4096xf32:D -i 1x32x4096xf32:D -o 1x32x4096xf32:D || FAIL=1
22+
python3 -m benchgc --verbose 0 --driver linalg --case div -i 1x32x4096xf32:D -i 1x32x4096xf32:D -o 1x32x4096xf32:D || FAIL=1
23+
24+
# element wise
25+
python3 -m benchgc --verbose 0 --driver linalg --case negf -i 32x4096xf32:D -o 32x4096xf32:D || FAIL=1
26+
python3 -m benchgc --verbose 0 --driver linalg --case exp -i 32x4096xf32:D -o 32x4096xf32:D || FAIL=1
27+
28+
# mlir
29+
python3 -m benchgc --verbose 0 --driver mlir --case ${CASE_DIR}/llama2.mlir \
30+
-i 1x32x4096xbf16:N:0:1 \
31+
-i 4096x4096xbf16:N:0:1 \
32+
-i 1x32x4096xbf16:N:0:1 \
33+
-i 1xf32:N:0:1 \
34+
-i 4096xbf16:N:0:1 \
35+
-i 11008x4096xbf16:N:0:1 \
36+
-i 11008x4096xbf16:N:0:1 \
37+
-i 4096x11008xbf16:N:0:1 \
38+
-i 1xf32:N:0:1 \
39+
-i 4096xbf16:N:0:1 \
40+
-o 1x32x4096xbf16:P:0.0078125:30.0 \
41+
-o 1x32x4096xbf16:P:0.0078125:30.0 || FAIL=1
942

10-
python3 -m benchgc --verbose 0 --driver linalg --case matmul_transpose_b -i 1024x512xf32:D -i 1024x512xf32:D -o 1024x1024xf32:D --cast cast_signed
43+
set +e
44+
exit $FAIL

‎test/benchgc/src/benchgc/__main__.py

Lines changed: 14 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@
7575
help="verbose level",
7676
choices=[
7777
benchgc.util.NO_VERBOSE,
78+
benchgc.util.MODULE_VERBOSE,
7879
benchgc.util.COMPARE_VERBOSE,
7980
benchgc.util.ERROR_OUTPUT_VERBOSE,
8081
benchgc.util.OUTPUT_VERBOSE,
@@ -97,50 +98,6 @@
9798
help="define the dimensions attribute in linalg op",
9899
type=int,
99100
)
100-
parser.add_argument(
101-
"--stride_w",
102-
required=False,
103-
default=1,
104-
help="define the stride attribute",
105-
type=int,
106-
)
107-
parser.add_argument(
108-
"--stride_h",
109-
required=False,
110-
default=1,
111-
help="define the stride attribute",
112-
type=int,
113-
)
114-
parser.add_argument(
115-
"--stride_d",
116-
required=False,
117-
default=1,
118-
help="define the stride attribute",
119-
type=int,
120-
)
121-
122-
parser.add_argument(
123-
"--dilation_w",
124-
required=False,
125-
default=1,
126-
help="define the dilation attribute",
127-
type=int,
128-
)
129-
parser.add_argument(
130-
"--dilation_h",
131-
required=False,
132-
default=1,
133-
help="define the dilation attribute",
134-
type=int,
135-
)
136-
parser.add_argument(
137-
"--dilation_d",
138-
required=False,
139-
default=1,
140-
help="define the dilation attribute",
141-
type=int,
142-
)
143-
144101
flags = parser.parse_args()
145102
benchgc.util.set_seed(flags.seed)
146103

@@ -174,7 +131,8 @@
174131
else:
175132
raise Exception("unsupported driver %s" % flags.driver)
176133

177-
print(module)
134+
if flags.verbose >= benchgc.util.MODULE_VERBOSE:
135+
print(module)
178136

179137
gc_args: List[Any] = []
180138
gc_out: List[torch.Tensor] = []
@@ -208,13 +166,18 @@
208166
engine = compiler.compile_and_jit(module)
209167
engine.invoke(entry, *mlir_args)
210168

169+
fail, mistrust = False, False
211170
for i in range(len(outs)):
212171
out = outs[i]
213172
out.set_default_compare_param(flags, ins, outs)
214173
res = compare_tensor(out, ref_out[i], gc_out[i], flags.verbose)
215-
if not res[0]:
216-
print("FAIL: %s.%s" % (flags.driver, flags.case))
217-
elif res[1]:
218-
print("MISTRUST: %s.%s" % (flags.driver, flags.case))
219-
else:
220-
print("PASSED: %s.%s" % (flags.driver, flags.case))
174+
fail = fail or (not res[0])
175+
if res[1] is not None:
176+
mistrust = mistrust | res[1]
177+
if fail:
178+
print("FAIL: %s.%s" % (flags.driver, flags.case))
179+
sys.exit(1)
180+
elif mistrust:
181+
print("MISTRUST: %s.%s" % (flags.driver, flags.case))
182+
else:
183+
print("PASSED: %s.%s" % (flags.driver, flags.case))

‎test/benchgc/src/benchgc/util.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,11 @@
2323

2424
# verbose level
2525
NO_VERBOSE = 0
26-
COMPARE_VERBOSE = 1 # + print threshold for comparison
27-
ERROR_OUTPUT_VERBOSE = 2 # + print all error data points if failed
28-
OUTPUT_VERBOSE = 3 # + print all result including passed tensor
29-
INPUT_VERBOSE = 4 # + print input torch tensors
26+
MODULE_VERBOSE = 1 # print the module will be executed
27+
COMPARE_VERBOSE = 2 # + print threshold for comparison
28+
ERROR_OUTPUT_VERBOSE = 3 # + print all error data points if failed
29+
OUTPUT_VERBOSE = 4 # + print all result including passed tensor
30+
INPUT_VERBOSE = 5 # + print input torch tensors
3031

3132
"""
3233
acc | acc | elems | value_range | worst case

0 commit comments

Comments
 (0)