Skip to content

Commit 04b3545

Browse files
WangJialei-AWangJialei-A
committed
test: benchgc: initial commit for benchgc
1 parent d09d815 commit 04b3545

27 files changed

+3316
-0
lines changed

‎CMakeLists.txt

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
2828
option(GC_LEGACY_ENABLE ON)
2929
option(GC_TEST_ENABLE "Build the tests" ON)
3030
option(GC_ENABLE_BINDINGS_PYTHON "Enable Graph Complier Python Binding" ON)
31+
option(GC_BENCH_ENABLE "Build benchgc. Only available when GC_TEST_ENABLE and GC_ENABLE_BINDINGS_PYTHON is enabled" ON)
3132
option(GC_DEV_LINK_LLVM_DYLIB "Link dynamic libraries of LLVM and MLIR. For developers only. Do not use it in packing the library." OFF)
3233

3334
if(GC_LEGACY_ENABLE)
@@ -58,6 +59,7 @@ endif()
5859

5960
if(GC_ENABLE_BINDINGS_PYTHON)
6061
include(MLIRDetectPythonEnv)
62+
find_package(Python 3.11 REQUIRED COMPONENTS Interpreter NumPy REQUIRED)
6163
mlir_configure_python_dev_packages()
6264
endif()
6365

@@ -94,6 +96,15 @@ if(GC_ENABLE_BINDINGS_PYTHON)
9496
add_subdirectory(python)
9597
endif()
9698

99+
100+
if(GC_ENABLE_BINDINGS_PYTHON AND GC_TEST_ENABLE AND GC_BENCH_ENABLE)
101+
message(STATUS "Enabling Benchgc")
102+
elseif()
103+
message(STATUS "Disabling Benchgc")
104+
set(GC_BENCH_ENABLE OFF CACHE BOOL "" FORCE)
105+
endif()
106+
107+
97108
set(GC_LIB_LINKED_LIBS
98109
MLIRLinalgx
99110
MLIRMicrokernel

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,4 +64,5 @@ Graph Compiler supports the following build-time options.
6464
| GC_TEST_ENABLE | **ON**, OFF | Controls building the tests |
6565
| GC_DEV_LINK_LLVM_DYLIB | ON, **OFF** | Controls dynamic link LLVM/MLIR libraries, mainly for developer |
6666
| GC_ENABLE_BINDINGS_PYTHON | **ON**, OFF | Controls building the Python API |
67+
| GC_BENCH_ENABLE | **ON**, OFF | Controls building benchgc. The configuration will only take effect when both GC_ENABLE_BINDING_PYTHON and GC_TEST_ENABLE are ON. |
6768

‎test/CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@ if(GC_ENABLE_BINDINGS_PYTHON)
4040
list(APPEND GC_OPT_TEST_DEPENDS GcPythonModules)
4141
endif()
4242

43+
if(GC_BENCH_ENABLE)
44+
add_subdirectory(benchgc)
45+
endif()
46+
4347
add_lit_testsuite(gc-check "Running the regression tests"
4448
${CMAKE_CURRENT_BINARY_DIR}
4549
DEPENDS ${GC_OPT_TEST_DEPENDS}

‎test/benchgc/.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
dist/
2+
src/benchgc.egg-info/
3+
build
4+
benchgc.egg-info/

‎test/benchgc/CMakeLists.txt

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
################################################################################
2+
# Copyright (C) 2024 Intel Corporation
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing,
11+
# software distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions
14+
# and limitations under the License.
15+
# SPDX-License-Identifier: Apache-2.0
16+
################################################################################
17+
18+
if(NOT GC_BENCH_ENABLE)
19+
message(STATUS "Benchgc is not enabled")
20+
return()
21+
endif()
22+
23+
configure_file(setup.py ${CMAKE_BINARY_DIR}/test/benchgc/setup.py COPYONLY)
24+
25+
file(GLOB PYTHON_SCRIPTS "*.py")
26+
foreach(PY_SCRIPT ${PYTHON_SCRIPTS})
27+
configure_file(${PY_SCRIPT} ${CMAKE_BINARY_DIR} COPYONLY)
28+
endforeach()
29+
30+
add_custom_target(benchgc
31+
COMMAND ${Python_EXECUTABLE} setup.py bdist_wheel
32+
WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/test/benchgc/"
33+
DEPENDS GcPythonModules)
34+
35+
add_subdirectory("src/benchgc")
36+
add_subdirectory("src/benchgc/ops")

‎test/benchgc/README.md

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# benchgc - benchmark tool for graph compiler
2+
3+
## Description
4+
5+
Benchgc is a tool used to verify the correctness and performance of graph compiler. Benchgc accepts MLIR files based on the OneDNN graph dialect as test cases and prepares test data for them. For correctness verification, Benchgc will use PyTorch as a reference for comparison.
6+
7+
## Prerequisite
8+
* python >= 3.11
9+
* torch >= 2.2
10+
* pybind11
11+
12+
## Build and install
13+
```
14+
# Please execute at the top level of the project
15+
16+
mkdir -p build
17+
cd build
18+
19+
cmake .. -DMLIR_DIR=$MLIR_PATH -DGC_TEST_ENABLE=ON -DGC_ENABLE_BINDINGS_PYTHON=ON -DGC_BENCH_ENABLE=ON
20+
make -j benchgc
21+
22+
python -m pip install test/benchgc/dist/benchgc-*.whl
23+
24+
```
25+
26+
## Synopsis
27+
```
28+
python -m benchgc [OPTIONS] [--mlir [FILE] --entry [FUNCTION] | --json [FILE]]
29+
```
30+
## Flags
31+
```
32+
--mlir [FILE]
33+
Required if --json is not provided. A mlir file describing the case
34+
--entry [FUNCTION]
35+
Required if --mlir is provided. A function name in the mlir file describing the entry
36+
--json [FILE]
37+
Required if --mlir is not provided. A json file describing the case.
38+
--seed [INT]
39+
Optional and default is 0. A random seed value to generate data filling. It is also used in reproducing the issue.
40+
--verbose [INT]
41+
Optional, default is 0 with no verbose. An integer value describes the verbose level.
42+
```

‎test/benchgc/cases/add.mlir

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
module {
2+
func.func @f32(%arg0: tensor<128x256xf32>, %arg1: tensor<128x256xf32>) -> tensor<128x256xf32> {
3+
%0 = onednn_graph.add %arg0, %arg1 : (tensor<128x256xf32>, tensor<128x256xf32>) -> tensor<128x256xf32>
4+
return %0 : tensor<128x256xf32>
5+
}
6+
}

‎test/benchgc/cases/mlp.mlir

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
module {
2+
func.func @bf16_layer2(%in: tensor<128x512xbf16>,
3+
%weight0: tensor<512x256xbf16>, %bias0: tensor<256xbf16>,
4+
%weight1: tensor<256x64xbf16>, %bias1: tensor<64xbf16>) -> tensor<128x64xbf16> {
5+
// layer 0
6+
%0 = onednn_graph.matmul %in, %weight0, %bias0 : (tensor<128x512xbf16>, tensor<512x256xbf16>, tensor<256xbf16>) -> tensor<128x256xbf16>
7+
%1 = onednn_graph.relu %0 : (tensor<128x256xbf16>) -> tensor<128x256xbf16>
8+
// layer 1
9+
%2 = onednn_graph.matmul %1, %weight1, %bias1 : (tensor<128x256xbf16>, tensor<256x64xbf16>, tensor<64xbf16>) -> tensor<128x64xbf16>
10+
%3 = onednn_graph.relu %2 : (tensor<128x64xbf16>) -> tensor<128x64xbf16>
11+
return %3 : tensor<128x64xbf16>
12+
}
13+
14+
func.func @bf16_layer5(%in: tensor<128x128xbf16>,
15+
%weight0: tensor<128x128xbf16>, %bias0: tensor<128xbf16>,
16+
%weight1: tensor<128x128xbf16>, %bias1: tensor<128xbf16>,
17+
%weight2: tensor<128x128xbf16>, %bias2: tensor<128xbf16>,
18+
%weight3: tensor<128x128xbf16>, %bias3: tensor<128xbf16>,
19+
%weight4: tensor<128x128xbf16>, %bias4: tensor<128xbf16>) -> tensor<128x128xbf16> {
20+
// layer 0
21+
%0 = onednn_graph.matmul %in, %weight0, %bias0 : (tensor<128x128xbf16>, tensor<128x128xbf16>, tensor<128xbf16>) -> tensor<128x128xbf16>
22+
%1 = onednn_graph.relu %0 : (tensor<128x128xbf16>) -> tensor<128x128xbf16>
23+
// layer 1
24+
%2 = onednn_graph.matmul %1, %weight1, %bias1 : (tensor<128x128xbf16>, tensor<128x128xbf16>, tensor<128xbf16>) -> tensor<128x128xbf16>
25+
%3 = onednn_graph.relu %2 : (tensor<128x128xbf16>) -> tensor<128x128xbf16>
26+
// layer 2
27+
%4 = onednn_graph.matmul %3, %weight2, %bias2 : (tensor<128x128xbf16>, tensor<128x128xbf16>, tensor<128xbf16>) -> tensor<128x128xbf16>
28+
%5 = onednn_graph.relu %4 : (tensor<128x128xbf16>) -> tensor<128x128xbf16>
29+
// layer 3
30+
%6 = onednn_graph.matmul %5, %weight3, %bias3 : (tensor<128x128xbf16>, tensor<128x128xbf16>, tensor<128xbf16>) -> tensor<128x128xbf16>
31+
%7 = onednn_graph.relu %6 : (tensor<128x128xbf16>) -> tensor<128x128xbf16>
32+
// layer 4
33+
%8 = onednn_graph.matmul %7, %weight4, %bias4 : (tensor<128x128xbf16>, tensor<128x128xbf16>, tensor<128xbf16>) -> tensor<128x128xbf16>
34+
%9 = onednn_graph.relu %8 : (tensor<128x128xbf16>) -> tensor<128x128xbf16>
35+
return %9 : tensor<128x128xbf16>
36+
}
37+
}

‎test/benchgc/setup.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
################################################################################
2+
# Copyright 2024 Intel Corporation
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
################################################################################
16+
17+
import setuptools
18+
19+
setuptools.setup(
20+
name="benchgc",
21+
description="benchmark tool for graph compiler",
22+
package_dir={
23+
"benchgc": "src/benchgc",
24+
"gc_mlir": "../../python_packages/gc_mlir_core/gc_mlir",
25+
},
26+
packages=setuptools.find_packages("src")
27+
+ setuptools.find_packages("../../python_packages/gc_mlir_core"),
28+
package_data={"gc_mlir": ["_mlir_libs/*.so"]},
29+
install_requires=["torch", "numpy"],
30+
)

‎test/benchgc/src/benchgc/CMakeLists.txt

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
################################################################################
2+
# Copyright (C) 2024 Intel Corporation
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing,
11+
# software distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions
14+
# and limitations under the License.
15+
# SPDX-License-Identifier: Apache-2.0
16+
################################################################################
17+
18+
19+
file(GLOB PYTHON_SCRIPTS "*.py")
20+
foreach(PY_SCRIPT ${PYTHON_SCRIPTS})
21+
configure_file(${PY_SCRIPT} ${CMAKE_BINARY_DIR}/test/benchgc/src/benchgc/ COPYONLY)
22+
endforeach()

‎test/benchgc/src/benchgc/__init__.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
################################################################################
2+
# Copyright 2024 Intel Corporation
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
################################################################################
16+
17+
import sys
18+
import pathlib
19+
20+
# add the path to $PYTHONPATH where pysc shared object is located
21+
sys.path.append(pathlib.Path(__file__).parent.resolve().__str__())

‎test/benchgc/src/benchgc/__main__.py

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
################################################################################
2+
# Copyright 2024 Intel Corporation
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
################################################################################
16+
17+
18+
import sys
19+
import argparse
20+
import gc_mlir.ir
21+
import gc_mlir.dialects.onednn_graph
22+
from . import graph, runner, gapi, util
23+
24+
try:
25+
parser = argparse.ArgumentParser(prog="benchmark tool for graph compiler")
26+
parser.add_argument("--mlir", default=None, required=False, help="a mlir case file", type=str)
27+
parser.add_argument("--entry", default=None, required=False, help="main entry function", type=str)
28+
parser.add_argument(
29+
"--json",
30+
required=False,
31+
default=None,
32+
help="a json file case file",
33+
type=str,
34+
)
35+
parser.add_argument(
36+
"--seed",
37+
required=False,
38+
default=0,
39+
type=int,
40+
help="a seed value to generate data filling",
41+
)
42+
parser.add_argument(
43+
"--verbose",
44+
type=int,
45+
default=util.NO_VERBOSE,
46+
help="verbose level",
47+
choices=[
48+
util.NO_VERBOSE,
49+
util.COMPARE_VERBOSE,
50+
util.ERROR_OUTPUT_VERBOSE,
51+
util.OUTPUT_VERBOSE,
52+
util.INPUT_VERBOSE,
53+
],
54+
)
55+
56+
args = parser.parse_args()
57+
util.set_seed(args.seed)
58+
except argparse.ArgumentError:
59+
sys.stderr.write("Argument parse failed\n")
60+
sys.exit(1)
61+
62+
if args.mlir is not None:
63+
with open(args.mlir, "r") as mlir_file:
64+
with gc_mlir.ir.Context() as ctx:
65+
gc_mlir.dialects.onednn_graph.register_dialect()
66+
module = gc_mlir.ir.Module.parse(mlir_file.read())
67+
mlir_graph = gapi.MLIRGraph(module)
68+
graph_object = mlir_graph.convert_to_json('"' + args.entry + '"')
69+
json_graph = gapi.Graph(graph_object)
70+
ref_graph = graph.Graph(json_graph)
71+
ref_graph.prepare_input(args.verbose)
72+
ref_runner = runner.RefRunner(ref_graph)
73+
ref_runner.execute()
74+
elif args.json is not None:
75+
# TODO
76+
pass
77+
else:
78+
raise Exception("No mlir or json case provided")

0 commit comments

Comments
 (0)