[WIP] Introduce benchgc, a benchmark tool for graph compiler

.github/workflows/python-format.yml

@@ -0,0 +1,28 @@
+name: Python Format
+
+on:
+  pull_request:
+    paths:
+      - '**.py'
+
+jobs:
+  python_format:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout base version
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 100
+          ref: ${{ github.event.pull_request.base.sha }}
+
+      - name: checkout head version
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 100
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: install darker
+        run: "python3 -m pip install darker"
+
+      - name: check python format
+        run: "python3 -m darker --check --diff -r `git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }}`...HEAD ."

CMakeLists.txt

@@ -28,6 +28,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 option(GC_LEGACY_ENABLE ON)
 option(GC_TEST_ENABLE "Build the tests" ON)
 option(GC_ENABLE_BINDINGS_PYTHON "Enable Graph Complier Python Binding" ON)
+option(GC_BENCH_ENABLE "Build benchgc. Only available when GC_TEST_ENABLE and GC_ENABLE_BINDINGS_PYTHON is enabled" ON)
 option(GC_DEV_LINK_LLVM_DYLIB "Link dynamic libraries of LLVM and MLIR. For developers only. Do not use it in packing the library." OFF)
 
 if(GC_LEGACY_ENABLE)
@@ -58,6 +59,7 @@ endif()
 
 if(GC_ENABLE_BINDINGS_PYTHON)
   include(MLIRDetectPythonEnv)
+  find_package(Python 3.11 REQUIRED COMPONENTS Interpreter NumPy REQUIRED)
   mlir_configure_python_dev_packages()
 endif()
 
@@ -94,6 +96,15 @@ if(GC_ENABLE_BINDINGS_PYTHON)
   add_subdirectory(python)
 endif()
 
+
+if(GC_ENABLE_BINDINGS_PYTHON AND GC_TEST_ENABLE AND GC_BENCH_ENABLE)
+  message(STATUS "Enabling Benchgc")
+elseif()
+  message(STATUS "Disabling Benchgc")
+  set(GC_BENCH_ENABLE OFF CACHE BOOL "" FORCE)
+endif()
+
+
 set(GC_LIB_LINKED_LIBS
         MLIRLinalgx
         MLIRMicrokernel

README.md

@@ -64,4 +64,5 @@ Graph Compiler supports the following build-time options.
 | GC_TEST_ENABLE                  | **ON**, OFF                            | Controls building the tests                                                            |
 | GC_DEV_LINK_LLVM_DYLIB          | ON, **OFF**                            | Controls dynamic link LLVM/MLIR libraries, mainly for developer                        |
 | GC_ENABLE_BINDINGS_PYTHON       | **ON**, OFF                            | Controls building the Python API                                                       |
+| GC_BENCH_ENABLE                 | **ON**, OFF                            | Controls building benchgc. The configuration will only take effect when both GC_ENABLE_BINDING_PYTHON and GC_TEST_ENABLE are ON.                           |
 

test/CMakeLists.txt

@@ -40,6 +40,10 @@ if(GC_ENABLE_BINDINGS_PYTHON)
         list(APPEND GC_OPT_TEST_DEPENDS GcPythonModules)
 endif()
 
+if(GC_BENCH_ENABLE)
+        add_subdirectory(benchgc)
+endif()
+
 add_lit_testsuite(gc-check "Running the regression tests"
         ${CMAKE_CURRENT_BINARY_DIR}
         DEPENDS ${GC_OPT_TEST_DEPENDS}

test/benchgc/.gitignore

@@ -0,0 +1,4 @@
+dist/
+src/benchgc.egg-info/
+build
+benchgc.egg-info/

test/benchgc/CMakeLists.txt

@@ -0,0 +1,36 @@
+################################################################################
+# Copyright (C) 2024 Intel Corporation
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
+################################################################################
+
+if(NOT GC_BENCH_ENABLE)
+    message(STATUS "Benchgc is not enabled")
+    return()
+endif()
+
+configure_file(setup.py ${CMAKE_BINARY_DIR}/test/benchgc/setup.py COPYONLY)
+
+file(GLOB PYTHON_SCRIPTS "*.py")
+foreach(PY_SCRIPT ${PYTHON_SCRIPTS})
+  configure_file(${PY_SCRIPT} ${CMAKE_BINARY_DIR} COPYONLY)
+endforeach()
+
+add_custom_target(benchgc
+    COMMAND ${Python_EXECUTABLE} setup.py bdist_wheel
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/test/benchgc/"
+    DEPENDS GcPythonModules) 
+
+add_subdirectory("src/benchgc")
+add_subdirectory("src/benchgc/ops")

test/benchgc/README.md

@@ -0,0 +1,42 @@
+# benchgc - benchmark tool for graph compiler
+
+## Description
+
+Benchgc is a tool used to verify the correctness and performance of graph compiler. Benchgc accepts MLIR files based on the OneDNN graph dialect as test cases and prepares test data for them. For correctness verification, Benchgc will use PyTorch as a reference for comparison.
+
+## Prerequisite
+* python >= 3.11
+* torch >= 2.2
+* pybind11
+
+## Build and install
+```
+# Please execute at the top level of the project
+
+mkdir -p build
+cd build
+
+cmake .. -DMLIR_DIR=$MLIR_PATH -DGC_TEST_ENABLE=ON -DGC_ENABLE_BINDINGS_PYTHON=ON -DGC_BENCH_ENABLE=ON
+make -j benchgc
+
+python -m pip install test/benchgc/dist/benchgc-*.whl
+
+```
+
+## Synopsis
+```
+python -m benchgc [OPTIONS] [--mlir [FILE] --entry [FUNCTION] | --json [FILE]]
+```
+## Flags
+```
+--mlir [FILE]
+    Required if --json is not provided. A mlir file describing the case
+--entry [FUNCTION]
+    Required if --mlir is provided. A function name in the mlir file describing the entry
+--json [FILE]
+    Required if --mlir is not provided. A json file describing the case.
+--seed [INT]
+    Optional and default is 0. A random seed value to generate data filling. It is also used in reproducing the issue.
+--verbose [INT]
+    Optional, default is 0 with no verbose. An integer value describes the verbose level.
+```

test/benchgc/cases/add.mlir

@@ -0,0 +1,6 @@
+module {
+  func.func @f32(%arg0: tensor<128x256xf32>, %arg1: tensor<128x256xf32>) -> tensor<128x256xf32> {
+    %0 = onednn_graph.add %arg0, %arg1 : (tensor<128x256xf32>, tensor<128x256xf32>) -> tensor<128x256xf32>
+    return %0 : tensor<128x256xf32>
+  }
+}

test/benchgc/cases/mlp.mlir

@@ -0,0 +1,37 @@
+module {
+  func.func @bf16_layer2(%in: tensor<128x512xbf16>, 
+                 %weight0: tensor<512x256xbf16>, %bias0: tensor<256xbf16>, 
+                 %weight1: tensor<256x64xbf16>, %bias1: tensor<64xbf16>) -> tensor<128x64xbf16> {
+    // layer 0
+    %0 = onednn_graph.matmul %in, %weight0, %bias0 : (tensor<128x512xbf16>, tensor<512x256xbf16>, tensor<256xbf16>) -> tensor<128x256xbf16>
+    %1 = onednn_graph.relu %0 : (tensor<128x256xbf16>) -> tensor<128x256xbf16>
+    // layer 1
+    %2 = onednn_graph.matmul %1, %weight1, %bias1 : (tensor<128x256xbf16>, tensor<256x64xbf16>, tensor<64xbf16>) -> tensor<128x64xbf16>
+    %3 = onednn_graph.relu %2 : (tensor<128x64xbf16>) -> tensor<128x64xbf16>
+    return %3 : tensor<128x64xbf16>
+  }
+
+  func.func @bf16_layer5(%in: tensor<128x128xbf16>, 
+                 %weight0: tensor<128x128xbf16>, %bias0: tensor<128xbf16>, 
+                 %weight1: tensor<128x128xbf16>, %bias1: tensor<128xbf16>, 
+                 %weight2: tensor<128x128xbf16>, %bias2: tensor<128xbf16>, 
+                 %weight3: tensor<128x128xbf16>, %bias3: tensor<128xbf16>, 
+                 %weight4: tensor<128x128xbf16>, %bias4: tensor<128xbf16>) -> tensor<128x128xbf16> {
+    // layer 0
+    %0 = onednn_graph.matmul %in, %weight0, %bias0 : (tensor<128x128xbf16>, tensor<128x128xbf16>, tensor<128xbf16>) -> tensor<128x128xbf16>
+    %1 = onednn_graph.relu %0 : (tensor<128x128xbf16>) -> tensor<128x128xbf16>
+    // layer 1
+    %2 = onednn_graph.matmul %1, %weight1, %bias1 : (tensor<128x128xbf16>, tensor<128x128xbf16>, tensor<128xbf16>) -> tensor<128x128xbf16>
+    %3 = onednn_graph.relu %2 : (tensor<128x128xbf16>) -> tensor<128x128xbf16>
+    // layer 2
+    %4 = onednn_graph.matmul %3, %weight2, %bias2 : (tensor<128x128xbf16>, tensor<128x128xbf16>, tensor<128xbf16>) -> tensor<128x128xbf16>
+    %5 = onednn_graph.relu %4 : (tensor<128x128xbf16>) -> tensor<128x128xbf16>
+    // layer 3
+    %6 = onednn_graph.matmul %5, %weight3, %bias3 : (tensor<128x128xbf16>, tensor<128x128xbf16>, tensor<128xbf16>) -> tensor<128x128xbf16>
+    %7 = onednn_graph.relu %6 : (tensor<128x128xbf16>) -> tensor<128x128xbf16>
+    // layer 4
+    %8 = onednn_graph.matmul %7, %weight4, %bias4 : (tensor<128x128xbf16>, tensor<128x128xbf16>, tensor<128xbf16>) -> tensor<128x128xbf16>
+    %9 = onednn_graph.relu %8 : (tensor<128x128xbf16>) -> tensor<128x128xbf16>
+    return %9 : tensor<128x128xbf16>
+  }
+}

test/benchgc/setup.py

@@ -0,0 +1,30 @@
+################################################################################
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import setuptools
+
+setuptools.setup(
+    name="benchgc",
+    description="benchmark tool for graph compiler",
+    package_dir={
+        "benchgc": "src/benchgc",
+        "gc_mlir": "../../python_packages/gc_mlir_core/gc_mlir",
+    },
+    packages=setuptools.find_packages("src")
+    + setuptools.find_packages("../../python_packages/gc_mlir_core"),
+    package_data={"gc_mlir": ["_mlir_libs/*.so"]},
+    install_requires=["torch", "numpy"],
+)

test/benchgc/src/benchgc/CMakeLists.txt

@@ -0,0 +1,22 @@
+################################################################################
+# Copyright (C) 2024 Intel Corporation
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
+################################################################################
+
+
+file(GLOB PYTHON_SCRIPTS "*.py")
+foreach(PY_SCRIPT ${PYTHON_SCRIPTS})
+  configure_file(${PY_SCRIPT} ${CMAKE_BINARY_DIR}/test/benchgc/src/benchgc/ COPYONLY)
+endforeach()

test/benchgc/src/benchgc/__init__.py

@@ -0,0 +1,21 @@
+################################################################################
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import sys
+import pathlib
+
+# add the path to $PYTHONPATH where pysc shared object is located
+sys.path.append(pathlib.Path(__file__).parent.resolve().__str__())

test/benchgc/src/benchgc/__main__.py

@@ -0,0 +1,78 @@
+################################################################################
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+
+import sys
+import argparse
+import gc_mlir.ir
+import gc_mlir.dialects.onednn_graph
+from . import graph, runner, gapi, util
+
+try:
+    parser = argparse.ArgumentParser(prog="benchmark tool for graph compiler")
+    parser.add_argument("--mlir", default=None, required=False, help="a mlir case file", type=str)
+    parser.add_argument("--entry", default=None, required=False, help="main entry function", type=str)
+    parser.add_argument(
+        "--json",
+        required=False,
+        default=None,
+        help="a json file case file",
+        type=str,
+    )
+    parser.add_argument(
+        "--seed",
+        required=False,
+        default=0,
+        type=int,
+        help="a seed value to generate data filling",
+    )
+    parser.add_argument(
+        "--verbose",
+        type=int,
+        default=util.NO_VERBOSE,
+        help="verbose level",
+        choices=[
+            util.NO_VERBOSE,
+            util.COMPARE_VERBOSE,
+            util.ERROR_OUTPUT_VERBOSE,
+            util.OUTPUT_VERBOSE,
+            util.INPUT_VERBOSE,
+        ],
+    )
+
+    args = parser.parse_args()
+    util.set_seed(args.seed)
+except argparse.ArgumentError:
+    sys.stderr.write("Argument parse failed\n")
+    sys.exit(1)
+
+if args.mlir is not None:
+    with open(args.mlir, "r") as mlir_file:
+        with gc_mlir.ir.Context() as ctx:
+            gc_mlir.dialects.onednn_graph.register_dialect()
+            module = gc_mlir.ir.Module.parse(mlir_file.read())
+            mlir_graph = gapi.MLIRGraph(module)
+            graph_object = mlir_graph.convert_to_json('"' + args.entry + '"')
+            json_graph = gapi.Graph(graph_object)
+            ref_graph = graph.Graph(json_graph)
+            ref_graph.prepare_input(args.verbose)
+            ref_runner = runner.RefRunner(ref_graph)
+            ref_runner.execute()
+elif args.json is not None:
+    # TODO 
+    pass
+else:
+    raise Exception("No mlir or json case provided")