Skip to content

Commit cec69bc

Browse files
JehandadKhanChao Liu
JehandadKhan
and
Chao Liu
authored
Add host API (ROCm#220)
* Add host API * manually rebase on develop * clean * manually rebase on develop * exclude tests from all target * address review comments * update client app name * fix missing lib name * clang-format update * refactor * refactor * refactor * refactor * refactor * fix test issue * refactor * refactor * refactor * upate cmake and readme Co-authored-by: Chao Liu <chao.liu2@amd.com>
1 parent 0f912e2 commit cec69bc

131 files changed

Lines changed: 1664 additions & 1087 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CMakeLists.txt‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
2727
set(CMAKE_CXX_EXTENSIONS OFF)
2828
message("CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}")
2929

30+
option(CK_TIME_KERNEL "Turning off will disable kernel timing globally" ON)
31+
3032
## OpenMP
3133
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
3234
# workaround issue hipcc in rocm3.5 cannot find openmp
@@ -72,8 +74,9 @@ message(STATUS "Build with HIP ${HIP_VERSION}")
7274

7375

7476
rocm_create_package(
75-
NAME CK-${CK_BACKEND}
77+
NAME composablekernel
7678
DESCRIPTION "High Performance Composable Kernel for AMD GPUs"
79+
MAINTAINER "MIOpen Kernels Dev Team <dl.MIOpen@amd.com>"
7780
LDCONFIG
7881
)
7982

@@ -226,15 +229,14 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
226229
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
227230
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
228231

229-
configure_file("${PROJECT_SOURCE_DIR}/include/ck/hip_version.hpp.in" "${PROJECT_BINARY_DIR}/include/ck/hip_version.hpp")
232+
configure_file("${PROJECT_SOURCE_DIR}/include/ck/options.hpp.in" "${PROJECT_BINARY_DIR}/include/ck/options.hpp")
230233

231234
include_directories(BEFORE
232235
${PROJECT_SOURCE_DIR}/include
233236
${PROJECT_BINARY_DIR}/include
234237
${PROJECT_SOURCE_DIR}/library/include
235238
)
236239

237-
include(googletest)
238240

239241
SET(BUILD_DEV ON CACHE BOOL "BUILD_DEV")
240242
if(BUILD_DEV)
@@ -247,3 +249,25 @@ add_subdirectory(library)
247249
add_subdirectory(example)
248250
add_subdirectory(test)
249251
add_subdirectory(profiler)
252+
253+
#Create an interface target for the include only files and call it "composablekernels"
254+
include(CMakePackageConfigHelpers)
255+
256+
set(version 1.0.0)
257+
write_basic_package_version_file(
258+
"${CMAKE_CURRENT_BINARY_DIR}/composable_kernelConfigVersion.cmake"
259+
VERSION "${version}"
260+
COMPATIBILITY AnyNewerVersion
261+
)
262+
263+
configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in
264+
"${CMAKE_CURRENT_BINARY_DIR}/composable_kernelConfig.cmake"
265+
INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/composable_kernel
266+
NO_CHECK_REQUIRED_COMPONENTS_MACRO
267+
)
268+
269+
install(FILES
270+
"${CMAKE_CURRENT_BINARY_DIR}/composable_kernelConfig.cmake"
271+
"${CMAKE_CURRENT_BINARY_DIR}/composable_kernelConfigVersion.cmake"
272+
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/composable_kernel
273+
)

‎Config.cmake.in‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
@PACKAGE_INIT@
2+
3+
set(_composable_kernel_supported_components device_operations host_tensor)
4+
5+
foreach(_comp ${composable_kernel_FIND_COMPONENTS})
6+
if(NOT _comp IN_LIST _composable_kernel_supported_components)
7+
set(composable_kernel_FOUND False)
8+
set(composable_kernel_NOT_FOUND_MESSAGE "Unsupported component: ${_comp}")
9+
endif()
10+
include("${CMAKE_CURRENT_LIST_DIR}/composable_kernel${_comp}Targets.cmake")
11+
endforeach()

‎Dockerfile‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,32 +11,23 @@ ARG DEB_ROCM_REPO=http://repo.radeon.com/rocm/apt/.apt_$ROCMVERSION/
1111
RUN apt-get update
1212
RUN apt-get install -y wget gnupg
1313
RUN wget -qO - http://repo.radeon.com/rocm/rocm.gpg.key | apt-key add -
14-
RUN if ! [ -z $OSDB_BKC_VERSION ]; then \
15-
echo "Using BKC VERISION: $OSDB_BKC_VERSION";\
16-
sh -c "echo deb [arch=amd64 trusted=yes] http://compute-artifactory.amd.com/artifactory/list/rocm-osdb-deb/ compute-rocm-dkms-no-npi-hipclang ${OSDB_BKC_VERSION} > /etc/apt/sources.list.d/rocm.list" ;\
17-
cat /etc/apt/sources.list.d/rocm.list;\
18-
else \
19-
sh -c "echo deb [arch=amd64] $DEB_ROCM_REPO ubuntu main > /etc/apt/sources.list.d/rocm.list" ;\
20-
fi
14+
RUN sh -c "echo deb [arch=amd64] $DEB_ROCM_REPO ubuntu main > /etc/apt/sources.list.d/rocm.list"
2115
RUN wget --no-check-certificate -qO - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add -
2216
RUN sh -c "echo deb https://apt.kitware.com/ubuntu/ bionic main | tee -a /etc/apt/sources.list"
2317

2418
# ADD requirements.txt requirements.txt
2519
# Install dependencies
2620
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \
2721
apt-utils \
28-
sshpass \
2922
build-essential \
3023
cmake-data=3.15.1-0kitware1 \
3124
cmake=3.15.1-0kitware1 \
3225
curl \
33-
doxygen \
3426
g++ \
3527
gdb \
3628
git \
3729
hip-rocclr \
3830
jq \
39-
lcov \
4031
libelf-dev \
4132
libncurses5-dev \
4233
libnuma-dev \
@@ -62,8 +53,6 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-
6253
apt-get clean && \
6354
rm -rf /var/lib/apt/lists/*
6455

65-
# RUN pip3 install --default-timeout=100000 -r requirements.txt
66-
6756
# Setup ubsan environment to printstacktrace
6857
RUN ln -s /usr/bin/llvm-symbolizer-3.8 /usr/local/bin/llvm-symbolizer
6958
ENV UBSAN_OPTIONS=print_stacktrace=1
@@ -92,5 +81,3 @@ ADD rbuild.ini /rbuild.ini
9281
ADD dev-requirements.txt dev-requirements.txt
9382
RUN rbuild prepare -s develop -d $PREFIX
9483
RUN groupadd -f render
95-
# RUN cget install -f min-requirements.txt
96-
# RUN CXXFLAGS='-isystem $PREFIX/include' cget install -f ./mlir-requirements.txt

‎Jenkinsfile‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,7 @@ pipeline {
320320
{
321321
agent{ label rocmnode("gfx908")}
322322
environment{
323-
setup_args = """ -D CMAKE_CXX_FLAGS="--offload-arch=gfx908 -O3 " -DBUILD_DEV=On """
323+
setup_args = """ -D CMAKE_CXX_FLAGS=" --offload-arch=gfx900 --offload-arch=gfx906 --offload-arch=gfx908 --offload-arch=gfx90a -O3 " -DBUILD_DEV=On """
324324
}
325325
steps{
326326
buildHipClangJobAndReboot(setup_args:setup_args, config_targets: "check", no_reboot:true, build_type: 'Release')
@@ -341,6 +341,23 @@ pipeline {
341341

342342
}
343343
}
344+
stage("Client App")
345+
{
346+
parallel
347+
{
348+
stage("Run Client App")
349+
{
350+
agent{ label rocmnode("gfx908")}
351+
environment{
352+
setup_args = """ -D -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=../install CMAKE_CXX_FLAGS="--offload-arch=gfx908 -O3 " """
353+
execute_args = """ cd ../test/client_app && rm -rf build && mkdir build && cd build && cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" .. && make """
354+
}
355+
steps{
356+
buildHipClangJobAndReboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local')
357+
}
358+
}
359+
}
360+
}
344361
stage("Performance Tests")
345362
{
346363
parallel

‎README.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,13 @@ Instructions for running each individual examples are under ```example/```
4343
make -j ckProfiler
4444
```
4545
Instructions for running ckProfiler are under ```profiler/```
46+
47+
48+
## Caveat
49+
### Kernel Timing and Verification
50+
CK's own kernel timer will warn up kernel once, and then run it multiple times
51+
to get average kernel time. For some kernels that use atomic add, this will cause
52+
output buffer to be accumulated multiple times, causing verfication failure.
53+
To work around it, do not use CK's own timer and do verification at the same time.
54+
CK's own timer and verification in each example and ckProfiler can be enabled or
55+
disabled from command line.

‎cmake/googletest.cmake‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ list(APPEND GTEST_CMAKE_CXX_FLAGS
1919
-Wno-zero-as-null-pointer-constant
2020
-Wno-unused-member-function
2121
-Wno-comma
22+
-Wno-old-style-cast
2223
)
2324
message(STATUS "Suppressing googltest warnings with flags: ${GTEST_CMAKE_CXX_FLAGS}")
2425

@@ -35,4 +36,4 @@ FetchContent_MakeAvailable(googletest)
3536
target_compile_options(gtest PRIVATE ${GTEST_CMAKE_CXX_FLAGS})
3637
target_compile_options(gtest_main PRIVATE ${GTEST_CMAKE_CXX_FLAGS})
3738
target_compile_options(gmock PRIVATE ${GTEST_CMAKE_CXX_FLAGS})
38-
39+
target_compile_options(gmock_main PRIVATE ${GTEST_CMAKE_CXX_FLAGS})

‎example/01_gemm/gemm_xdl_bf16.cpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,9 @@ using ReferenceGemmInstance = ck::tensor_operation::host::
8888

8989
int main(int argc, char* argv[])
9090
{
91-
bool do_verification = 0;
92-
int init_method = 0;
93-
int nrepeat = 5;
91+
bool do_verification = true;
92+
int init_method = 1;
93+
bool time_kernel = false;
9494

9595
// GEMM shape
9696
ck::index_t M = 3840;
@@ -105,13 +105,13 @@ int main(int argc, char* argv[])
105105
{
106106
do_verification = std::stoi(argv[1]);
107107
init_method = std::stoi(argv[2]);
108-
nrepeat = std::stoi(argv[3]);
108+
time_kernel = std::stoi(argv[3]);
109109
}
110110
else if(argc == 10)
111111
{
112112
do_verification = std::stoi(argv[1]);
113113
init_method = std::stoi(argv[2]);
114-
nrepeat = std::stoi(argv[3]);
114+
time_kernel = std::stoi(argv[3]);
115115

116116
M = std::stoi(argv[4]);
117117
N = std::stoi(argv[5]);
@@ -125,7 +125,7 @@ int main(int argc, char* argv[])
125125
{
126126
printf("arg1: verification (0=no, 1=yes)\n");
127127
printf("arg2: initialization (0=no init, 1=integer value, 2=decimal value)\n");
128-
printf("arg3: run kernel # of times (>1)\n");
128+
printf("arg3: time kernel (0=n0, 1=yes)\n");
129129
printf("arg4 to 9: M (256x), N(128x), K(32x), StrideA, StrideB, StrideC\n");
130130
exit(0);
131131
}
@@ -198,7 +198,7 @@ int main(int argc, char* argv[])
198198
"not support this GEMM problem");
199199
}
200200

201-
float ave_time = invoker.Run(argument, nrepeat);
201+
float ave_time = invoker.Run(argument, StreamConfig{nullptr, time_kernel});
202202

203203
std::size_t flop = std::size_t(2) * M * N * K;
204204
std::size_t num_btype =

‎example/01_gemm/gemm_xdl_fp16.cpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,9 @@ using ReferenceGemmInstance = ck::tensor_operation::host::
5656

5757
int main(int argc, char* argv[])
5858
{
59-
bool do_verification = 0;
60-
int init_method = 0;
61-
int nrepeat = 5;
59+
bool do_verification = true;
60+
int init_method = 1;
61+
bool time_kernel = false;
6262

6363
// GEMM shape
6464
ck::index_t M = 3840;
@@ -73,13 +73,13 @@ int main(int argc, char* argv[])
7373
{
7474
do_verification = std::stoi(argv[1]);
7575
init_method = std::stoi(argv[2]);
76-
nrepeat = std::stoi(argv[3]);
76+
time_kernel = std::stoi(argv[3]);
7777
}
7878
else if(argc == 10)
7979
{
8080
do_verification = std::stoi(argv[1]);
8181
init_method = std::stoi(argv[2]);
82-
nrepeat = std::stoi(argv[3]);
82+
time_kernel = std::stoi(argv[3]);
8383

8484
M = std::stoi(argv[4]);
8585
N = std::stoi(argv[5]);
@@ -93,7 +93,7 @@ int main(int argc, char* argv[])
9393
{
9494
printf("arg1: verification (0=no, 1=yes)\n");
9595
printf("arg2: initialization (0=no init, 1=integer value, 2=decimal value)\n");
96-
printf("arg3: run kernel # of times (>1)\n");
96+
printf("arg3: time kernel (0=n0, 1=yes)\n");
9797
printf("arg4 to 9: M (256x), N(128x), K(32x), StrideA, StrideB, StrideC\n");
9898
exit(0);
9999
}
@@ -171,7 +171,7 @@ int main(int argc, char* argv[])
171171
"not support this GEMM problem");
172172
}
173173

174-
float ave_time = invoker.Run(argument, nrepeat);
174+
float ave_time = invoker.Run(argument, StreamConfig{nullptr, time_kernel});
175175

176176
std::size_t flop = std::size_t(2) * M * N * K;
177177
std::size_t num_btype =

‎example/01_gemm/gemm_xdl_int8.cpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,9 @@ using ReferenceGemmInstance = ck::tensor_operation::host::
8383

8484
int main(int argc, char* argv[])
8585
{
86-
bool do_verification = 0;
87-
int init_method = 0;
88-
int nrepeat = 5;
86+
bool do_verification = true;
87+
int init_method = 1;
88+
bool time_kernel = false;
8989

9090
// GEMM shape
9191
ck::index_t M = 3840;
@@ -100,13 +100,13 @@ int main(int argc, char* argv[])
100100
{
101101
do_verification = std::stoi(argv[1]);
102102
init_method = std::stoi(argv[2]);
103-
nrepeat = std::stoi(argv[3]);
103+
time_kernel = std::stoi(argv[3]);
104104
}
105105
else if(argc == 10)
106106
{
107107
do_verification = std::stoi(argv[1]);
108108
init_method = std::stoi(argv[2]);
109-
nrepeat = std::stoi(argv[3]);
109+
time_kernel = std::stoi(argv[3]);
110110

111111
M = std::stoi(argv[4]);
112112
N = std::stoi(argv[5]);
@@ -120,7 +120,7 @@ int main(int argc, char* argv[])
120120
{
121121
printf("arg1: verification (0=no, 1=yes)\n");
122122
printf("arg2: initialization (0=no init, 1=integer value, 2=decimal value)\n");
123-
printf("arg3: run kernel # of times (>1)\n");
123+
printf("arg3: time kernel (0=n0, 1=yes)\n");
124124
printf("arg4 to 9: M (256x), N(128x), K(32x), StrideA, StrideB, StrideC\n");
125125
exit(0);
126126
}
@@ -194,7 +194,7 @@ int main(int argc, char* argv[])
194194
"not support this GEMM problem");
195195
}
196196

197-
float ave_time = invoker.Run(argument, nrepeat);
197+
float ave_time = invoker.Run(argument, StreamConfig{nullptr, time_kernel});
198198

199199
std::size_t flop = std::size_t(2) * M * N * K;
200200
std::size_t num_btype =

‎example/02_gemm_alpha_beta/gemm_xdl_alpha_beta.cpp‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -86,9 +86,9 @@ using ReferenceGemmInstance = ck::tensor_operation::host::ReferenceGemmBias2D<AD
8686

8787
int main(int argc, char* argv[])
8888
{
89-
bool do_verification = 0;
90-
int init_method = 0;
91-
int nrepeat = 5;
89+
bool do_verification = true;
90+
int init_method = 1;
91+
bool time_kernel = false;
9292

9393
// GEMM shape
9494
ck::index_t M = 3840;
@@ -106,13 +106,13 @@ int main(int argc, char* argv[])
106106
{
107107
do_verification = std::stoi(argv[1]);
108108
init_method = std::stoi(argv[2]);
109-
nrepeat = std::stoi(argv[3]);
109+
time_kernel = std::stoi(argv[3]);
110110
}
111111
else if(argc == 6)
112112
{
113113
do_verification = std::stoi(argv[1]);
114114
init_method = std::stoi(argv[2]);
115-
nrepeat = std::stoi(argv[3]);
115+
time_kernel = std::stoi(argv[3]);
116116

117117
alpha = std::stof(argv[4]);
118118
beta = std::stof(argv[5]);
@@ -121,7 +121,7 @@ int main(int argc, char* argv[])
121121
{
122122
do_verification = std::stoi(argv[1]);
123123
init_method = std::stoi(argv[2]);
124-
nrepeat = std::stoi(argv[3]);
124+
time_kernel = std::stoi(argv[3]);
125125

126126
M = std::stoi(argv[4]);
127127
N = std::stoi(argv[5]);
@@ -138,7 +138,7 @@ int main(int argc, char* argv[])
138138
{
139139
printf("arg1: verification (0=no, 1=yes)\n");
140140
printf("arg2: initialization (0=no init, 1=integer value, 2=decimal value)\n");
141-
printf("arg3: run kernel # of times (>1)\n");
141+
printf("arg3: time kernel (0=n0, 1=yes)\n");
142142
printf("arg4 to 9: M (256x), N(128x), K(32x), StrideA, StrideB, StrideC, alpha, beta\n");
143143
exit(0);
144144
}
@@ -216,7 +216,7 @@ int main(int argc, char* argv[])
216216
"not support this GEMM problem");
217217
}
218218

219-
float ave_time = invoker.Run(argument, nrepeat);
219+
float ave_time = invoker.Run(argument, StreamConfig{nullptr, time_kernel});
220220

221221
std::size_t flop = std::size_t(2) * M * N * K;
222222
std::size_t num_btype =

0 commit comments

Comments
 (0)