Skip to content

Commit

Permalink
[Hexagon] Do not auto-build apps when building TVM (apache#9970)
Browse files Browse the repository at this point in the history
* [Hexagon] Do not auto-build apps when building TVM

The Hexagon cmakes have recently become unwieldy due to a complex
network of dependencies between various automatically built components.
This was in large part because of trying to automatically build some
apps, which then tried to build TVM runtimes again, but with their
own configurations.

This patch removes the ability to automatically build any Hexagon-
-related apps from the main TVM build. The following cmake options
are now deprecated:
  - `USE_HEXAGON_LAUNCHER`
  - `USE_HEXAGON_PROXY_RPC`

In order to build the binaries needed for HexagonLauncher from
tvm.contrib.hexagon:
  - Build TVM+runtime for x86, with codegen for Hexagon enabled.
    This can be done via `USE_HEXAGON_DEVICE=sim` or `target`.
  - Build Android runtime and tvm_rpc with `-DUSE_RPC=ON`,
    `-DUSE_CPP_RPC=ON`, and `-DUSE_HEXAGON_RPC=ON`.
  - Build Hexagon runtime with `-DUSE_HEXAGON_RPC=ON`, and
    `-DBUILD_STATIC_RUNTIME=ON`.

* Add README.md

* Restart CI

* Add optional variable to set output directory
  • Loading branch information
Krzysztof Parzyszek authored and ylc committed Feb 16, 2022
1 parent 4d8c5d6 commit 1dc9092
Show file tree
Hide file tree
Showing 6 changed files with 232 additions and 442 deletions.
22 changes: 13 additions & 9 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@ tvm_option(ROCM_PATH "The path to rocm" /opt/rocm)
tvm_option(USE_HEXAGON_DEVICE "Build with Hexagon device support in TVM runtime" OFF)
tvm_option(USE_HEXAGON_SDK "Path to the Hexagon SDK root (required for Hexagon support in TVM runtime or for building TVM runtime for Hexagon)" /path/to/sdk)
tvm_option(USE_HEXAGON_RPC "Enable Hexagon RPC using minRPC implementation over Android." OFF)
tvm_option(USE_HEXAGON_LAUNCHER "Build the Hexagon graph launcher application" OFF)
tvm_option(USE_HEXAGON_PROXY_RPC "Build the Hexagon Proxy RPC server application" OFF)
tvm_option(USE_RPC "Build with RPC" ON)
tvm_option(USE_THREADS "Build with thread support" ON)
tvm_option(USE_LLVM "Build with LLVM, can be set to specific llvm-config path" OFF)
Expand Down Expand Up @@ -339,7 +337,7 @@ if(BUILD_FOR_HEXAGON)
# runtime, since it would cause multiple definition errors with the
# static one.
if(NOT BUILD_STATIC_RUNTIME)
list(APPEND RUNTIME_SRCS src/runtime/hexagon/hexagon_posix.cc)
list(APPEND RUNTIME_SRCS src/runtime/hexagon/android/hexagon_posix.cc)
# Allow undefined symbols (there will be some from libc).
set(TVM_NO_UNDEFINED_SYMBOLS "")
endif()
Expand Down Expand Up @@ -580,12 +578,6 @@ else()
target_compile_definitions(tvm_libinfo_objs PRIVATE "USE_FALLBACK_STL_MAP=0")
endif(USE_FALLBACK_STL_MAP)

if(BUILD_FOR_HEXAGON)
# Wrap pthread_create to allow setting custom stack size.
set_property(TARGET tvm_runtime APPEND PROPERTY LINK_FLAGS
"-Wl,--wrap=pthread_create")
endif()

if(USE_THREADS AND NOT BUILD_FOR_HEXAGON)
message(STATUS "Build with thread support...")
set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
Expand Down Expand Up @@ -721,6 +713,18 @@ if(APPLE AND TVM_IS_DEBUG_BUILD)
)
endif()

if(BUILD_FOR_HEXAGON)
# Wrap pthread_create to allow setting custom stack size.
set_property(TARGET tvm_runtime APPEND PROPERTY LINK_FLAGS
"-Wl,--wrap=pthread_create")
# Link tvm_runtime into the RPC skel library. Make sure it's built
# as a part of the "runtime" target.
if(USE_HEXAGON_RPC)
target_link_libraries(hexagon_rpc_skel -Wl,--whole-archive tvm_runtime -Wl,--no-whole-archive)
add_dependencies(runtime hexagon_rpc_skel)
endif()
endif()

#Caches the build.
#Note that ccache-3.x doesn't support nvcc well, so CUDA kernels may never hit the cache and still
#need to be re-compiled every time. Using ccache 4.0+ can resolve this issue.
Expand Down
90 changes: 90 additions & 0 deletions apps/hexagon_api/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
cmake_minimum_required(VERSION 3.2)

project(hexagon_api)

include(ExternalProject)

# Required variables:
# ANDROID_ABI
# ANDROID_PLATFORM
# USE_ANDROID_TOOLCHAIN (Android toolchain .cmake file)
# USE_HEXAGON_ARCH
# USE_HEXAGON_SDK
# USE_HEXAGON_TOOLCHAIN (Path to Hexagon toolchain ending with "Tools")
# Optional variable:
# USE_OUTPUT_BINARY_DIR (Path to copy the output binaries to)

set(TVM_SOURCE_DIR "${CMAKE_SOURCE_DIR}/../..")

if(DEFINED USE_OUTPUT_BINARY_DIR)
set(HEXAGON_API_BINARY_DIR "${USE_OUTPUT_BINARY_DIR}")
else()
set(HEXAGON_API_BINARY_DIR "${CMAKE_BINARY_DIR}/hexagon_rpc")
endif()
file(MAKE_DIRECTORY ${HEXAGON_API_BINARY_DIR})

# Build Android binaries:
# - libtvm_runtime.so
# - tvm_rpc_android

ExternalProject_Add(android_tvm_runtime_rpc
SOURCE_DIR "${TVM_SOURCE_DIR}"
BUILD_COMMAND $(MAKE) runtime tvm_rpc
CMAKE_ARGS
"-DCMAKE_TOOLCHAIN_FILE=${USE_ANDROID_TOOLCHAIN}"
"-DANDROID_PLATFORM=${ANDROID_PLATFORM}"
"-DANDROID_ABI=${ANDROID_ABI}"
"-DUSE_HEXAGON_SDK=${USE_HEXAGON_SDK}"
"-DUSE_HEXAGON_ARCH=${USE_HEXAGON_ARCH}"
"-DCMAKE_CXX_STANDARD=14"
"-DUSE_LIBBACKTRACE=OFF"
"-DUSE_LLVM=OFF"
"-DUSE_RPC=ON"
"-DUSE_CPP_RPC=ON"
"-DUSE_HEXAGON_RPC=ON"
INSTALL_COMMAND ""
BUILD_ALWAYS ON
)
ExternalProject_Get_Property(android_tvm_runtime_rpc BINARY_DIR)
ExternalProject_Add_Step(android_tvm_runtime_rpc copy_runtime
COMMAND ${CMAKE_COMMAND} -E copy_if_different
${BINARY_DIR}/libtvm_runtime.so
${HEXAGON_API_BINARY_DIR}
DEPENDEES install
)
ExternalProject_Add_Step(android_tvm_runtime_rpc copy_rpc_server
COMMAND ${CMAKE_COMMAND} -E copy_if_different
${BINARY_DIR}/tvm_rpc
${HEXAGON_API_BINARY_DIR}/tvm_rpc_android
DEPENDEES install
)


# Build Hexagon binaries:
# - libhexagon_rpc_skel.so
# - libtvm_runtime.a

ExternalProject_Add(hexagon_tvm_runtime_rpc
SOURCE_DIR "${TVM_SOURCE_DIR}"
BUILD_COMMAND $(MAKE) runtime
CMAKE_ARGS
"-DCMAKE_C_COMPILER=${USE_HEXAGON_TOOLCHAIN}/bin/hexagon-clang"
"-DCMAKE_CXX_COMPILER=${USE_HEXAGON_TOOLCHAIN}/bin/hexagon-clang++"
"-DUSE_HEXAGON_SDK=${USE_HEXAGON_SDK}"
"-DUSE_HEXAGON_ARCH=${USE_HEXAGON_ARCH}"
"-DUSE_LIBBACKTRACE=OFF"
"-DUSE_RPC=OFF"
"-DUSE_HEXAGON_RPC=ON"
"-DBUILD_STATIC_RUNTIME=ON"
INSTALL_COMMAND ""
BUILD_ALWAYS ON
)
ExternalProject_Get_Property(hexagon_tvm_runtime_rpc BINARY_DIR)
ExternalProject_Add_Step(hexagon_tvm_runtime_rpc copy_binaries
COMMAND ${CMAKE_COMMAND} -E copy_if_different
${BINARY_DIR}/libtvm_runtime.a
${BINARY_DIR}/libhexagon_rpc_skel.so
${HEXAGON_API_BINARY_DIR}
DEPENDEES install
)

58 changes: 58 additions & 0 deletions apps/hexagon_api/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
<!--- Licensed to the Apache Software Foundation (ASF) under one -->
<!--- or more contributor license agreements. See the NOTICE file -->
<!--- distributed with this work for additional information -->
<!--- regarding copyright ownership. The ASF licenses this file -->
<!--- to you under the Apache License, Version 2.0 (the -->
<!--- "License"); you may not use this file except in compliance -->
<!--- with the License. You may obtain a copy of the License at -->

<!--- http://www.apache.org/licenses/LICENSE-2.0 -->

<!--- Unless required by applicable law or agreed to in writing, -->
<!--- software distributed under the License is distributed on an -->
<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
<!--- KIND, either express or implied. See the License for the -->
<!--- specific language governing permissions and limitations -->
<!--- under the License. -->

# Hexagon API app

This is a meta-app that build the necessary binaries for use with
the `HexagonLauncher` utility from `tvm.contrib.hexagon`.

It will build the TVM runtime for Android, the RPC server application
for Android, and the RPC library for Hexagon with the TVM runtime for
Hexagon built into it.

## Configuration

There is a set of configuration variables that are required for cmake:
- `ANDROID_ABI`: Set this to `arm64-v8a`.
- `ANDROID_PLATFORM`: This can be `android-28`.
- `USE_ANDROID_TOOLCHAIN`: The path to the Android toolchain file, i.e.
`android.toolchain.cmake`. This file is a part of the Android NDK.
- `USE_HEXAGON_ARCH`: The version string of the Hexagon architecture
to use, i.e. vNN. The typical setting would be `v68` or later.
- `USE_HEXAGON_SDK`: The path to the Hexagon SDK. Set this path in such
a way that `${USE_HEXAGON_SDK}/setup_sdk_env.source` exists.
- `USE_HEXAGON_TOOLCHAIN`: Path to Hexagon toolchain. It can be the
Hexagon toolchain included in the SDK, for example
`${USE_HEXAGON_TOOLCHAIN}/tools/HEXAGON_Tools/x.y.z/Tools`. The `x.y.z`
in the path is the toolchain version number, which is specific to the
version of the SDK.

Additionally, the variable `USE_OUTPUT_BINARY_DIR` can be set to indicate
the location where the generated binaries will be placed. If not set, it
defaults to `hexagon_rpc` subdirectory in the current build directory.


## Build

The build will generate the following binaries:
- `tvm_runtime.so`: TVM runtime for Android (shared library).
- `tvm_rpc_android`: RPC server for Android.
- `libhexagon_rpc_skel.so`: RPC library for Hexagon.
- `libtvm_runtime.a`: TVM runtime for Hexagon (static library).

The RPC library for Hexagon contains the TVM runtime, so the static
TVM runtime for Hexagon is not strictly necessary.
37 changes: 0 additions & 37 deletions apps/hexagon_launcher/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,43 +31,6 @@ The supported Snapdragon architectures are 855, 865, and 888.
Android NDK can be downloaded from https://developer.android.com/ndk.
Hexagon SDK is available at //developer.qualcomm.com/software/hexagon-dsp-sdk.

### Compilation with TVM

Building the Hexagon launcher application as a component of the main TVM build
used for Hexagon codegen can be achieved by setting `USE_HEXAGON_LAUNCHER=ON`.
This option will compile core tvm, the android launcher binary and its corresponding
tvm_runtime, as well as the Hexagon launcher shared library and its corresponding
tvm_runtime. As described in the [Manual compilation](#Manual compilation) section
each component requires Hexagon and android dependencies. When building the launcher
along with TVM these configurations must be providing when invoking cmake. A minimal
example invocation for compiling TVM along with the Hexagon launcher is included below:

```
cmake -DCMAKE_C_COMPILER=/path/to/clang \
-DCMAKE_CXX_COMPILER=/path/to/clang++ \
-DCMAKE_CXX_FLAGS='-stdlib=libc++' \
-DCMAKE_CXX_STANDARD=14 \
-DUSE_LLVM=/path/to/llvm/bin/llvm-config \
-DUSE_HEXAGON_ARCH=v65|v66|v68 \
-DUSE_HEXAGON_LAUNCHER=ON \
-DUSE_HEXAGON_SDK=/path/to/hexagon/SDK \
-DUSE_HEXAGON_TOOLCHAIN=/path/to/hexagon/toolchain/ ..
-DANDROID_ABI=arm64-v8a \
-DANDROID_PLATFORM=android-28 \
-DUSE_ANDROID_TOOLCHAIN=/path/to/android-ndk/build/cmake/android.toolchain.cmake \
..
```

where `v65|v66|v68` means "one of" these architecture versions.
The Hexagon launcher application is an android binary and thus requires the use
of an android toolchain for compilation. Similarly, the Hexagon tvm runtime
requires the use of the Hexagon toolchain and depends on the Hexagon SDK. The
resulting hexagon launcher binaries can be found in the `apps_hexagon_launcher`
subdirectory of the cmake build directory. The above command
will build support for Hexagon codegen in the TVM library that requires
`USE_LLVM` to be set to an llvm-config that has the Hexagon target built in.


### Manual compilation

Since some source files are shared between the Hexagon and android builds,
Expand Down
Loading

0 comments on commit 1dc9092

Please sign in to comment.