-
Notifications
You must be signed in to change notification settings - Fork 3.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This change adds the code generation and minimal runtime API to use the Ahead Of Time (AOT) compilation flow. The main logic is contained in: - src/relay/backend/aot_codegen.cc Which produces a TIR PrimFunc traversing the Relay graph The runtime interface (authored by @Mousius) leaves a gap for future iterations using platform-specific features from RTOS. Currently AOT runs successfully on x86 in a host OS, running these tests on micro is coming soon. This PR is based on the RFC described here: https://discuss.tvm.apache.org/t/implementing-aot-in-tvm/9206 Co-authored-by: Christopher Sidebottom <Christopher.Sidebottom@arm.com> Change-Id: I9f731c953231f129e1472298915dddc01788efd7
- Loading branch information
Showing
41 changed files
with
2,140 additions
and
88 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| /*! | ||
| * \file include/tvm/runtime/crt/aot/tvm_backend.h | ||
| * \brief Backend functions for the AOT executor | ||
| * | ||
| * These are not designed to user-facing and may change without warning | ||
| */ | ||
|
|
||
| #ifndef TVM_RUNTIME_CRT_AOT_TVM_BACKEND_H_ | ||
| #define TVM_RUNTIME_CRT_AOT_TVM_BACKEND_H_ | ||
|
|
||
| #include <stddef.h> | ||
| #include <stdint.h> | ||
|
|
||
| #include "tvm_error.h" | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| /*! Memory alignment for allocator */ | ||
| #ifndef TVM_RUNTIME_ALLOC_ALIGNMENT | ||
| #define TVM_RUNTIME_ALLOC_ALIGNMENT 16 | ||
| #endif | ||
|
|
||
| /*! The AOT runtime links staticly */ | ||
| #define TVM_DLL | ||
|
|
||
| /*! | ||
| * \brief Minimal TVMValue | ||
| */ | ||
| typedef union { | ||
| int64_t v_int64; /** Currently used for parameter lookup */ | ||
| void* v_handle; /** Pointer to other values */ | ||
| } TVMValue; | ||
|
|
||
| /*! | ||
| * \brief Packed function signature definition | ||
| */ | ||
| typedef int32_t(tvm_function_t)(void* args, void* arg_type_ids, int32_t num_args, | ||
| void* out_ret_value, void* out_ret_tcode, void* resource_handle); | ||
|
|
||
| /*! | ||
| * \brief Workspace memory structure | ||
| */ | ||
| typedef struct { | ||
| uint8_t* next_alloc; /** Pointer to the next block of bytes to allocate */ | ||
| uint8_t* workspace; /** Pointer to start of the workspace */ | ||
| size_t workspace_size; /** Total number of bytes in the workspace */ | ||
| } tvm_workspace_t; | ||
|
|
||
| /** | ||
| * \brief Backend function to allocate temporal workspace. | ||
| * | ||
| * \note The result allocated space is ensured to be aligned to TVM_RUNTIME_ALLOC_ALIGNMENT. | ||
| * \note Currently matches CRT runtime signature but this will change in future to accommodate | ||
| * memory planning | ||
| * | ||
| * \param device_type Ignored | ||
| * \param device_id Ignored | ||
| * \param nbytes The size of the space requested. | ||
| * \param dtype_code_hint Ignored | ||
| * \param dtype_bits_hint Ignored | ||
| * \return void* NULL on error, a valid pointer on success | ||
| */ | ||
| void* TVMBackendAllocWorkspace(int device_type, int device_id, uint64_t nbytes, int dtype_code_hint, | ||
| int dtype_bits_hint); | ||
|
|
||
| /*! | ||
| * \brief Backend function to free temporal workspace. | ||
| * | ||
| * \note Currently matches CRT runtime signature but this will change in future to accomodate memory | ||
| * planning | ||
| * | ||
| * \param ptr The result allocated space pointer. | ||
| * \param device_type Ignored | ||
| * \param device_id Ignored | ||
| * \return tvm_crt_error_t Containing any error statuses | ||
| */ | ||
| tvm_crt_error_t TVMBackendFreeWorkspace(int device_type, int device_id, void* ptr); | ||
|
|
||
| #ifdef __cplusplus | ||
| } // extern "C" | ||
| #endif | ||
|
|
||
| #endif // TVM_RUNTIME_CRT_AOT_TVM_BACKEND_H_ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| /*! | ||
| * \file include/tvm/runtime/crt/aot/tvm_error.h | ||
| * \brief Defines a subset of error codes returned by the CRT AOT executor. | ||
| */ | ||
|
|
||
| #ifndef TVM_RUNTIME_CRT_AOT_TVM_ERROR_H_ | ||
| #define TVM_RUNTIME_CRT_AOT_TVM_ERROR_H_ | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| #define TVM_CRT_ERROR_CATEGORY_Pos 8 | ||
| #define TVM_CRT_ERROR_CATEGORY_Msk (0xff << TVM_CRT_ERROR_CATEGORY_Pos) | ||
| #define TVM_CRT_ERROR_CODE_Pos 0 | ||
| #define TVM_CRT_ERROR_CODE_Msk (0xff << TVM_CRT_ERROR_CODE_Pos) | ||
|
|
||
| #define DEFINE_TVM_CRT_ERROR(category, code) \ | ||
| (((category) << TVM_CRT_ERROR_CATEGORY_Pos) | ((code) << TVM_CRT_ERROR_CODE_Pos)) | ||
| typedef enum { | ||
| kTvmErrorCategoryPlatform = 5, | ||
| kTvmErrorCategoryFunctionCall = 8, | ||
| } tvm_crt_error_category_t; | ||
|
|
||
| typedef enum { | ||
| kTvmErrorNoError = 0, | ||
|
|
||
| // Platform | ||
| kTvmErrorPlatformCheckFailure = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryPlatform, 0), | ||
| kTvmErrorPlatformMemoryManagerInitialized = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryPlatform, 1), | ||
| kTvmErrorPlatformShutdown = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryPlatform, 2), | ||
| kTvmErrorPlatformNoMemory = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryPlatform, 3), | ||
| kTvmErrorPlatformTimerBadState = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryPlatform, 4), | ||
|
|
||
| // Function Calls - common problems encountered calling functions. | ||
| kTvmErrorFunctionCallNumArguments = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryFunctionCall, 0), | ||
| kTvmErrorFunctionCallWrongArgType = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryFunctionCall, 1), | ||
| kTvmErrorFunctionCallNotImplemented = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryFunctionCall, 2), | ||
|
|
||
| // System errors are always negative integers; this mask indicates presence of a system error. | ||
| // Cast tvm_crt_error_t to a signed integer to interpret the negative error code. | ||
| kTvmErrorSystemErrorMask = (1 << (sizeof(int) * 4 - 1)), | ||
| } tvm_crt_error_t; | ||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif // TVM_RUNTIME_CRT_AOT_TVM_ERROR_H_ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| /*! | ||
| * \file include/tvm/runtime/crt/aot/tvm_executor.h | ||
| * \brief TVM Executor for the Ahead-of-Time Runtime | ||
| * | ||
| * AOT models are described by the TVM model descriptor format | ||
| * which can be passed to tvm_runtime_run. These descriptors will be | ||
| * generated by the AOT compilation process. This can optionally be | ||
| * augmented with platform specific context to be passed to the TVM | ||
| * operators. | ||
| * | ||
| * Example: | ||
| * extern tvm_model_t my_network; | ||
| * int main() { | ||
| * void* data = get_data(); | ||
| * void* output[4] = {0, 0, 0, 0}; | ||
| * void* inputs = {data}; | ||
| * void* outputs = {output}; | ||
| * tvm_context_t my_context = { | ||
| * .driver = ...; | ||
| * }; | ||
| * tvm_runtime_run( | ||
| * &my_network, | ||
| * inputs, | ||
| * outputs | ||
| * &my_context | ||
| * ); | ||
| * return 0; | ||
| * } | ||
| */ | ||
|
|
||
| #ifndef TVM_RUNTIME_CRT_AOT_TVM_EXECUTOR_H_ | ||
| #define TVM_RUNTIME_CRT_AOT_TVM_EXECUTOR_H_ | ||
|
|
||
| #include <stdint.h> | ||
|
|
||
| #include "tvm_backend.h" | ||
| #include "tvm_error.h" | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| /*! | ||
| * \brief Context information for future integrations | ||
| * which is passed through to the operators. | ||
| * | ||
| * \note Can be used for drivers and platform specific information. | ||
| */ | ||
| typedef struct { | ||
| } tvm_context_t; | ||
|
|
||
| /*! | ||
| * \brief TVM Model descriptor to describe the | ||
| * model to the runtime. | ||
| */ | ||
| typedef struct { | ||
| uint32_t num_input_tensors; /** Number of expected input tensors */ | ||
| uint32_t num_output_tensors; /** Number of expected output tensors */ | ||
| tvm_function_t* run_func; /** Generated model function, called through tvm_runtime_run */ | ||
| tvm_workspace_t* workspace; /** Memory workspace for the model to use */ | ||
| } tvm_model_t; | ||
|
|
||
| /*! | ||
| * \brief Main entry point for | ||
| * \param model Model descriptor structure to reference for runtime information | ||
| * \param inputs Pointer to input pointer(s) | ||
| * \param outputs Pointer to output pointer(s) | ||
| * \param context Context information to be passed through to operators | ||
| * \return tvm_status_t containing success or errors from the model run | ||
| */ | ||
| tvm_crt_error_t tvm_runtime_run(const tvm_model_t* model, void** inputs, void** outputs, | ||
| tvm_context_t* context); | ||
|
|
||
| #ifdef __cplusplus | ||
| } // extern "C" | ||
| #endif | ||
|
|
||
| #endif // TVM_RUNTIME_CRT_AOT_TVM_EXECUTOR_H_ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.