diff --git a/backends/apple/coreml/CMakeLists.txt b/backends/apple/coreml/CMakeLists.txt index d670b60e6c..113b21bd69 100644 --- a/backends/apple/coreml/CMakeLists.txt +++ b/backends/apple/coreml/CMakeLists.txt @@ -13,6 +13,11 @@ if(NOT EXECUTORCH_ROOT) set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..) endif() +if(EXECUTORCH_BUILD_SDK) +# protobuf requires frtti +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -frtti" ) +endif() + option(COREML_BUILD_EXECUTOR_RUNNER "Build CoreML executor runner." OFF) # inmemoryfs sources @@ -59,6 +64,7 @@ set(SDK_SOURCES runtime/sdk/ETCoreMLModelAnalyzer.mm runtime/sdk/ETCoreMLModelStructurePath.mm runtime/sdk/ETCoreMLOperationProfilingInfo.mm + runtime/sdk/ETCoreMLModelDebugInfo.mm runtime/sdk/ETCoreMLModelDebugger.mm runtime/sdk/ETCoreMLModelProfiler.mm runtime/sdk/ETCoreMLPair.mm @@ -141,6 +147,8 @@ if(EXECUTORCH_BUILD_SDK) add_subdirectory( ${CMAKE_CURRENT_SOURCE_DIR}/third-party/coremltools/deps/protobuf/cmake ) + + target_link_options_shared_lib(libprotobuf-lite) target_link_libraries(coremldelegate PRIVATE libprotobuf-lite) endif() diff --git a/backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.mm b/backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.mm index 927df0483f..8d6d537385 100644 --- a/backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.mm +++ b/backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.mm @@ -5,36 +5,37 @@ // // Please refer to the license found in the LICENSE file in the root directory of the source tree. -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import +#import "ETCoreMLAsset.h" +#import "ETCoreMLAssetManager.h" +#import "ETCoreMLDefaultModelExecutor.h" +#import "ETCoreMLLogging.h" +#import "ETCoreMLModel.h" +#import "ETCoreMLModelCompiler.h" +#import "ETCoreMLModelExecutor.h" +#import "ETCoreMLModelLoader.h" +#import "ETCoreMLModelManager.h" +#import "ETCoreMLStrings.h" +#import "MLModel_Prewarm.h" +#import "MLMultiArray_Copy.h" #import -#import +#import "inmemory_filesystem_utils.hpp" #import #import -#import -#import -#import +#import "model_metadata.h" +#import "multiarray.h" +#import "objc_array_util.h" #import #import -#import +#import "serde_json.h" #import #import #import #if ET_EVENT_TRACER_ENABLED -#import -#import -#import +#import "ETCoreMLModelAnalyzer.h" +#import "ETCoreMLModelDebugInfo.h" +#import "ETCoreMLModelStructurePath.h" +#import "objc_safe_cast.h" #endif namespace { @@ -317,31 +318,14 @@ void add_compute_unit(std::string& identifier, MLComputeUnits compute_units) { return [[ETCoreMLAsset alloc] initWithBackingAsset:std::move(backingAsset.value())]; } -NSDictionary * _Nullable get_operation_path_to_symbol_name_map(const inmemoryfs::InMemoryFileSystem *inMemoryFS, - NSError * __autoreleasing *error) { +ETCoreMLModelDebugInfo * _Nullable get_model_debug_info(const inmemoryfs::InMemoryFileSystem *inMemoryFS, + NSError * __autoreleasing *error) { NSData *file_data = get_file_data(inMemoryFS, ETCoreMLStrings.debugInfoFileRelativePath); if (!file_data) { return nil; } - - id object = [NSJSONSerialization JSONObjectWithData:file_data options:(NSJSONReadingOptions)0 error:error]; - if (!object) { - return nil; - } - - NSDictionary *json_dict = SAFE_CAST(object, NSDictionary); - NSMutableDictionary *result = [NSMutableDictionary dictionaryWithCapacity:json_dict.count]; - NSDictionary *> *debug_symbol_to_operation_path_map = SAFE_CAST(json_dict[ETCoreMLStrings.debugSymbolToOperationPathKeyName], NSDictionary); - for (NSString *symbol_name in debug_symbol_to_operation_path_map) { - NSArray *> *components = SAFE_CAST(debug_symbol_to_operation_path_map[symbol_name], NSArray); - if (components.count == 0) { - continue; - } - ETCoreMLModelStructurePath *path = [[ETCoreMLModelStructurePath alloc] initWithComponents:components]; - result[path] = symbol_name; - } - - return result; + + return [ETCoreMLModelDebugInfo modelDebugInfoFromData:file_data error:error]; } #endif @@ -490,16 +474,16 @@ - (nullable NSURL *)compiledModelURLWithIdentifier:(NSString *)identifier } NSError *localError = nil; - NSDictionary *operation_path_to_symbol_name_map = get_operation_path_to_symbol_name_map(inMemoryFS, - &localError); + ETCoreMLModelDebugInfo *debug_info = get_model_debug_info(inMemoryFS, &localError); if (localError) { ETCoreMLLogError(localError, "Failed to parse debug info file"); } + return [[ETCoreMLModelAnalyzer alloc] initWithCompiledModelAsset:compiledModelAsset modelAsset:modelAsset + modelDebugInfo:debug_info metadata:metadata - operationPathToDebugSymbolMap:operation_path_to_symbol_name_map configuration:configuration assetManager:self.assetManager error:error]; diff --git a/backends/apple/coreml/runtime/delegate/ETCoreMLStrings.h b/backends/apple/coreml/runtime/delegate/ETCoreMLStrings.h index 74383769c4..7d0e010180 100644 --- a/backends/apple/coreml/runtime/delegate/ETCoreMLStrings.h +++ b/backends/apple/coreml/runtime/delegate/ETCoreMLStrings.h @@ -66,6 +66,8 @@ NS_ASSUME_NONNULL_BEGIN @property (class, copy, readonly, nonatomic, nullable) NSString* debugInfoFileRelativePath; /// The debug symbol to operation path key name. @property (class, copy, readonly, nonatomic, nullable) NSString* debugSymbolToOperationPathKeyName; +/// The debug symbol to handles key name. +@property (class, copy, readonly, nonatomic, nullable) NSString* debugSymbolToHandlesKeyName; @end diff --git a/backends/apple/coreml/runtime/delegate/ETCoreMLStrings.mm b/backends/apple/coreml/runtime/delegate/ETCoreMLStrings.mm index e8eb2d3cff..fb66f7b7c0 100644 --- a/backends/apple/coreml/runtime/delegate/ETCoreMLStrings.mm +++ b/backends/apple/coreml/runtime/delegate/ETCoreMLStrings.mm @@ -95,6 +95,11 @@ + (NSString *)debugSymbolToOperationPathKeyName { return ETCoreMLDebugSymbolToOperationPathKeyName; } ++ (NSString *)debugSymbolToHandlesKeyName { + static NSString * const ETCoreMLDebugSymbolToHandlesKeyName = @"debugSymbolToHandles"; + return ETCoreMLDebugSymbolToHandlesKeyName; +} + + (nullable NSString *)assetsDirectoryPath { static dispatch_once_t onceToken; static NSString *result = nil; diff --git a/backends/apple/coreml/runtime/delegate/coreml_backend_delegate.mm b/backends/apple/coreml/runtime/delegate/coreml_backend_delegate.mm index ab596575a2..fc3e5a47f9 100644 --- a/backends/apple/coreml/runtime/delegate/coreml_backend_delegate.mm +++ b/backends/apple/coreml/runtime/delegate/coreml_backend_delegate.mm @@ -124,7 +124,8 @@ ModelLoggingOptions get_logging_options(BackendExecutionContext& context) { auto event_tracer = context.event_tracer(); if (event_tracer) { options.log_profiling_info = true; - options.log_intermediate_tensors = event_tracer->intermediate_outputs_logging_status(); + auto debug_level = event_tracer->event_tracer_debug_level(); + options.log_intermediate_tensors = (debug_level >= EventTracerDebugLogLevel::kIntermediateOutputs); } return options; diff --git a/backends/apple/coreml/runtime/delegate/model_event_logger.h b/backends/apple/coreml/runtime/delegate/model_event_logger.h index c78ebcaac1..91fe3b18cf 100644 --- a/backends/apple/coreml/runtime/delegate/model_event_logger.h +++ b/backends/apple/coreml/runtime/delegate/model_event_logger.h @@ -34,8 +34,8 @@ class ModelEventLogger { /// /// @param op_path_to_value_map A dictionary with the operation path as the key and the operation's value as the /// value. - /// @param op_path_to_debug_symbol_name_map A dictionary with the operation path as the key and the symbol name as - /// the value. The symbol name is the delegate handle. + /// @param op_path_to_debug_symbol_name_map A dictionary with the operation path as the key and the debug symbol + /// name as the value. virtual void log_intermediate_tensors( NSDictionary* op_path_to_value_map, NSDictionary* op_path_to_debug_symbol_name_map) const noexcept = 0; diff --git a/backends/apple/coreml/runtime/sdk/ETCoreMLModelAnalyzer.h b/backends/apple/coreml/runtime/sdk/ETCoreMLModelAnalyzer.h index 4048dae5fe..a8efe2171c 100644 --- a/backends/apple/coreml/runtime/sdk/ETCoreMLModelAnalyzer.h +++ b/backends/apple/coreml/runtime/sdk/ETCoreMLModelAnalyzer.h @@ -7,7 +7,7 @@ #import -#import +#import "ETCoreMLModelExecutor.h" namespace executorchcoreml { struct ModelMetadata; @@ -15,6 +15,7 @@ struct ModelMetadata; @class ETCoreMLAsset; @class ETCoreMLAssetManager; +@class ETCoreMLModelDebugInfo; @class ETCoreMLModelStructurePath; @protocol ETCoreMLModelEventLogger; @@ -32,16 +33,15 @@ __attribute__((objc_subclassing_restricted)) /// /// @param compiledModelAsset The compiled model asset (mlmodelc). /// @param modelAsset The model asset (mlpackage). +/// @param modelDebugInfo The model debug info. /// @param metadata The model metadata. -/// @param operationPathToDebugSymbolMap The operation path to debug symbol map. /// @param configuration The model configuration. /// @param assetManager The asset manager used to manage storage of compiled models. /// @param error On failure, error is filled with the failure information. - (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset*)compiledModelAsset modelAsset:(nullable ETCoreMLAsset*)modelAsset + modelDebugInfo:(nullable ETCoreMLModelDebugInfo*)modelDebugInfo metadata:(const executorchcoreml::ModelMetadata&)metadata - operationPathToDebugSymbolMap: - (nullable NSDictionary*)operationPathToDebugSymbolMap configuration:(MLModelConfiguration*)configuration assetManager:(ETCoreMLAssetManager*)assetManager error:(NSError* __autoreleasing*)error NS_DESIGNATED_INITIALIZER; diff --git a/backends/apple/coreml/runtime/sdk/ETCoreMLModelAnalyzer.mm b/backends/apple/coreml/runtime/sdk/ETCoreMLModelAnalyzer.mm index 57212445e5..1740faf00e 100644 --- a/backends/apple/coreml/runtime/sdk/ETCoreMLModelAnalyzer.mm +++ b/backends/apple/coreml/runtime/sdk/ETCoreMLModelAnalyzer.mm @@ -5,22 +5,24 @@ // // Please refer to the license found in the LICENSE file in the root directory of the source tree. -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import -#import +#import "ETCoreMLModelAnalyzer.h" + +#import "ETCoreMLAsset.h" +#import "ETCoreMLAssetManager.h" +#import "ETCoreMLDefaultModelExecutor.h" +#import "ETCoreMLLogging.h" +#import "ETCoreMLModel.h" +#import "ETCoreMLModelLoader.h" +#import "ETCoreMLModelStructurePath.h" +#import "ETCoreMLModelDebugInfo.h" +#import "ETCoreMLModelDebugger.h" +#import "ETCoreMLModelProfiler.h" +#import "ETCoreMLStrings.h" +#import "model_logging_options.h" +#import "model_event_logger.h" +#import "model_metadata.h" +#import "model_package_info.h" +#import "objc_safe_cast.h" namespace { using namespace executorchcoreml; @@ -34,7 +36,7 @@ @interface ETCoreMLModelAnalyzer () @property (strong, nonatomic, nullable) ETCoreMLModelProfiler *profiler; @property (strong, nonatomic, nullable) ETCoreMLModelDebugger *debugger; @property (strong, nonatomic, nullable) id executor; -@property (readonly, copy, nonatomic, nullable) NSDictionary *operationPathToDebugSymbolMap; +@property (readonly, copy, nonatomic, nullable) ETCoreMLModelDebugInfo *modelDebugInfo; @property (readonly, strong, nonatomic) MLModelConfiguration *configuration; @end @@ -43,8 +45,8 @@ @implementation ETCoreMLModelAnalyzer - (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset *)compiledModelAsset modelAsset:(nullable ETCoreMLAsset *)modelAsset + modelDebugInfo:(nullable ETCoreMLModelDebugInfo *)modelDebugInfo metadata:(const executorchcoreml::ModelMetadata&)metadata - operationPathToDebugSymbolMap:(nullable NSDictionary *)operationPathToDebugSymbolMap configuration:(MLModelConfiguration *)configuration assetManager:(ETCoreMLAssetManager *)assetManager error:(NSError * __autoreleasing *)error { @@ -72,9 +74,9 @@ - (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset *)compiledMod if (self) { _model = model; _modelAsset = modelAsset; + _modelDebugInfo = modelDebugInfo; _assetManager = assetManager; _configuration = configuration; - _operationPathToDebugSymbolMap = operationPathToDebugSymbolMap; _executor = [[ETCoreMLDefaultModelExecutor alloc] initWithModel:model]; } @@ -113,7 +115,7 @@ - (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset *)compiledMod return nil; } - eventLogger->log_profiling_infos(profilingInfos, self.operationPathToDebugSymbolMap); + eventLogger->log_profiling_infos(profilingInfos, self.modelDebugInfo.pathToDebugSymbolMap); return modelOutputs; } @@ -131,6 +133,7 @@ - (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset *)compiledMod if (!self.debugger) { self.debugger = [[ETCoreMLModelDebugger alloc] initWithModelAsset:self.modelAsset + modelDebugInfo:self.modelDebugInfo outputNames:self.model.orderedOutputNames configuration:self.configuration assetManager:self.assetManager @@ -143,6 +146,7 @@ - (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset *)compiledMod NSArray *modelOutputs = nil; NSArray *operationPaths = self.debugger.operationPaths; + NSDictionary *operationPathToDebugSymbolMap = self.debugger.operationPathToDebugSymbolMap; NSInteger n = operationPaths.count/MAX_MODEL_OUTPUTS_COUNT + (operationPaths.count % MAX_MODEL_OUTPUTS_COUNT == 0 ? 0 : 1); for (NSInteger i = 0; i < n; i++) { @autoreleasepool { @@ -157,7 +161,7 @@ - (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset *)compiledMod } if (outputs.count > 0) { - eventLogger->log_intermediate_tensors(outputs, self.operationPathToDebugSymbolMap); + eventLogger->log_intermediate_tensors(outputs, operationPathToDebugSymbolMap); } } } diff --git a/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugInfo.h b/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugInfo.h new file mode 100644 index 0000000000..ab8fdaaedd --- /dev/null +++ b/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugInfo.h @@ -0,0 +1,47 @@ +// +// ETCoreMLModelDebugInfo.h +// +// Copyright © 2024 Apple Inc. All rights reserved. +// +// Please refer to the license found in the LICENSE file in the root directory of the source tree. + +#import + +@class ETCoreMLModelStructurePath; + +NS_ASSUME_NONNULL_BEGIN + +/// A class representing the profiling info of an operation. +__attribute__((objc_subclassing_restricted)) +@interface ETCoreMLModelDebugInfo : NSObject + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + + +/// Constructs an `ETCoreMLModelDebugInfo` instance. +/// +/// @param pathToDebugSymbolMap Operation path to debug symbol map. +/// @param pathToDebugHandlesMap Operation path to debug handles map. +- (instancetype)initWithPathToDebugSymbolMap:(NSDictionary*)pathToDebugSymbolMap + pathToDebugHandlesMap: + (NSDictionary*>*)pathToDebugHandlesMap + NS_DESIGNATED_INITIALIZER; + +/// Constructs an `ETCoreMLModelDebugInfo` instance. +/// +/// @param data The json data. +/// @param error On failure, error is filled with the failure information. ++ (nullable instancetype)modelDebugInfoFromData:(NSData*)data error:(NSError* __autoreleasing*)error; + +/// Operation path to debug symbol map. +@property (readonly, strong, nonatomic) NSDictionary* pathToDebugSymbolMap; + +/// Operation path to debug handles map. +@property (readonly, strong, nonatomic) + NSDictionary*>* pathToDebugHandlesMap; + +@end + +NS_ASSUME_NONNULL_END diff --git a/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugInfo.mm b/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugInfo.mm new file mode 100644 index 0000000000..4e14d2502d --- /dev/null +++ b/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugInfo.mm @@ -0,0 +1,71 @@ +// +// ETCoreMLModelDebugInfo.mm +// +// Copyright © 2024 Apple Inc. All rights reserved. +// +// Please refer to the license found in the LICENSE file in the root directory of the source tree. + + +#import "ETCoreMLModelDebugInfo.h" + +#import "ETCoreMLStrings.h" +#import "ETCoreMLModelStructurePath.h" +#import "objc_safe_cast.h" + + +@implementation ETCoreMLModelDebugInfo + +- (instancetype)initWithPathToDebugSymbolMap:(NSDictionary *)pathToDebugSymbolMap + pathToDebugHandlesMap:(NSDictionary *> *)pathToDebugHandlesMap { + self = [super init]; + if (self) { + _pathToDebugSymbolMap = [pathToDebugSymbolMap copy]; + _pathToDebugHandlesMap = [pathToDebugHandlesMap copy]; + } + + return self; +} + ++ (nullable instancetype)modelDebugInfoFromData:(NSData *)data error:(NSError * __autoreleasing *)error { + id object = [NSJSONSerialization JSONObjectWithData:data options:(NSJSONReadingOptions)0 error:error]; + if (!object) { + return nil; + } + + NSDictionary *jsonDict = SAFE_CAST(object, NSDictionary); + // Construct operation path to debug symbol map. + NSDictionary *> *debugSymbolToPathMap = SAFE_CAST(jsonDict[ETCoreMLStrings.debugSymbolToOperationPathKeyName], NSDictionary); + NSMutableDictionary *pathToDebugSymbolMap = [NSMutableDictionary dictionaryWithCapacity:debugSymbolToPathMap.count]; + for (NSString *symbolName in debugSymbolToPathMap) { + NSArray *> *components = SAFE_CAST(debugSymbolToPathMap[symbolName], NSArray); + if (components.count == 0) { + continue; + } + ETCoreMLModelStructurePath *path = [[ETCoreMLModelStructurePath alloc] initWithComponents:components]; + pathToDebugSymbolMap[path] = symbolName; + + } + // Construct operation path to debug handles map. + NSDictionary *> *debugSymbolToHandles = SAFE_CAST(jsonDict[ETCoreMLStrings.debugSymbolToHandlesKeyName], NSDictionary); + NSMutableDictionary *> *pathToDebugHandlesMap = [NSMutableDictionary dictionaryWithCapacity:debugSymbolToHandles.count]; + for (NSString *debugSymbol in debugSymbolToHandles) { + NSArray *components = debugSymbolToPathMap[debugSymbol]; + if (components.count == 0) { + continue; + } + + NSArray *debugHandles = debugSymbolToHandles[debugSymbol]; + if (debugHandles.count == 0) { + continue; + } + + ETCoreMLModelStructurePath *path = [[ETCoreMLModelStructurePath alloc] initWithComponents:components]; + pathToDebugHandlesMap[path] = debugHandles; + } + + return [[ETCoreMLModelDebugInfo alloc] initWithPathToDebugSymbolMap:pathToDebugSymbolMap + pathToDebugHandlesMap:pathToDebugHandlesMap]; + +} + +@end diff --git a/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugger.h b/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugger.h index 7221086318..40b9e32394 100644 --- a/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugger.h +++ b/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugger.h @@ -9,6 +9,7 @@ @class ETCoreMLAsset; @class ETCoreMLAssetManager; +@class ETCoreMLModelDebugInfo; @class ETCoreMLModelStructurePath; typedef NSDictionary ETCoreMLModelOutputs; @@ -25,11 +26,13 @@ __attribute__((objc_subclassing_restricted)) /// Constructs an `ETCoreMLModelDebugger` instance. /// /// @param modelAsset The model asset (mlpackage). +/// @param modelDebugInfo The model debug info. /// @param outputNames The model output names. /// @param configuration The model configuration. /// @param assetManager The asset manager used to manage storage of compiled models. /// @param error On failure, error is filled with the failure information. - (nullable instancetype)initWithModelAsset:(ETCoreMLAsset*)modelAsset + modelDebugInfo:(nullable ETCoreMLModelDebugInfo*)modelDebugInfo outputNames:(NSOrderedSet*)outputNames configuration:(MLModelConfiguration*)configuration assetManager:(ETCoreMLAssetManager*)assetManager @@ -55,6 +58,10 @@ __attribute__((objc_subclassing_restricted)) /// The paths to all the operations for which we can get the outputs. @property (readonly, copy, nonatomic) NSArray* operationPaths; +/// Operation path to debug symbol map. +@property (readonly, copy, nonatomic) + NSDictionary* operationPathToDebugSymbolMap; + @end NS_ASSUME_NONNULL_END diff --git a/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugger.mm b/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugger.mm index 32a629c6f3..3be28b56d6 100644 --- a/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugger.mm +++ b/backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugger.mm @@ -5,21 +5,23 @@ // // Please refer to the license found in the LICENSE file in the root directory of the source tree. +#import "ETCoreMLModelDebugger.h" + #import -#import -#import -#import -#import -#import -#import -#import -#import +#import "ETCoreMLAsset.h" +#import "ETCoreMLAssetManager.h" +#import "ETCoreMLLogging.h" +#import "ETCoreMLModelCompiler.h" +#import "ETCoreMLModelDebugInfo.h" +#import "ETCoreMLModelStructurePath.h" +#import "ETCoreMLPair.h" +#import "ETCoreMLStrings.h" #import #import #import #import -#import -#import +#import "model_package_info.h" +#import "objc_json_serde.h" #import #import @@ -68,10 +70,6 @@ BOOL is_const_operation(const MILSpec::Operation& operation) { return operation.type() == "const"; } -BOOL is_cast_operation(const MILSpec::Operation& operation) { - return operation.type() == "cast"; -} - BOOL is_datatype_supported_as_model_output(MILSpec::DataType datatype) { switch (datatype) { case MILSpec::DataType::INT32: @@ -95,11 +93,7 @@ BOOL is_operation_output_supported_as_model_output(const MILSpec::Operation& ope if (is_const_operation(operation)) { return NO; } - - if (is_cast_operation(operation)) { - return NO; - } - + return YES; } @@ -316,7 +310,6 @@ void update_model_spec_version_to_include_fp16_output(Model& model_spec) { return nil; } - // Compile the model. return [ETCoreMLModelCompiler compileModelAtURL:model_url maxWaitTimeInSeconds:(5 * 60) error:error]; @@ -383,6 +376,108 @@ void set_intermediate_outputs(id output_features, result[path] = multi_array; } } + +NSArray *get_operation_dependencies(const MILSpec::Operation &operation, + ETCoreMLModelStructurePath *path, + NSSet *paths) { + const auto& inputs = operation.inputs(); + const auto cppPath = path.underlyingValue; + NSMutableArray *deps = [NSMutableArray arrayWithCapacity:inputs.size()]; + for (const auto& [_, arg] : inputs) { + const auto& bindings = arg.arguments(); + for (const auto& binding : bindings) { + if (binding.has_value()) { + continue; + } + + const auto& name = binding.name(); + auto dep = cppPath; + dep.remove_last_component(); + dep.append_component(Path::Program::Operation(name)); + ETCoreMLModelStructurePath *path = [[ETCoreMLModelStructurePath alloc] initWithUnderlyingValue:dep]; + if ([paths containsObject:path]) { + [deps addObject:path]; + } + } + } + + return deps; +} + +NSDictionary *> *get_debug_handle_to_operation_paths_map(ETCoreMLModelDebugInfo *debug_info) { + NSUInteger capacity = debug_info.pathToDebugHandlesMap.count; + NSMutableDictionary *> *result = [NSMutableDictionary dictionaryWithCapacity:capacity]; + [debug_info.pathToDebugHandlesMap enumerateKeysAndObjectsUsingBlock:^(ETCoreMLModelStructurePath *path, + NSArray *debug_handles, + BOOL * _Nonnull __unused stop) { + for (NSString *debug_handle in debug_handles) { + NSMutableArray *paths = result[debug_handle]; + if (!paths) { + paths = [NSMutableArray array]; + result[debug_handle] = paths; + } + + [paths addObject:path]; + } + + }]; + + return result; +} + +BOOL is_node_terminal_node(ETCoreMLModelStructurePath *node, + NSArray *nodes, + NSDictionary *> *dependencies) { + NSMutableSet *nodes_dependencies = [NSMutableSet set]; + for (ETCoreMLModelStructurePath *current_node in nodes) { + if ([current_node isEqual:node]) { + continue; + } + NSArray *node_dependencies = dependencies[current_node]; + if (node_dependencies.count > 0) { + [nodes_dependencies addObjectsFromArray:node_dependencies]; + } + } + + return ![nodes_dependencies containsObject:node]; +} + +ETCoreMLModelStructurePath *_Nullable find_terminal_node_from_nodes(NSArray *nodes, + NSDictionary *> *dependencies) { + if (nodes.count < 2) { + return nodes.firstObject; + } + + for (ETCoreMLModelStructurePath *node in nodes) { + if (is_node_terminal_node(node, nodes, dependencies)) { + return node; + } + } + + return nil; +} + +NSDictionary *get_operation_path_to_debug_symbol_map(ETCoreMLModelDebugInfo *model_debug_info, + NSDictionary *> *debug_handle_to_operation_paths_map, + NSDictionary *> *dependencies) { + // When decomposing an EXIR operation into a MIL graph, it is essential to report the output of the terminal node of the MIL graph. + // This output corresponds directly to the output of the original EXIR operation. + NSUInteger capacity = debug_handle_to_operation_paths_map.count; + NSMutableDictionary *operation_path_to_debug_symbol_map = [NSMutableDictionary dictionaryWithCapacity:capacity]; + [debug_handle_to_operation_paths_map enumerateKeysAndObjectsUsingBlock:^(NSString *debug_handle, + NSArray *operation_paths, + BOOL * __unused stop) { + ETCoreMLModelStructurePath *operation_path = find_terminal_node_from_nodes(operation_paths, dependencies); + NSString *debug_symbol = (operation_path != nil) ? model_debug_info.pathToDebugSymbolMap[operation_path] : nil; + if (debug_symbol) { + operation_path_to_debug_symbol_map[operation_path] = debug_symbol; + } + + }]; + + return operation_path_to_debug_symbol_map; +} + } @interface ETCoreMLModelDebugger () @@ -390,6 +485,8 @@ @interface ETCoreMLModelDebugger () @property (readonly, copy, nonatomic) NSOrderedSet *outputNames; /// The model asset. @property (readonly, copy, nonatomic) ETCoreMLAsset *modelAsset; +/// The model debug info. +@property (readonly, copy, nonatomic, nullable) ETCoreMLModelDebugInfo *modelDebugInfo; /// The asset manager. @property (readonly, copy, nonatomic) ETCoreMLAssetManager *assetManager; /// The model configuration. @@ -404,6 +501,7 @@ @implementation ETCoreMLModelDebugger { } - (nullable instancetype)initWithModelAsset:(ETCoreMLAsset *)modelAsset + modelDebugInfo:(nullable ETCoreMLModelDebugInfo *)modelDebugInfo outputNames:(NSOrderedSet *)outputNames configuration:(MLModelConfiguration *)configuration assetManager:(ETCoreMLAssetManager *)assetManager @@ -422,15 +520,27 @@ - (nullable instancetype)initWithModelAsset:(ETCoreMLAsset *)modelAsset if (!modelSpec) { return nil; } - + + __block NSMutableDictionary *> *dependencies = [NSMutableDictionary dictionary]; __block NSMutableArray *operationPaths = [NSMutableArray array]; + __block NSMutableSet *allOperationPaths = [NSMutableSet set]; visit_program_operation(*modelSpec, ^BOOL(const MILSpec::Operation &operation, ETCoreMLModelStructurePath *operationPath) { + dependencies[operationPath] = get_operation_dependencies(operation, operationPath, allOperationPaths); + [allOperationPaths addObject:operationPath]; if (is_operation_output_supported_as_model_output(operation)) { [operationPaths addObject:operationPath]; } + return YES; }); - + + + NSDictionary *> *debugHandleToOperationPathsMap = get_debug_handle_to_operation_paths_map(modelDebugInfo); + + NSDictionary *operationPathToDebugSymbolMap = get_operation_path_to_debug_symbol_map(modelDebugInfo, + debugHandleToOperationPathsMap, + dependencies); + self = [super init]; if (self) { _modelAsset = modelAsset; @@ -440,6 +550,8 @@ - (nullable instancetype)initWithModelAsset:(ETCoreMLAsset *)modelAsset _modelSpec = std::move(modelSpec); _modelSpecURL = modelSpecURL; _operationPaths = operationPaths; + _operationPathToDebugSymbolMap = operationPathToDebugSymbolMap; + _modelDebugInfo = modelDebugInfo; } return self; diff --git a/backends/apple/coreml/runtime/sdk/ETCoreMLModelProfiler.h b/backends/apple/coreml/runtime/sdk/ETCoreMLModelProfiler.h index a2fbb98582..07a384a516 100644 --- a/backends/apple/coreml/runtime/sdk/ETCoreMLModelProfiler.h +++ b/backends/apple/coreml/runtime/sdk/ETCoreMLModelProfiler.h @@ -5,10 +5,11 @@ // // Please refer to the license found in the LICENSE file in the root directory of the source tree. +#import "ETCoreMLPair.h" #import -#import #import +@class ETCoreMLAsset; @class ETCoreMLModel; @class ETCoreMLModelStructurePath; @class ETCoreMLOperationProfilingInfo; diff --git a/backends/apple/coreml/runtime/sdk/ETCoreMLModelProfiler.mm b/backends/apple/coreml/runtime/sdk/ETCoreMLModelProfiler.mm index 927fb7700e..c9ad324a6c 100644 --- a/backends/apple/coreml/runtime/sdk/ETCoreMLModelProfiler.mm +++ b/backends/apple/coreml/runtime/sdk/ETCoreMLModelProfiler.mm @@ -5,16 +5,17 @@ // // Please refer to the license found in the LICENSE file in the root directory of the source tree. -#import -#import -#import -#import -#import -#import -#import +#import "ETCoreMLModelProfiler.h" + +#import "ETCoreMLAsset.h" +#import "ETCoreMLLogging.h" +#import "ETCoreMLModelStructurePath.h" +#import "ETCoreMLOperationProfilingInfo.h" +#import "ETCoreMLPair.h" +#import "ETCoreMLStrings.h" #import #import -#import +#import "program_path.h" namespace { using namespace executorchcoreml::modelstructure; @@ -331,10 +332,10 @@ - (nullable ETCoreMLModelProfilingResult *)profilingInfoForOperationsAtPaths:(NS return nil; } -- (nullable ETCoreMLModelProfilingResult *)profilingInfoForAllOperationsWithOptions:(MLPredictionOptions *)options - inputs:(id)inputs - modelOutputs:(NSArray *_Nullable __autoreleasing *_Nonnull)modelOutputs - error:(NSError* __autoreleasing *)error { +- (nullable ETCoreMLModelProfilingResult *)profilingInfoForOperationsAtPaths:(MLPredictionOptions *)options + inputs:(id)inputs + modelOutputs:(NSArray *_Nullable __autoreleasing *_Nonnull)modelOutputs + error:(NSError* __autoreleasing *)error { #if MODEL_PROFILING_IS_AVAILABLE if (@available(macOS 14.4, iOS 17.4, tvOS 17.4, watchOS 10.4, *)) { __block NSMutableArray *paths = [NSMutableArray array]; @@ -344,7 +345,7 @@ - (nullable ETCoreMLModelProfilingResult *)profilingInfoForAllOperationsWithOpti } return YES; }); - + return [self profilingInfoForOperationsAtPaths:paths options:options inputs:inputs diff --git a/backends/apple/coreml/runtime/sdk/ETCoreMLModelStructurePath.mm b/backends/apple/coreml/runtime/sdk/ETCoreMLModelStructurePath.mm index ad09e10244..419618ac5e 100644 --- a/backends/apple/coreml/runtime/sdk/ETCoreMLModelStructurePath.mm +++ b/backends/apple/coreml/runtime/sdk/ETCoreMLModelStructurePath.mm @@ -7,7 +7,7 @@ #import "ETCoreMLModelStructurePath.h" -#import +#import "objc_safe_cast.h" namespace { using namespace executorchcoreml::modelstructure; diff --git a/backends/apple/coreml/runtime/sdk/ETCoreMLOperationProfilingInfo.mm b/backends/apple/coreml/runtime/sdk/ETCoreMLOperationProfilingInfo.mm index 4ede54dd1e..c687f47ca2 100644 --- a/backends/apple/coreml/runtime/sdk/ETCoreMLOperationProfilingInfo.mm +++ b/backends/apple/coreml/runtime/sdk/ETCoreMLOperationProfilingInfo.mm @@ -5,9 +5,10 @@ // // Please refer to the license found in the LICENSE file in the root directory of the source tree. -#import -#import -#import +#import "ETCoreMLOperationProfilingInfo.h" + +#import "hash_util.h" +#import "model_event_logger_impl.h" namespace { NSString *const kPreferredComputeUnitKey = @"preferredComputeUnit"; diff --git a/backends/apple/coreml/runtime/sdk/ETCoreMLPair.mm b/backends/apple/coreml/runtime/sdk/ETCoreMLPair.mm index 078be4aeb1..087144a24c 100644 --- a/backends/apple/coreml/runtime/sdk/ETCoreMLPair.mm +++ b/backends/apple/coreml/runtime/sdk/ETCoreMLPair.mm @@ -5,7 +5,7 @@ // // Please refer to the license found in the LICENSE file in the root directory of the source tree. -#import +#import "ETCoreMLPair.h" @implementation ETCoreMLPair diff --git a/backends/apple/coreml/runtime/sdk/model_event_logger_impl.h b/backends/apple/coreml/runtime/sdk/model_event_logger_impl.h index e88d9754ee..474bad2424 100644 --- a/backends/apple/coreml/runtime/sdk/model_event_logger_impl.h +++ b/backends/apple/coreml/runtime/sdk/model_event_logger_impl.h @@ -7,8 +7,8 @@ #pragma once +#import "model_event_logger.h" #import -#import namespace torch::executor { class EventTracer; @@ -27,8 +27,8 @@ class ModelEventLoggerImpl final : public ModelEventLogger { /// /// @param op_path_to_profiling_info_map A dictionary with the operation path as the key and operation's profiling /// info as the value. - /// @param op_path_to_debug_symbol_name_map A dictionary with the operation path as the key and the symbol name as - /// the value. The symbol name is the delegate handle. + /// @param op_path_to_debug_symbol_name_map A dictionary with the operation path as the key and the debug symbol + /// name as the value. void log_profiling_infos( NSDictionary* op_path_to_profiling_info_map, NSDictionary* op_path_to_debug_symbol_name_map) const noexcept override; @@ -37,11 +37,11 @@ class ModelEventLoggerImpl final : public ModelEventLogger { /// /// @param op_path_to_value_map A dictionary with the operation path as the key and the operation's value as the /// value. - /// @param op_path_to_debug_symbol_name_map A dictionary with the operation path as the key and the symbol name as - /// the value. The symbol name is the delegate handle. + /// @param op_path_to_debug_symbol_name_map A dictionary with the operation path as the key and the debug symbol + /// name as the value. void log_intermediate_tensors( NSDictionary* op_path_to_value_map, - NSDictionary* op_path_to_debug_symbol_name_map) const noexcept override; + NSDictionary* op_path_to_debug_symbol_map) const noexcept override; private: torch::executor::EventTracer* tracer_; diff --git a/backends/apple/coreml/runtime/sdk/model_event_logger_impl.mm b/backends/apple/coreml/runtime/sdk/model_event_logger_impl.mm index f90a8f1a41..1d358583a8 100644 --- a/backends/apple/coreml/runtime/sdk/model_event_logger_impl.mm +++ b/backends/apple/coreml/runtime/sdk/model_event_logger_impl.mm @@ -5,13 +5,20 @@ // // Please refer to the license found in the LICENSE file in the root directory of the source tree. -#import -#import +#import "model_event_logger_impl.h" + +#import "ETCoreMLModelStructurePath.h" +#import "ETCoreMLOperationProfilingInfo.h" #import +#import "objc_array_util.h" #import -#import +#import +#import "MLMultiArray_Copy.h" namespace { + +using namespace torch::executor; + uint64_t time_units_to_nano_seconds(uint64_t time_units) { static mach_timebase_info_data_t info; static dispatch_once_t onceToken; @@ -22,6 +29,51 @@ uint64_t time_units_to_nano_seconds(uint64_t time_units) { return time_units * info.numer / info.denom; } +std::optional to_scalar_type(MLMultiArrayDataType data_type) { + switch (data_type) { + case MLMultiArrayDataTypeFloat16: { + return ScalarType::Half; + } + case MLMultiArrayDataTypeFloat32: { + return ScalarType::Float; + } + case MLMultiArrayDataTypeDouble: { + return ScalarType::Double; + } + case MLMultiArrayDataTypeInt32: { + return ScalarType::Int; + } + default: { + return std::nullopt; + } + } +} + +MLMultiArrayDataType get_supported_data_type(MLMultiArrayDataType data_type) { + switch (data_type) { + case MLMultiArrayDataTypeFloat16: { + return MLMultiArrayDataTypeFloat32; + } + default: { + return data_type; + } + } +} + +bool is_packed(NSArray *shape, NSArray *strides) { + if (shape.count == 0) { + return true; + } + size_t product = 1; + for (size_t i = shape.count; i > 0; i--) { + if (![strides[i - 1] isEqual:@(product)]) { + return false; + } + product *= shape[i - 1].unsignedLongValue; + } + + return true; +} } namespace executorchcoreml { @@ -53,7 +105,60 @@ uint64_t time_units_to_nano_seconds(uint64_t time_units) { } void ModelEventLoggerImpl::log_intermediate_tensors(NSDictionary *op_path_to_value_map, - NSDictionary *op_path_to_debug_symbol_name_map) const noexcept { - //TODO: Implement logging for intermediate tensors once ExecuTorch has support. + NSDictionary *op_path_to_debug_symbol_name_map) const noexcept { + [op_path_to_value_map enumerateKeysAndObjectsUsingBlock:^(ETCoreMLModelStructurePath *path, + MLMultiArray *intermediate_value, + BOOL * _Nonnull __unused stop) { + using namespace torch::executor; + + @autoreleasepool { + NSString *debug_symbol = op_path_to_debug_symbol_name_map[path]; + if (debug_symbol == nil) { + return; + } + + MLMultiArray *value = op_path_to_value_map[path]; + if (value == nil || value.count == 0) { + return; + } + + MLMultiArray *supported_value = value; + NSArray *shape = supported_value.shape; + NSError *local_error = nil; + MLMultiArrayDataType data_type = get_supported_data_type(value.dataType); + + if (!is_packed(shape, value.strides) || (supported_value.dataType != data_type)) { + supported_value = [[MLMultiArray alloc] initWithShape:shape + dataType:data_type + error:&local_error]; + NSCAssert(supported_value != nil, + @"ModelEventLoggerImpl: Failed to create packed multiarray with shape=%@, dataType=%ld, error=%@.", + shape, + static_cast(value.dataType), + local_error); + [value copyInto:supported_value]; + } + + + [supported_value getBytesWithHandler:^(const void * _Nonnull bytes, NSInteger size) { + auto sizes = to_vector(shape); + auto strides = to_vector(supported_value.strides); + auto scalar_type = to_scalar_type(data_type); + auto dim_order = std::vector(shape.count); + std::iota(std::begin(dim_order), std::end(dim_order), 0); + + NSCAssert(scalar_type.has_value(), @"ModelEventLoggerImpl: MultiArray dataType=%ld is not supported.", static_cast(data_type)); + auto tensor_impl = TensorImpl( + scalar_type.value(), + static_cast(sizes.size()), + sizes.data(), + const_cast(bytes), + dim_order.data(), + strides.data()); + auto tensor = Tensor(&tensor_impl); + tracer_->log_intermediate_output_delegate(debug_symbol.UTF8String, -1, tensor); + }]; + } + }]; } } // namespace executorchcoreml diff --git a/backends/apple/coreml/runtime/sdk/model_package_info.h b/backends/apple/coreml/runtime/sdk/model_package_info.h index f694a62305..51978002a4 100644 --- a/backends/apple/coreml/runtime/sdk/model_package_info.h +++ b/backends/apple/coreml/runtime/sdk/model_package_info.h @@ -9,7 +9,7 @@ #import -#import +#import "serde_json.h" #import #import diff --git a/backends/apple/coreml/runtime/sdk/model_package_info.mm b/backends/apple/coreml/runtime/sdk/model_package_info.mm index 5e52974b41..b7b26178fd 100644 --- a/backends/apple/coreml/runtime/sdk/model_package_info.mm +++ b/backends/apple/coreml/runtime/sdk/model_package_info.mm @@ -7,9 +7,9 @@ #import "model_package_info.h" -#import -#import -#import +#import "ETCoreMLLogging.h" +#import "objc_json_serde.h" +#import "serde_json.h" namespace { struct ModelPackageInfoKeys { diff --git a/backends/apple/coreml/runtime/test/ETCoreMLModelDebuggerTests.mm b/backends/apple/coreml/runtime/test/ETCoreMLModelDebuggerTests.mm index 95c84ab674..2464ec8dbb 100644 --- a/backends/apple/coreml/runtime/test/ETCoreMLModelDebuggerTests.mm +++ b/backends/apple/coreml/runtime/test/ETCoreMLModelDebuggerTests.mm @@ -16,41 +16,53 @@ #import namespace { - using namespace executorchcoreml::modelstructure; - - using NotifyFn = std::function *op_path_to_value_map, - NSDictionary *op_path_to_debug_symbol_name_map)>; - - class ModelProfilingEventLoggerImpl: public executorchcoreml::ModelEventLogger { - public: - explicit ModelProfilingEventLoggerImpl(NotifyFn fn) - :fn_(fn) - {} - - void log_profiling_infos(NSDictionary *op_path_to_profiling_info_map, - NSDictionary *op_path_to_debug_symbol_name_map) const noexcept {} - - void log_intermediate_tensors(NSDictionary *op_path_to_value_map, - NSDictionary *op_path_to_debug_symbol_name_map) const noexcept { - fn_(op_path_to_value_map, op_path_to_debug_symbol_name_map); +using namespace executorchcoreml::modelstructure; + +using NotifyFn = std::function *op_path_to_value_map, + NSDictionary *op_path_to_debug_symbol_name_map)>; + +class ModelEventLoggerImpl: public executorchcoreml::ModelEventLogger { +public: + explicit ModelEventLoggerImpl(NotifyFn fn) + :fn_(fn) + {} + + void log_profiling_infos(NSDictionary *op_path_to_profiling_info_map, + NSDictionary *op_path_to_debug_symbol_name_map) const noexcept {} + + void log_intermediate_tensors(NSDictionary *op_path_to_value_map, + NSDictionary *op_path_to_debug_symbol_map) const noexcept { + fn_(op_path_to_value_map, op_path_to_debug_symbol_map); + } + +private: + NotifyFn fn_; +}; + +ETCoreMLModelStructurePath *make_path_with_output_name(const std::string& output_name, + const std::string& function_name = "main") { + Path path; + path.append_component(Path::Program()); + path.append_component(Path::Program::Function(function_name)); + path.append_component(Path::Program::Block(-1)); + path.append_component(Path::Program::Operation(output_name)); + + return [[ETCoreMLModelStructurePath alloc] initWithUnderlyingValue:std::move(path)]; +} + +void add_debugging_result(NSDictionary *debugging_result, + NSDictionary *path_to_symbol_name_map, + NSMutableDictionary *debugging_results) { + for (ETCoreMLModelStructurePath *path in debugging_result) { + NSString *debug_symbol = path_to_symbol_name_map[path]; + if (debug_symbol) { + debugging_results[path] = debugging_result[path]; } - - private: - NotifyFn fn_; - }; - - ETCoreMLModelStructurePath *make_path_with_output_name(const std::string& output_name, - const std::string& function_name = "main") { - Path path; - path.append_component(Path::Program()); - path.append_component(Path::Program::Function(function_name)); - path.append_component(Path::Program::Block(-1)); - path.append_component(Path::Program::Operation(output_name)); - - return [[ETCoreMLModelStructurePath alloc] initWithUnderlyingValue:std::move(path)]; } } +} + @interface ETCoreMLModelDebuggerTests : XCTestCase @end @@ -87,8 +99,8 @@ - (void)debugModelWithName:(NSString *)modelName MLPredictionOptions *predictionOptions = [[MLPredictionOptions alloc] init]; executorchcoreml::ModelLoggingOptions loggingOptions; loggingOptions.log_intermediate_tensors = true; - ModelProfilingEventLoggerImpl eventLogger(notify); - + ModelEventLoggerImpl eventLogger(notify); + NSArray *outputs = [analyzer executeModelWithInputs:inputs predictionOptions:predictionOptions loggingOptions:loggingOptions @@ -125,21 +137,39 @@ - (void)testMulProgramDebugging { } - (void)testMV3ProgramDebugging { - XCTSkip(@"There is a device specialization issue when getting on of the outputs, will fix after investigation."); - NotifyFn notify = [](NSDictionary *debuggingResult, - NSDictionary *pathToSymbolNameMap) { - // There are more than 200 ops, we verify the outputs for specific ops. - XCTAssertNotNil(debuggingResult[make_path_with_output_name("aten__native_batch_norm_legit_no_training_default_13_cast_fp16")]); - XCTAssertNotNil(debuggingResult[make_path_with_output_name("_inversed_aten_div_tensor_24_cast_fp16")]); - XCTAssertNotNil(debuggingResult[make_path_with_output_name("aten_mean_dim_7_cast_fp16")]); - XCTAssertNotNil(debuggingResult[make_path_with_output_name("aten_clamp_default_54_cast_fp16")]); - XCTAssertNotNil(debuggingResult[make_path_with_output_name("aten__native_batch_norm_legit_no_training_default_22_cast_fp16")]); - XCTAssertNotNil(debuggingResult[make_path_with_output_name("aten_mul_tensor_27_cast_fp16")]); + NSMutableDictionary *debuggingResults = [NSMutableDictionary new]; + NotifyFn notify = [debuggingResults](NSDictionary *debuggingResult, + NSDictionary *pathToSymbolNameMap) mutable { + add_debugging_result(debuggingResult, pathToSymbolNameMap, debuggingResults); }; [self debugModelWithName:@"mv3_coreml_all" repeatedInputValues:@[@(1), @(2)] notify:notify]; + + // There are more than 200 ops, we verify the outputs for specific ops. + XCTAssertNotNil(debuggingResults[make_path_with_output_name("aten__native_batch_norm_legit_no_training_default_13_cast_fp16")]); + XCTAssertNotNil(debuggingResults[make_path_with_output_name("_inversed_aten_div_tensor_24_cast_fp16")]); + XCTAssertNotNil(debuggingResults[make_path_with_output_name("aten_mean_dim_7_cast_fp16")]); + XCTAssertNotNil(debuggingResults[make_path_with_output_name("aten_clamp_default_54_cast_fp16")]); + XCTAssertNotNil(debuggingResults[make_path_with_output_name("aten__native_batch_norm_legit_no_training_default_22_cast_fp16")]); + XCTAssertNotNil(debuggingResults[make_path_with_output_name("aten_mul_tensor_27_cast_fp16")]); +} + +- (void)testAddMulProgramDebugging { + NSMutableDictionary *debuggingResults = [NSMutableDictionary new]; + NotifyFn notify = [debuggingResults](NSDictionary *debuggingResult, + NSDictionary *pathToSymbolNameMap) mutable { + add_debugging_result(debuggingResult, pathToSymbolNameMap, debuggingResults); + }; + + [self debugModelWithName:@"add_mul_coreml_all" + repeatedInputValues:@[@(1), @(2)] + notify:notify]; + + // There are more than 200 ops, we verify the outputs for specific ops. + XCTAssertNotNil(debuggingResults[make_path_with_output_name("aten_add_tensor")]); + XCTAssertNotNil(debuggingResults[make_path_with_output_name("aten_mm_default_cast_fp16")]); } @end diff --git a/backends/apple/coreml/runtime/test/ETCoreMLTestUtils.mm b/backends/apple/coreml/runtime/test/ETCoreMLTestUtils.mm index 72ad71adc9..3c0908201a 100644 --- a/backends/apple/coreml/runtime/test/ETCoreMLTestUtils.mm +++ b/backends/apple/coreml/runtime/test/ETCoreMLTestUtils.mm @@ -8,16 +8,17 @@ #import "ETCoreMLTestUtils.h" -#import -#import -#import -#import -#import +#import "ETCoreMLAsset.h" +#import "ETCoreMLLogging.h" +#import "ETCoreMLModelDebugInfo.h" +#import "ETCoreMLModelAnalyzer.h" +#import "ETCoreMLModelCompiler.h" +#import "ETCoreMLStrings.h" #import -#import +#import "inmemory_filesystem_utils.hpp" #import #import -#import +#import "model_metadata.h" namespace { NSURL * _Nullable create_directory_if_needed(NSURL *url, @@ -239,6 +240,7 @@ + (nullable MLMultiArray *)filledMultiArrayWithConstraint:(MLMultiArrayConstrain + (BOOL)extractModelAssetAndMetadataFromAOTData:(NSData *)data modelAsset:(ETCoreMLAsset *_Nullable __autoreleasing *_Nonnull)modelAsset + modelDebugInfo:(ETCoreMLModelDebugInfo *_Nullable __autoreleasing *_Nonnull)modelDebugInfo metadata:(executorchcoreml::ModelMetadata&)metadata dstURL:(NSURL *)dstURL fileManager:(NSFileManager *)fileManager @@ -295,18 +297,35 @@ + (BOOL)extractModelAssetAndMetadataFromAOTData:(NSData *)data if (modelAsset) { *modelAsset = localAsset; } - + + __block auto debugInfoBuffer = inMemoryFS->get_file_content({ETCoreMLStrings.debugInfoFileRelativePath.UTF8String}, ec); + if (debugInfoBuffer && debugInfoBuffer->size() > 0) { + + NSData *data = [[NSData alloc] initWithBytesNoCopy:debugInfoBuffer->data() + length:debugInfoBuffer->size() + deallocator:^(void * _Nonnull __unused bytes, NSUInteger __unused length) { + debugInfoBuffer.reset(); + }]; + + ETCoreMLModelDebugInfo *lModelDebugInfo = [ETCoreMLModelDebugInfo modelDebugInfoFromData:data error:nil]; + if (modelDebugInfo) { + *modelDebugInfo = lModelDebugInfo; + } + } + return YES; } + (ETCoreMLModelAnalyzer *)createAnalyzerWithAOTData:(NSData *)data - dstURL:(NSURL *)dstURL - error:(NSError * __autoreleasing *)error { + dstURL:(NSURL *)dstURL + error:(NSError * __autoreleasing *)error { ETCoreMLAsset *modelAsset = nil; + ETCoreMLModelDebugInfo *modelDebugInfo = nil; executorchcoreml::ModelMetadata metadata; NSFileManager *fileManager = [[NSFileManager alloc] init]; if (![self extractModelAssetAndMetadataFromAOTData:data modelAsset:&modelAsset + modelDebugInfo:&modelDebugInfo metadata:metadata dstURL:dstURL fileManager:fileManager @@ -343,12 +362,12 @@ + (ETCoreMLModelAnalyzer *)createAnalyzerWithAOTData:(NSData *)data MLModelConfiguration *configuration = [[MLModelConfiguration alloc] init]; ETCoreMLModelAnalyzer *analyzer = [[ETCoreMLModelAnalyzer alloc] initWithCompiledModelAsset:compiledModelAsset modelAsset:modelAsset + modelDebugInfo:modelDebugInfo metadata:metadata - operationPathToDebugSymbolMap:nil configuration:configuration assetManager:assetManager error:error]; - + return analyzer; } diff --git a/backends/apple/coreml/runtime/util/objc_array_util.h b/backends/apple/coreml/runtime/util/objc_array_util.h index 5f4c8c7bc2..b3446a2288 100644 --- a/backends/apple/coreml/runtime/util/objc_array_util.h +++ b/backends/apple/coreml/runtime/util/objc_array_util.h @@ -18,6 +18,8 @@ template <> inline size_t to_value(NSNumber* value) { return value.unsignedLongV template <> inline ssize_t to_value(NSNumber* value) { return value.longLongValue; } +template <> inline int to_value(NSNumber* value) { return value.intValue; } + template ::value, T>::type> inline NSArray* to_array(const std::vector& array) { NSMutableArray* result = [NSMutableArray arrayWithCapacity:array.size()]; diff --git a/backends/apple/coreml/runtime/workspace/executorchcoreml.xcodeproj/project.pbxproj b/backends/apple/coreml/runtime/workspace/executorchcoreml.xcodeproj/project.pbxproj index d8a5e61107..c347c56db0 100644 --- a/backends/apple/coreml/runtime/workspace/executorchcoreml.xcodeproj/project.pbxproj +++ b/backends/apple/coreml/runtime/workspace/executorchcoreml.xcodeproj/project.pbxproj @@ -7,6 +7,9 @@ objects = { /* Begin PBXBuildFile section */ + 83BB78A02C65DA7300274ED7 /* ETCoreMLModelDebugInfo.mm in Sources */ = {isa = PBXBuildFile; fileRef = 83BB789F2C65DA7300274ED7 /* ETCoreMLModelDebugInfo.mm */; }; + 83BB78BF2C66AAAE00274ED7 /* add_mul_coreml_all.bin in Resources */ = {isa = PBXBuildFile; fileRef = 83BB78BD2C66AAAE00274ED7 /* add_mul_coreml_all.bin */; }; + 83BB78C02C66AAAE00274ED7 /* add_mul_coreml_all.pte in Resources */ = {isa = PBXBuildFile; fileRef = 83BB78BE2C66AAAE00274ED7 /* add_mul_coreml_all.pte */; }; C945E8E02B997ECE009C3FAC /* ETCoreMLModelProfiler.mm in Sources */ = {isa = PBXBuildFile; fileRef = C945E8CF2B997ECD009C3FAC /* ETCoreMLModelProfiler.mm */; }; C945E8E12B997ECE009C3FAC /* ETCoreMLModelAnalyzer.mm in Sources */ = {isa = PBXBuildFile; fileRef = C945E8D42B997ECD009C3FAC /* ETCoreMLModelAnalyzer.mm */; }; C945E8E22B997ECE009C3FAC /* program_path.mm in Sources */ = {isa = PBXBuildFile; fileRef = C945E8D52B997ECD009C3FAC /* program_path.mm */; }; @@ -100,8 +103,8 @@ C9E7D7952AB3F9BF00CCAE5D /* ETCoreMLModelManagerTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = C9E7D78D2AB3F9BF00CCAE5D /* ETCoreMLModelManagerTests.mm */; }; C9E7D7962AB3F9BF00CCAE5D /* KeyValueStoreTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = C9E7D78E2AB3F9BF00CCAE5D /* KeyValueStoreTests.mm */; }; C9E7D7A22AB3FBB200CCAE5D /* CoreMLBackendDelegateTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = C9E7D7A12AB3FBB200CCAE5D /* CoreMLBackendDelegateTests.mm */; }; - F24817E52BC655E100E80D98 /* libexecutorch_no_prim_ops.a in Frameworks */ = {isa = PBXBuildFile; fileRef = F24817E42BC655E100E80D98 /* libexecutorch_no_prim_ops.a */; }; C9EC7E1B2BC73B3200A6B166 /* MultiArrayTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = C9EC7E1A2BC73B3200A6B166 /* MultiArrayTests.mm */; }; + F24817E52BC655E100E80D98 /* libexecutorch_no_prim_ops.a in Frameworks */ = {isa = PBXBuildFile; fileRef = F24817E42BC655E100E80D98 /* libexecutorch_no_prim_ops.a */; }; /* End PBXBuildFile section */ /* Begin PBXCopyFilesBuildPhase section */ @@ -117,6 +120,10 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ + 83BB789E2C65DA7300274ED7 /* ETCoreMLModelDebugInfo.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = ETCoreMLModelDebugInfo.h; path = ../sdk/ETCoreMLModelDebugInfo.h; sourceTree = ""; }; + 83BB789F2C65DA7300274ED7 /* ETCoreMLModelDebugInfo.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; name = ETCoreMLModelDebugInfo.mm; path = ../sdk/ETCoreMLModelDebugInfo.mm; sourceTree = ""; }; + 83BB78BD2C66AAAE00274ED7 /* add_mul_coreml_all.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; name = add_mul_coreml_all.bin; path = ../test/models/add_mul_coreml_all.bin; sourceTree = ""; }; + 83BB78BE2C66AAAE00274ED7 /* add_mul_coreml_all.pte */ = {isa = PBXFileReference; lastKnownFileType = file; name = add_mul_coreml_all.pte; path = ../test/models/add_mul_coreml_all.pte; sourceTree = ""; }; C945E8CD2B997ECD009C3FAC /* ETCoreMLModelProfiler.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ETCoreMLModelProfiler.h; path = ../sdk/ETCoreMLModelProfiler.h; sourceTree = ""; }; C945E8CE2B997ECD009C3FAC /* ETCoreMLModelAnalyzer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ETCoreMLModelAnalyzer.h; path = ../sdk/ETCoreMLModelAnalyzer.h; sourceTree = ""; }; C945E8CF2B997ECD009C3FAC /* ETCoreMLModelProfiler.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; name = ETCoreMLModelProfiler.mm; path = ../sdk/ETCoreMLModelProfiler.mm; sourceTree = ""; }; @@ -299,9 +306,9 @@ C9EA3DB22B71A2B200B7D7BD /* CoreML.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreML.framework; path = System/Library/Frameworks/CoreML.framework; sourceTree = SDKROOT; }; C9EA3FDE2B73EEA000B7D7BD /* libsqlite3.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = libsqlite3.tbd; path = usr/lib/libsqlite3.tbd; sourceTree = SDKROOT; }; C9EA3FE52B73EF6300B7D7BD /* Accelerate.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Accelerate.framework; path = System/Library/Frameworks/Accelerate.framework; sourceTree = SDKROOT; }; - F24817E42BC655E100E80D98 /* libexecutorch_no_prim_ops.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libexecutorch_no_prim_ops.a; path = ../libraries/libexecutorch_no_prim_ops.a; sourceTree = ""; }; C9EC7E092BC662A300A6B166 /* objc_array_util.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = objc_array_util.h; path = ../util/objc_array_util.h; sourceTree = ""; }; C9EC7E1A2BC73B3200A6B166 /* MultiArrayTests.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; name = MultiArrayTests.mm; path = ../test/MultiArrayTests.mm; sourceTree = ""; }; + F24817E42BC655E100E80D98 /* libexecutorch_no_prim_ops.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libexecutorch_no_prim_ops.a; path = ../libraries/libexecutorch_no_prim_ops.a; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -329,6 +336,8 @@ C945E8D42B997ECD009C3FAC /* ETCoreMLModelAnalyzer.mm */, C945E8D32B997ECD009C3FAC /* ETCoreMLModelDebugger.h */, C945E8DF2B997ECE009C3FAC /* ETCoreMLModelDebugger.mm */, + 83BB789E2C65DA7300274ED7 /* ETCoreMLModelDebugInfo.h */, + 83BB789F2C65DA7300274ED7 /* ETCoreMLModelDebugInfo.mm */, C945E8CD2B997ECD009C3FAC /* ETCoreMLModelProfiler.h */, C945E8CF2B997ECD009C3FAC /* ETCoreMLModelProfiler.mm */, C945E8D02B997ECD009C3FAC /* ETCoreMLModelStructurePath.h */, @@ -596,6 +605,8 @@ C985519C2AD2542D009143F9 /* mul_coreml_all.pte */, C985519B2AD2542D009143F9 /* mv3_coreml_all.bin */, C98551982AD2542D009143F9 /* mv3_coreml_all.pte */, + 83BB78BD2C66AAAE00274ED7 /* add_mul_coreml_all.bin */, + 83BB78BE2C66AAAE00274ED7 /* add_mul_coreml_all.pte */, ); name = models; sourceTree = ""; @@ -661,6 +672,8 @@ files = ( C98551A12AD2542D009143F9 /* mv3_coreml_all.bin in Resources */, C985519F2AD2542D009143F9 /* mul_coreml_all.bin in Resources */, + 83BB78BF2C66AAAE00274ED7 /* add_mul_coreml_all.bin in Resources */, + 83BB78C02C66AAAE00274ED7 /* add_mul_coreml_all.pte in Resources */, C985519E2AD2542D009143F9 /* mv3_coreml_all.pte in Resources */, C98551A02AD2542D009143F9 /* add_coreml_all.bin in Resources */, C98551A22AD2542D009143F9 /* mul_coreml_all.pte in Resources */, @@ -695,6 +708,7 @@ C94D51042ABDF84100AF47FD /* ETCoreMLStrings.mm in Sources */, C9E7D7932AB3F9BF00CCAE5D /* ETCoreMLAssetManagerTests.mm in Sources */, C945E8E42B997ECE009C3FAC /* model_event_logger_impl.mm in Sources */, + 83BB78A02C65DA7300274ED7 /* ETCoreMLModelDebugInfo.mm in Sources */, C945E9472B997EEE009C3FAC /* SoundAnalysisPreprocessing.pb.cc in Sources */, C97716B52AEA21B600FC0DAC /* inmemory_filesystem_utils.mm in Sources */, C9E7D7922AB3F9BF00CCAE5D /* DatabaseTests.mm in Sources */, diff --git a/backends/apple/coreml/scripts/generate_test_models.sh b/backends/apple/coreml/scripts/generate_test_models.sh index 7beca63726..bbe9809ff8 100755 --- a/backends/apple/coreml/scripts/generate_test_models.sh +++ b/backends/apple/coreml/scripts/generate_test_models.sh @@ -20,7 +20,7 @@ mkdir "$COREML_DIR_PATH/runtime/test/models/" echo "Executorch: Generating test models" cd "$EXECUTORCH_ROOT_PATH" -MODELS=("add" "mul" "mv3") +MODELS=("add" "add_mul" "mul" "mv3") for MODEL in "${MODELS[@]}" do # TODO: Don't use the script in examples directory. diff --git a/examples/apple/coreml/scripts/build_executor_runner.sh b/examples/apple/coreml/scripts/build_executor_runner.sh index 2623a3c76f..16c5dea02a 100755 --- a/examples/apple/coreml/scripts/build_executor_runner.sh +++ b/examples/apple/coreml/scripts/build_executor_runner.sh @@ -38,9 +38,9 @@ cmake "$EXECUTORCH_ROOT_PATH" -B"$CMAKE_BUILD_DIR_PATH" \ -DEXECUTORCH_BUILD_XNNPACK=OFF \ -DEXECUTORCH_BUILD_SDK=ON \ -DEXECUTORCH_BUILD_COREML=ON \ --DCOREML_BUILD_EXECUTOR_RUNNER=ON \ -Dprotobuf_BUILD_TESTS=OFF \ -Dprotobuf_BUILD_EXAMPLES=OFF \ +-DCOREML_BUILD_EXECUTOR_RUNNER=ON \ -DCMAKE_MACOSX_BUNDLE=OFF \ cmake --build "$CMAKE_BUILD_DIR_PATH" -j9 -t coremldelegate diff --git a/examples/apple/coreml/scripts/debugger_cli.py b/examples/apple/coreml/scripts/debugger_cli.py new file mode 100644 index 0000000000..cb978de074 --- /dev/null +++ b/examples/apple/coreml/scripts/debugger_cli.py @@ -0,0 +1,181 @@ +# Copyright © 2024 Apple Inc. All rights reserved. +# +# Please refer to the license found in the LICENSE file in the root directory of the source tree. + +import argparse +import sys +import tempfile +from pathlib import Path +from typing import List, Tuple + +import coremltools as ct +from executorch.backends.apple.coreml.compiler import CoreMLBackend +from executorch.exir import EdgeProgramManager + +from executorch.exir.backend.compile_spec_schema import CompileSpec +from executorch.exir.tracer import Value +from tabulate import tabulate + + +def get_root_dir_path() -> Path: + return Path(__file__).resolve().parent.parent.parent.parent.parent + + +sys.path.append(str((get_root_dir_path() / "examples").resolve())) + +from inspector_utils import ( + build_sdk_runner_including_coreml, + ComparisonResult, + create_inspector_coreml, + create_inspector_reference, + get_comparison_result, + module_to_edge, +) + +from models import MODEL_NAME_TO_MODEL +from models.model_factory import EagerModelFactory + + +def args_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + + parser.add_argument( + "-m", + "--model_name", + required=True, + help=f"Provide model name. Valid ones: {list(MODEL_NAME_TO_MODEL.keys())}", + ) + + parser.add_argument( + "-c", + "--compute_unit", + required=False, + default=ct.ComputeUnit.ALL.name.lower(), + help=f"Provide compute unit for the model. Valid ones: {[[compute_unit.name.lower() for compute_unit in ct.ComputeUnit]]}", + ) + + parser.add_argument( + "-precision", + "--compute_precision", + required=False, + default=ct.precision.FLOAT16.value, + help=f"Provide compute precision for the model. Valid ones: {[[precision.value for precision in ct.precision]]}", + ) + + parser.add_argument( + "--compile", + action=argparse.BooleanOptionalAction, + required=False, + default=False, + ) + + parser.add_argument( + "-env", + "--conda_environment_name", + required=False, + default="executorch", + help="Provide conda environment name.", + ) + + return parser + + +def get_compile_specs_from_args(args): + model_type = CoreMLBackend.MODEL_TYPE.MODEL + if args.compile: + model_type = CoreMLBackend.MODEL_TYPE.COMPILED_MODEL + + compute_precision = ct.precision(args.compute_precision) + compute_unit = ct.ComputeUnit[args.compute_unit.upper()] + + return CoreMLBackend.generate_compile_specs( + compute_precision=compute_precision, + compute_unit=compute_unit, + model_type=model_type, + minimum_deployment_target=ct.target.iOS17, + ) + + +def compare_intermediate_tensors( + edge_program: EdgeProgramManager, + example_inputs: Tuple[Value, ...], + coreml_compile_specs: List[CompileSpec], + model_name: str, + working_dir_path: Path, +) -> ComparisonResult: + inspector_coreml = create_inspector_coreml( + edge_program=edge_program, + compile_specs=coreml_compile_specs, + example_inputs=example_inputs, + model_name=model_name, + working_dir_path=working_dir_path, + root_dir_path=get_root_dir_path(), + ) + + inspector_reference = create_inspector_reference( + edge_program=edge_program, + example_inputs=example_inputs, + model_name=model_name, + working_dir_path=working_dir_path, + root_dir_path=get_root_dir_path(), + ) + + return get_comparison_result( + inspector1=inspector_reference, + tag1="reference", + inspector2=inspector_coreml, + tag2="coreml", + ) + + +def main() -> None: + parser = args_parser() + args = parser.parse_args() + + if args.model_name not in MODEL_NAME_TO_MODEL: + raise RuntimeError( + f"Model {args.model_name} is not a valid name. " + f"Available models are {list(MODEL_NAME_TO_MODEL.keys())}." + ) + + valid_compute_units = [compute_unit.name.lower() for compute_unit in ct.ComputeUnit] + if args.compute_unit not in valid_compute_units: + raise RuntimeError( + f"{args.compute_unit} is invalid. " + f"Valid compute units are {valid_compute_units}." + ) + + build_sdk_runner_including_coreml( + root_dir_path=get_root_dir_path(), conda_env_name=args.conda_environment_name + ) + + model, example_inputs, _ = EagerModelFactory.create_model( + *MODEL_NAME_TO_MODEL[args.model_name] + ) + + model.eval() + edge_program = module_to_edge( + module=model, + example_inputs=example_inputs, + ) + + coreml_compile_specs = get_compile_specs_from_args(args) + + with tempfile.TemporaryDirectory() as temp_dir_name: + working_dir_path = Path(temp_dir_name) / "debugger" + working_dir_path.mkdir(parents=True, exist_ok=True) + comparison_result = compare_intermediate_tensors( + edge_program=edge_program, + example_inputs=example_inputs, + coreml_compile_specs=coreml_compile_specs, + model_name=args.model_name, + working_dir_path=working_dir_path, + ) + + print( + tabulate(comparison_result.to_dataframe(), headers="keys", tablefmt="grid") + ) + + +if __name__ == "__main__": + main() # pragma: no cover diff --git a/examples/apple/coreml/scripts/extract_coreml_models.py b/examples/apple/coreml/scripts/extract_coreml_models.py index 6317b0f3d3..d2812d37ab 100644 --- a/examples/apple/coreml/scripts/extract_coreml_models.py +++ b/examples/apple/coreml/scripts/extract_coreml_models.py @@ -57,7 +57,7 @@ def extract_coreml_models(pte_data: bytes): model_index += 1 if len(coreml_delegates) == 0: - print("The model isn't delegated to CoreML.") + print("The model isn't delegated to Core ML.") if __name__ == "__main__": diff --git a/examples/apple/coreml/scripts/inspector_cli.py b/examples/apple/coreml/scripts/inspector_cli.py index 077c8c26ef..768465f770 100644 --- a/examples/apple/coreml/scripts/inspector_cli.py +++ b/examples/apple/coreml/scripts/inspector_cli.py @@ -1,43 +1,24 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. +# Copyright © 2024 Apple Inc. All rights reserved. # -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. +# Please refer to the license found in the LICENSE file in the root directory of the source tree. import argparse -import json -from typing import Any, Dict, Final, List, Tuple, Union +import sys + +from pathlib import Path from executorch.sdk import Inspector from executorch.sdk.inspector._inspector_utils import compare_results -COREML_METADATA_KEYS: Final[List[Tuple[str, str]]] = [ - ("operatorName", "coreml_operator"), - ("estimatedCost", "coreml_estimated_cost"), - ("preferredComputeUnit", "coreml_preferred_device"), - ("supportedComputeUnits", "coreml_supported_devices"), -] - -def parse_coreml_delegate_metadata(delegate_metadatas: List[str]) -> Dict[str, Any]: - try: - coreml_metadata: Dict[str, Any] = json.loads(delegate_metadatas[0]) - result: Dict[str, str] = {} - for col_key, col_name in COREML_METADATA_KEYS: - value = coreml_metadata.get(col_key, None) - if value is not None: - result[col_name] = value - return result +def get_root_dir_path() -> Path: + return Path().resolve().parent.parent.parent.parent - except ValueError: - return {} +sys.path.append(str((get_root_dir_path() / "examples").resolve())) -def convert_coreml_delegate_time( - event_name: Union[str, int], input_time: Union[int, float] -) -> Union[int, float]: - return input_time / (1000 * 1000) +from inspector_utils import convert_coreml_delegate_time, parse_coreml_delegate_metadata def main() -> None: @@ -55,7 +36,7 @@ def main() -> None: parser.add_argument( "--debug_buffer_path", required=False, - help="Provide an optional buffer file path.", + help="Provide an optional debug buffer file path.", ) parser.add_argument("--compare_results", action="store_true") diff --git a/examples/apple/coreml/scripts/inspector_utils.py b/examples/apple/coreml/scripts/inspector_utils.py new file mode 100644 index 0000000000..1736c2cefb --- /dev/null +++ b/examples/apple/coreml/scripts/inspector_utils.py @@ -0,0 +1,428 @@ +# Copyright © 2024 Apple Inc. All rights reserved. +# +# Please refer to the license found in the LICENSE file in the root directory of the source tree. + +import copy +import errno +import json +import os + +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from typing import Any, Dict, Final, List, Optional, Tuple, Union + +import executorch.exir as exir + +import pandas as pd +import torch +from executorch.backends.apple.coreml.compiler import CoreMLBackend +from executorch.backends.apple.coreml.partition import CoreMLPartitioner + +from executorch.exir import ( + EdgeProgramManager, + ExecutorchBackendConfig, + ExecutorchProgramManager, + ExirExportedProgram, + to_edge, +) +from executorch.exir.backend.compile_spec_schema import CompileSpec +from executorch.exir.tracer import Value + +from executorch.sdk import BundledProgram, generate_etrecord, Inspector + +from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite +from executorch.sdk.bundled_program.serialize import ( + serialize_from_bundled_program_to_flatbuffer, +) +from executorch.sdk.inspector import Event + +from torch.export import export, ExportedProgram + +COREML_METADATA_KEYS: Final[List[Tuple[str, str]]] = [ + ("operatorName", "coreml_operator"), + ("estimatedCost", "coreml_estimated_cost"), + ("preferredComputeUnit", "coreml_preferred_device"), + ("supportedComputeUnits", "coreml_supported_devices"), +] + + +def build_sdk_runner_including_coreml( + root_dir_path: Path, + conda_env_name: str, + force: bool = False, +): + if not force: + sdk_executable_path = ( + root_dir_path / "cmake-out" / "examples" / "sdk" / "sdk_example_runner" + ) + print(sdk_executable_path) + if sdk_executable_path.is_file(): + return + + cd_root_command: str = f"cd {root_dir_path.resolve()}" + conda_activate_env_command: str = f"source conda activate {conda_env_name}" + build_sdk_runner_command: str = ( + "./examples/sdk/build_sdk_example_runner.sh --coreml" + ) + build_command: str = ( + f"{cd_root_command} && {conda_activate_env_command} && {build_sdk_runner_command}" + ) + subprocess.run( + f'bash -c "{build_command}"', shell=True, check=True + ).check_returncode() + + +_EDGE_COMPILE_CONFIG = exir.EdgeCompileConfig( + _check_ir_validity=False, + _skip_dim_order=True, +) + +_EDGE_BACKEND_CONFIG = exir.ExecutorchBackendConfig( + extract_constant_segment=False, + extract_delegate_segments=True, +) + + +def to_core_aten( + module: torch.nn.Module, + example_inputs: Tuple[Value, ...], +) -> ExportedProgram: + core_aten_program = export( + mod=module, + args=example_inputs, + ) + return core_aten_program + + +def core_aten_to_edge( + core_aten_program: ExportedProgram, + edge_compile_config: exir.EdgeCompileConfig, +) -> EdgeProgramManager: + edge_manager = to_edge( + programs=core_aten_program, + compile_config=edge_compile_config, + ) + return edge_manager + + +def module_to_edge( + module: torch.nn.Module, + example_inputs: Tuple[Value, ...], + edge_compile_config: exir.EdgeCompileConfig = _EDGE_COMPILE_CONFIG, +) -> EdgeProgramManager: + module.eval() + core_aten_program = to_core_aten( + module=module, + example_inputs=example_inputs, + ) + return core_aten_to_edge( + core_aten_program=core_aten_program, + edge_compile_config=edge_compile_config, + ) + + +def lower_and_export_edge_to_coreml( + edge_program: EdgeProgramManager, + compile_specs: List[CompileSpec], + config: ExecutorchBackendConfig, + skip_ops_for_coreml_delegation: Optional[List[str]] = None, +) -> ExirExportedProgram: + partitioner = CoreMLPartitioner( + skip_ops_for_coreml_delegation=skip_ops_for_coreml_delegation, + compile_specs=compile_specs, + ) + delegated_program_manager = edge_program.to_backend( + partitioner, + ) + executorch_program = delegated_program_manager.to_executorch( + config=config, + ) + return executorch_program + + +def write_to_file(buffer: bytes, file_path: Path): + with open(file_path.resolve(), "wb") as file: + file.write(buffer) + + +def generate_bundled_program( + executorch_program: ExecutorchProgramManager, + example_inputs: Tuple[Value, ...], + method_name: str, + bundled_program_path: Path, +): + method_test_suites = [ + MethodTestSuite( + method_name=method_name, + test_cases=[MethodTestCase(inputs=example_inputs)], + ) + ] + + bundled_program = BundledProgram(executorch_program, method_test_suites) + bundled_program_buffer = serialize_from_bundled_program_to_flatbuffer( + bundled_program + ) + + write_to_file(buffer=bundled_program_buffer, file_path=bundled_program_path) + + +def generate_etdump_with_intermediate_values( + root_dir_path: Path, + bundled_program_path: Path, + et_dump_path: Path, + debug_buffer_path: Path, + debug_buffer_size: int, +): + sdk_executable_path = ( + root_dir_path / "cmake-out" / "examples" / "sdk" / "sdk_example_runner" + ) + if not sdk_executable_path.is_file(): + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), str(sdk_executable_path.resolve()) + ) + + sdk_runner_command: str = f""" + {sdk_executable_path.resolve()} -dump_intermediate_outputs\ + -bundled_program_path {bundled_program_path.resolve()}\ + -etdump_path {et_dump_path.resolve()}\ + -debug_output_path {debug_buffer_path.resolve()}\ + -debug_buffer_size {debug_buffer_size}""" + subprocess.run( + f'bash -c "{sdk_runner_command}"', shell=True, check=True + ).check_returncode() + + +def create_inspector( + edge_program: EdgeProgramManager, + executorch_program: ExecutorchProgramManager, + example_inputs: Tuple[Value, ...], + model_name: str, + root_dir_path: Path, + working_dir_path: Path, + method_name: str = "forward", + debug_buffer_size: int = 1 * 1024 * 1024 * 1024, + delegate_metadata_parser=None, + delegate_time_scale_converter=None, +) -> Inspector: + et_record_path = working_dir_path / f"{model_name}_etrecord.bin" + generate_etrecord( + et_record=et_record_path.resolve(), + edge_dialect_program=edge_program, + executorch_program=executorch_program, + ) + + bundled_program_path = working_dir_path / f"{model_name}.bpte" + generate_bundled_program( + executorch_program=executorch_program, + example_inputs=example_inputs, + method_name=method_name, + bundled_program_path=bundled_program_path, + ) + + et_dump_path: Path = working_dir_path / f"{model_name}_etdump.etdp" + debug_buffer_path: Path = working_dir_path / f"{model_name}_debug_output.bin" + generate_etdump_with_intermediate_values( + root_dir_path=root_dir_path, + bundled_program_path=bundled_program_path, + et_dump_path=et_dump_path, + debug_buffer_path=debug_buffer_path, + debug_buffer_size=debug_buffer_size, + ) + + return Inspector( + etdump_path=str(et_dump_path.resolve()), + etrecord=str(et_record_path.resolve()), + debug_buffer_path=str(debug_buffer_path.resolve()), + enable_module_hierarchy=True, + delegate_metadata_parser=delegate_metadata_parser, + delegate_time_scale_converter=delegate_time_scale_converter, + ) + + +def parse_coreml_delegate_metadata(delegate_metadatas: List[str]) -> Dict[str, Any]: + if len(delegate_metadatas) == 0: + return + try: + coreml_metadata: Dict[str, Any] = json.loads(delegate_metadatas[0]) + result: Dict[str, str] = {} + for col_key, col_name in COREML_METADATA_KEYS: + value = coreml_metadata.get(col_key, None) + if value is not None: + result[col_name] = value + return result + + except ValueError: + return {} + + +def convert_coreml_delegate_time( + event_name: Union[str, int], input_time: Union[int, float] +) -> Union[int, float]: + return input_time / (1000 * 1000) + + +def create_inspector_coreml( + edge_program: EdgeProgramManager, + compile_specs: List[CompileSpec], + example_inputs: Tuple[Value, ...], + model_name: str, + root_dir_path: Path, + working_dir_path: Path, + method_name: str = "forward", + debug_buffer_size: int = 1 * 1024 * 1024 * 1024, +) -> Inspector: + edge_program_copy = copy.deepcopy(edge_program) + executorch_program = lower_and_export_edge_to_coreml( + edge_program=edge_program_copy, + compile_specs=compile_specs, + config=_EDGE_BACKEND_CONFIG, + ) + return create_inspector( + edge_program=edge_program, + executorch_program=executorch_program, + example_inputs=example_inputs, + root_dir_path=root_dir_path, + model_name=f"{model_name}_coreml", + working_dir_path=working_dir_path, + method_name=method_name, + debug_buffer_size=debug_buffer_size, + delegate_metadata_parser=parse_coreml_delegate_metadata, + delegate_time_scale_converter=convert_coreml_delegate_time, + ) + + +def create_inspector_reference( + edge_program: EdgeProgramManager, + example_inputs: Tuple[Value, ...], + model_name: str, + root_dir_path: Path, + working_dir_path: Path, + method_name: str = "forward", + debug_buffer_size: int = 1 * 1024 * 1024 * 1024, +) -> Inspector: + edge_program_copy = copy.deepcopy(edge_program) + return create_inspector( + edge_program=edge_program, + executorch_program=edge_program_copy.to_executorch(), + example_inputs=example_inputs, + root_dir_path=root_dir_path, + model_name=f"{model_name}_default", + working_dir_path=working_dir_path, + method_name=method_name, + debug_buffer_size=debug_buffer_size, + ) + + +def get_debug_handle_to_event_map( + inspector: Inspector, + event_block_name: str = "Execute", +) -> Dict[int, Event]: + result = {} + + def is_not_blank(s): + return bool(s and not s.isspace()) + + event_names_to_ignore = {"DELEGATE_CALL", "OPERATOR_CALL"} + for event_block in inspector.event_blocks: + if event_block.name == event_block_name: + for event in event_block.events: + if is_not_blank(event.name) and event.name not in event_names_to_ignore: + debug_handles = [] + if isinstance(event.debug_handles, int): + debug_handles.append(event.debug_handles) + elif isinstance(event.debug_handles, list): + debug_handles.extend(event.debug_handles) + debug_handles.sort() + for debug_handle in debug_handles: + if len(event.debug_data) > 0: + result[debug_handle] = event + return result + + +@dataclass +class EventData: + tag: str + event: Event + + +@dataclass +class ComparisonResult: + datas: List[tuple[EventData, EventData]] + + def to_dataframe( + self, + atol: float = 1e-3, + rtol: float = 1e-3, + ) -> pd.DataFrame: + def get_compute_device(event: Event) -> str: + if event.delegate_backend_name == CoreMLBackend.__name__: + return event.delegate_debug_metadatas.get( + "coreml_preferred_device", "CPU" + ) + + return "CPU" + + if len(self.datas) == 0: + return + + (data1, data2) = self.datas[0] + dict = { + data1.tag: [], + f"{data1.tag}_compute_unit": [], + data2.tag: [], + f"{data2.tag}_compute_unit": [], + "max_diff": [], + } + + for data1, data2 in self.datas: + event1 = data1.event + event2 = data2.event + debug_data1 = event1.debug_data[0] + debug_data2 = event2.debug_data[0] + + if debug_data1.size() != debug_data2.size(): + continue + + max_diff = 0.0 + indices = torch.isclose( + debug_data1, debug_data2, atol=atol, rtol=rtol + ).logical_not() + + # Find the maximum difference + if torch.count_nonzero(indices) > 0: + values1 = torch.masked_select(debug_data1, indices) + values2 = torch.masked_select(debug_data2, indices) + diff = torch.abs(values1 - values2) + max_diff = torch.max(diff).item() + + dict[f"{data1.tag}_compute_unit"].append(get_compute_device(event1)) + dict[f"{data2.tag}_compute_unit"].append(get_compute_device(event2)) + dict["max_diff"].append(max_diff) + dict[data1.tag].append(event1.name) + dict[data2.tag].append(event2.name) + + return pd.DataFrame(dict) + + +def get_comparison_result( + inspector1: Inspector, + tag1: str, + inspector2: Inspector, + tag2: str, +) -> ComparisonResult: + debug_handle_event_map_1 = get_debug_handle_to_event_map(inspector1) + debug_handle_event_map_2 = get_debug_handle_to_event_map(inspector2) + + event_datas = [] + for handle, event1 in debug_handle_event_map_1.items(): + event2 = debug_handle_event_map_2.get(handle, None) + if event2 is None: + continue + + event_data1 = EventData(tag=tag1, event=event1) + event_data2 = EventData(tag=tag2, event=event2) + event_datas.append((event_data1, event_data2)) + + return ComparisonResult(datas=event_datas) diff --git a/examples/sdk/CMakeLists.txt b/examples/sdk/CMakeLists.txt index 5345c0b537..76034b0760 100644 --- a/examples/sdk/CMakeLists.txt +++ b/examples/sdk/CMakeLists.txt @@ -9,6 +9,8 @@ cmake_minimum_required(VERSION 3.19) project(sdk_example) +option(EXECUTORCH_BUILD_COREML "Build the Core ML backend" OFF) + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) if(NOT CMAKE_CXX_STANDARD) set(CMAKE_CXX_STANDARD 17) @@ -61,3 +63,28 @@ target_link_libraries( portable_ops_lib portable_kernels ) + +if(EXECUTORCH_BUILD_COREML) +find_library(ACCELERATE_FRAMEWORK Accelerate) +find_library(COREML_FRAMEWORK CoreML) +find_library(FOUNDATION_FRAMEWORK Foundation) +find_library(SQLITE_LIBRARY sqlite3) + +set(PROTOBUF_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../backends/apple/coreml/third-party/coremltools/deps/protobuf/cmake) +find_library(PROTOBUF_LITE REQUIRED NAMES libprotobuf-lite.a PATHS ${PROTOBUF_LIB_DIR} NO_DEFAULT_PATH) + +target_link_libraries( + sdk_example_runner + "-Wl,-force_load" + coremldelegate +) + +target_link_libraries( + sdk_example_runner + ${PROTOBUF_LITE} + ${ACCELERATE_FRAMEWORK} + ${COREML_FRAMEWORK} + ${FOUNDATION_FRAMEWORK} + ${SQLITE_LIBRARY} +) +endif() diff --git a/examples/sdk/build_sdk_example_runner.sh b/examples/sdk/build_sdk_example_runner.sh index 291febb36d..be0e61cef7 100755 --- a/examples/sdk/build_sdk_example_runner.sh +++ b/examples/sdk/build_sdk_example_runner.sh @@ -17,15 +17,46 @@ readonly EXECUTORCH_ROOT="${SCRIPT_DIR}/../.." # Allow overriding the number of build jobs. Default to 9. export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-9}" +BUILD_COREML=OFF + +usage() { + echo "Builds sdk example runner." + echo "Options:" + echo " --coreml Include this flag to enable Core ML backend when building the SDK." + exit 0 +} + +for arg in "$@"; do + case $arg in + -h|--help) usage ;; + --coreml) BUILD_COREML=ON ;; + *) + esac +done + main() { cd "${EXECUTORCH_ROOT}" rm -rf cmake-out - cmake -DCMAKE_INSTALL_PREFIX=cmake-out \ + + if [[ "${BUILD_COREML}" == "ON" ]]; then + cmake -DCMAKE_INSTALL_PREFIX=cmake-out \ + -DCMAKE_BUILD_TYPE=Release \ + -DEXECUTORCH_BUILD_SDK=ON \ + -DEXECUTORCH_ENABLE_EVENT_TRACER=ON \ + -DEXECUTORCH_BUILD_COREML=ON \ + -Dprotobuf_BUILD_TESTS=OFF \ + -Dprotobuf_BUILD_EXAMPLES=OFF \ + -DEXECUTORCH_ENABLE_LOGGING=ON \ + -Bcmake-out . + else + cmake -DCMAKE_INSTALL_PREFIX=cmake-out \ -DCMAKE_BUILD_TYPE=Release \ -DEXECUTORCH_BUILD_SDK=ON \ -DEXECUTORCH_ENABLE_EVENT_TRACER=ON \ -Bcmake-out . + fi + cmake --build cmake-out --target install --config Release local example_dir=examples/sdk @@ -34,6 +65,7 @@ main() { rm -rf ${build_dir} cmake -DCMAKE_PREFIX_PATH="${cmake_prefix_path}" \ -DCMAKE_BUILD_TYPE=Release \ + -DEXECUTORCH_BUILD_COREML=$BUILD_COREML \ -B"${build_dir}" \ "${example_dir}" cmake --build "${build_dir}" --config Release