Update

[ghstack-poisoned]

Author: swolchok

.ci/scripts/gather_test_models.py

@@ -90,7 +90,7 @@ def model_should_run_on_event(model: str, event: str) -> bool:
     We put higher priority and fast models to pull request and rest to push.
     """
     if event == "pull_request":
-        return model in ["mv3", "vit"]
+        return model in ["mv3", "vit", "qwen2_5"]  # TODO: remove, just to test the ci
     elif event == "push":
         # These are super slow. Only run it periodically
         return model not in ["dl3", "edsr", "emformer_predict"]

.ci/scripts/test_model.sh

@@ -91,7 +91,17 @@ test_model() {
     # Install requirements for llama vision.
     bash examples/models/llama3_2_vision/install_requirements.sh
   fi
-  # python3 -m examples.portable.scripts.export --model_name="llama2" should works too
+  if [[ "${MODEL_NAME}" == "qwen2_5" ]]; then
+      # Install requirements for export_llama
+      bash examples/models/llama/install_requirements.sh
+      # Test export_llama script: python3 -m examples.models.llama.export_llama.
+      # Use Llama random checkpoint with Qwen 2.5 1.5b model configuration.
+      "${PYTHON_EXECUTABLE}" -m examples.models.llama.export_llama --model "${MODEL_NAME}" -c examples/models/llama/params/demo_rand_params.pth -p examples/models/qwen2_5/1_5b_config.json
+      rm "./${MODEL_NAME}.pte"
+      return  # Skip running with portable executor runnner since portable doesn't support Qwen's biased linears.
+  fi
+
+  # Export a basic .pte and run the model.
   "${PYTHON_EXECUTABLE}" -m examples.portable.scripts.export --model_name="${MODEL_NAME}" "${STRICT}"
   run_portable_executor_runner
 }

CODEOWNERS

@@ -32,7 +32,7 @@
 /examples/llm_manual @larryliu0820
 /examples/llm_pte_finetuning @JacobSzwejbka
 /examples/mediatek @cccclai
-/examples/models @lucylq
+/examples/models @lucylq @jackzhxng
 /examples/portable @larryliu0820 @manuelcandales
 /examples/qualcomm @cccclai
 /examples/selective_build @lucylq @larryliu0820 @JacobSzwejbka

backends/arm/operator_support/__init__.py

@@ -6,7 +6,6 @@
 # pyre-unsafe
 
 from . import (  # noqa
-    bitwise_support,
     convolution_support,
     pool_2d_support,
     reduce_sum_support,

backends/arm/operator_support/bitwise_support.py

@@ -11,13 +11,13 @@
 from typing import final, Optional, Sequence, Type
 
 import torch
-
 import torch.fx as fx
+
 from executorch.backends.arm._passes.arm_pass_utils import get_first_fake_tensor
 from executorch.backends.arm._passes.fuse_quantized_activation_pass import (
     FuseQuantizedActivationPass,
 )
-from executorch.backends.arm.tosa_specification import TosaSpecification
+from executorch.backends.arm.tosa_specification import Tosa_0_80, TosaSpecification
 from executorch.exir.dialects._ops import ops as exir_ops
 from torch.fx.passes.operator_support import any_chain, chain, OperatorSupportBase
 from torch.fx.passes.utils.source_matcher_utils import get_source_partitions
@@ -90,6 +90,7 @@ def tosa_support_factory(
     if not tosa_spec.support_float():
         negative_checks.append(NeedsDecompositionCheck())
         negative_checks.append(CheckProperQuantization())
+        negative_checks.append(EthosU55NotSupported(tosa_spec))
     return chain(
         any_chain(
             BaseTOSASupportList(),
@@ -111,6 +112,9 @@ def is_node_supported(
         supported = node.op == "call_function" and node.target in [
             exir_ops.edge.aten.abs.default,
             exir_ops.edge.aten.add.Tensor,
+            exir_ops.edge.aten.bitwise_and.Tensor,
+            exir_ops.edge.aten.bitwise_or.Tensor,
+            exir_ops.edge.aten.bitwise_xor.Tensor,
             exir_ops.edge.aten.expand_copy.default,
             exir_ops.edge.aten.cat.default,
             exir_ops.edge.aten.clamp.default,
@@ -170,6 +174,31 @@ def is_node_supported(
         return supported
 
 
+class EthosU55NotSupported(OperatorSupportBase):
+    """
+    Certain operators are not supported on U55. These are listed in `unsupported` in
+    is_node_supported().
+    """
+
+    def __init__(self, tosa_spec: TosaSpecification):
+        self.tosa_spec = tosa_spec
+
+    def is_node_supported(
+        self, submodules: typing.Mapping[str, torch.nn.Module], node: fx.Node
+    ) -> bool:
+        if isinstance(self.tosa_spec, Tosa_0_80) and self.tosa_spec.is_U55_subset:
+            unsupported_ops = [
+                exir_ops.edge.aten.bitwise_and.Tensor,
+                exir_ops.edge.aten.bitwise_or.Tensor,
+                exir_ops.edge.aten.bitwise_xor.Tensor,
+            ]
+
+            if node.target in unsupported_ops:
+                return False
+
+        return True
+
+
 class NeedsDecompositionCheck(OperatorSupportBase):
     """
     Targeted operators need to be decomposed prior to quantization in order to get a pair of q-dq-nodes surrounding
@@ -310,11 +339,11 @@ def is_node_supported(
         if not input_quantized:
             return False
 
-        output_quantized = output_quantized or all(
-            (output_node.target == self.q_op)
-            or (not get_first_fake_tensor(output_node).dtype.is_floating_point)
-            for output_node in node.users
+        all_q_users = all(
+            (output_node.target == self.q_op) for output_node in node.users
         )
+        is_floating_point = get_first_fake_tensor(node).dtype.is_floating_point
+        output_quantized = output_quantized or all_q_users or not is_floating_point
 
         if not output_quantized:
             return False

backends/arm/operator_support/tosa_supported_operators.py

@@ -16,18 +16,17 @@ et_root_dir=$(realpath ${et_root_dir})
 toolchain_cmake=${script_dir}/../../../examples/arm/ethos-u-setup/arm-none-eabi-gcc.cmake
 toolchain_cmake=$(realpath ${toolchain_cmake})
 
-
-
 et_build_root="${et_root_dir}/arm_test"
 build_type="Release"
+build_devtools=false
 build_with_etdump=false
 
-
 help() {
     echo "Usage: $(basename $0) [options]"
     echo "Options:"
     echo "  --et_build_root=<FOLDER>  Build output root folder to use, defaults to ${et_build_root}"
     echo "  --build_type=<TYPE>       Build with Release, Debug or RelWithDebInfo, default is ${build_type}"
+    echo "  --devtools                Build Devtools libs"
     echo "  --etdump                  Adds Devtools etdump support to track timing, etdump area will be base64 encoded in the log"
     exit 0
 }
@@ -37,32 +36,33 @@ for arg in "$@"; do
       -h|--help) help ;;
       --et_build_root=*) et_build_root="${arg#*=}";;
       --build_type=*) build_type="${arg#*=}";;
+      --devtools) build_devtools=true ;;
       --etdump) build_with_etdump=true ;;
       *)
       ;;
     esac
 done
 
 et_build_dir="${et_build_root}/cmake-out"
+
+# Used for flatcc host excutable if Devtools is used
 et_build_host_dir=${et_build_root}/cmake-out-host-tools
 
 set -x
 cd "${et_root_dir}"
 
-build_with_etdump_flags=""
 if [ "$build_with_etdump" = true ] ; then
     ( set +x ;
         echo "--------------------------------------------------------------------------------" ;
-        echo "Build ExecuTorch Libraries host flatcc bin ${build_type} into ${et_build_host_dir} - ${et_build_host_dir}/bin/flatcc" ;
+        echo "Build ExecuTorch Libraries host flatcc bin ${build_type} into ${et_build_host_dir}/bin/flatcc" ;
         echo "--------------------------------------------------------------------------------" )
 
-
     # Build host flatcc bin
     # This is a way to work around that the flatcc executable get build for target (e.g. Arm) later
     # and get replaced. flatcc is a tool used on the host for etdump and BundleIO handling.
     # The way to solve this is to generate it once for the host, then copy it to ${et_build_host_dir}/bin
     # and later point that out with -DFLATCC_EXECUTABLE=${et_build_host_dir}/bin/flatcc later.
-    mkdir -p ${et_build_host_dir}
+
     cmake                                                 \
         -DCMAKE_INSTALL_PREFIX=${et_build_host_dir}       \
         -DCMAKE_BUILD_TYPE=${build_type}                  \
@@ -79,25 +79,39 @@ if [ "$build_with_etdump" = true ] ; then
         -B"${et_build_host_dir}"                          \
         "${et_root_dir}"
 
-    # Copy host flatcc excutable to it's saved when we build for target (Arm) later
+    # third-party/flatcc/bin/flatcc gets build already in the in the cmake config step above
+    # so there is no cmake building step done
+
+    # Copy host flatcc excutable so it's saved when we build for target (Arm) later
+    et_build_host_dir=$(realpath ${et_build_host_dir})
     mkdir -p ${et_build_host_dir}/bin
     cp third-party/flatcc/bin/flatcc ${et_build_host_dir}/bin
-
-    # Add DevTools flags use in the Target build below
-    build_with_etdump_flags="-DEXECUTORCH_BUILD_DEVTOOLS=ON                    \
-                                -DEXECUTORCH_ENABLE_EVENT_TRACER=ON               \
-                                -DEXECUTORCH_SEPARATE_FLATCC_HOST_PROJECT=OFF     \
-                                -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=OFF      \
-                                -DFLATCC_ALLOW_WERROR=OFF                         \
-                                -DFLATCC_EXECUTABLE=${et_build_host_dir}/bin/flatcc "
-    echo "build_with_etdump_flags=$build_with_etdump_flags"
 fi
 
 ( set +x ;
     echo "--------------------------------------------------------------------------------" ;
     echo "Build ExecuTorch target libs ${build_type} into '${et_build_dir}'" ;
     echo "--------------------------------------------------------------------------------" )
 
+build_devtools_flags=" -DEXECUTORCH_BUILD_DEVTOOLS=OFF "
+if [ "$build_devtools" = true ] ; then
+    build_devtools_flags=" -DEXECUTORCH_BUILD_DEVTOOLS=ON "
+fi
+
+build_with_etdump_flags=" -DEXECUTORCH_ENABLE_EVENT_TRACER=OFF "
+if [ "$build_with_etdump" = true ] ; then
+    # Add DevTools flags use in the Target build below
+    build_with_etdump_flags="-DEXECUTORCH_BUILD_DEVTOOLS=ON                    \
+                            -DEXECUTORCH_ENABLE_EVENT_TRACER=ON               \
+                            -DEXECUTORCH_SEPARATE_FLATCC_HOST_PROJECT=OFF     \
+                            -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=OFF      \
+                            -DFLATCC_ALLOW_WERROR=OFF                         \
+                            -DFLATCC_EXECUTABLE=${et_build_host_dir}/bin/flatcc "
+fi
+
+echo "Building with Devtools: ${build_devtools_flags} ${build_with_etdump_flags}"
+
+
 # Build
 cmake                                                 \
     -DCMAKE_INSTALL_PREFIX=${et_build_dir}            \
@@ -108,6 +122,7 @@ cmake                                                 \
     -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON           \
     -DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON       \
     -DEXECUTORCH_ENABLE_LOGGING=ON                    \
+    ${build_devtools_flags}                           \
     ${build_with_etdump_flags}                        \
     -DFLATC_EXECUTABLE="$(which flatc)"               \
     -B"${et_build_dir}"                               \

backends/arm/scripts/build_executorch.sh

@@ -15,13 +15,17 @@ pte_file=""
 target="ethos-u55-128"
 build_type="Release"
 system_config=""
+bundleio=false
 build_with_etdump=false
 extra_build_flags=""
 output_folder_set=false
 output_folder="."
 et_build_root="${et_root_dir}/arm_test"
 ethosu_tools_dir=${et_root_dir}/examples/arm/ethos-u-scratch
 
+build_bundleio_flags=" -DET_BUNDLE_IO=OFF "
+build_with_etdump_flags=" -DEXECUTORCH_ENABLE_EVENT_TRACER=OFF "
+
 help() {
     echo "Usage: $(basename $0) [options]"
     echo "Options:"
@@ -30,6 +34,7 @@ help() {
     echo "  --build_type=<TYPE>             Build with Release, Debug or RelWithDebInfo, default is ${build_type}"
     echo "  --system_config=<CONFIG>        System configuration to select from the Vela configuration file (see vela.ini). Default: Ethos_U55_High_End_Embedded for EthosU55 targets, Ethos_U85_SYS_DRAM_Mid for EthosU85 targets."
     echo "                                     NOTE: If given, this option must match the given target. This option also sets timing adapter values customized for specific hardware, see ./executor_runner/CMakeLists.txt."
+    echo "  --bundleio                      Support both pte and Bundle IO bpte using Devtools BundelIO with Input/RefOutput included"
     echo "  --etdump                        Adds Devtools etdump support to track timing, etdump area will be base64 encoded in the log"
     echo "  --extra_build_flags=<FLAGS>     Extra flags to pass to cmake like -DET_ARM_BAREMETAL_METHOD_ALLOCATOR_POOL_SIZE=60000 Default: none "
     echo "  --output=<FOLDER>               Output folder Default: <MODEL>/<MODEL>_<TARGET INFO>.pte"
@@ -45,6 +50,7 @@ for arg in "$@"; do
       --target=*) target="${arg#*=}";;
       --build_type=*) build_type="${arg#*=}";;
       --system_config=*) system_config="${arg#*=}";;
+      --bundleio) bundleio=true ;;
       --etdump) build_with_etdump=true ;;
       --extra_build_flags=*) extra_build_flags="${arg#*=}";;
       --output=*) output_folder="${arg#*=}" ; output_folder_set=true ;;
@@ -64,9 +70,8 @@ et_build_dir=${et_build_root}/cmake-out
 et_build_dir=$(realpath ${et_build_dir})
 
 if [ "$output_folder_set" = false ] ; then
-    pte_folder=$(cd -- "$( dirname -- "${pte_file}" )" &> /dev/null && pwd)
-    pte_short_name=$(basename -- "${pte_file}" ".pte")
-    output_folder="$pte_folder/$pte_short_name"
+    # remove file ending
+    output_folder=${pte_file%.*}
 fi
 
 if [[ ${system_config} == "" ]]
@@ -86,28 +91,32 @@ else
     target_cpu=cortex-m85
 fi
 echo "--------------------------------------------------------------------------------"
-echo "Build Arm Baremetal executor_runner for ${target} with ${pte_file} using ${system_config} to '${output_folder}/cmake-out'"
+echo "Build Arm Baremetal executor_runner for ${target} with ${pte_file} using ${system_config} ${extra_build_flags} to '${output_folder}/cmake-out'"
 echo "--------------------------------------------------------------------------------"
 
 cd ${et_root_dir}/examples/arm/executor_runner
 
-build_with_etdump_flags=""
+if [ "$bundleio" = true ] ; then
+    build_bundleio_flags=" -DET_BUNDLE_IO=ON "
+fi
+
 if [ "$build_with_etdump" = true ] ; then
-    echo "Building with etdump e.g. -DEXECUTORCH_ENABLE_EVENT_TRACER=ON"
     build_with_etdump_flags=" -DEXECUTORCH_ENABLE_EVENT_TRACER=ON "
 fi
 
-mkdir -p "$output_folder"
+echo "Building with BundleIO/etdump/extra flags: ${build_bundleio_flags} ${build_with_etdump_flags} ${extra_build_flags}"
+mkdir -p "${output_folder}"
 
 cmake \
     -DCMAKE_BUILD_TYPE=${build_type}            \
     -DCMAKE_TOOLCHAIN_FILE=${toolchain_cmake}   \
     -DTARGET_CPU=${target_cpu}                  \
     -DET_DIR_PATH:PATH=${et_root_dir}           \
     -DET_BUILD_DIR_PATH:PATH=${et_build_dir}    \
-    -DET_PTE_FILE_PATH:PATH="${pte_file}"            \
+    -DET_PTE_FILE_PATH:PATH="${pte_file}"       \
     -DETHOS_SDK_PATH:PATH=${ethos_u_root_dir}   \
     -DETHOSU_TARGET_NPU_CONFIG=${target}        \
+    ${build_bundleio_flags}                     \
     ${build_with_etdump_flags}                  \
     -DPYTHON_EXECUTABLE=$(which python3)        \
     -DSYSTEM_CONFIG=${system_config}            \