Arm backend: Add pypi package install option for VGF backend (#15551)

mansnils · perheld · ArmRyan · web-flow · commit 0bd635eeb563 · 2025-11-07T08:01:34.000+01:00
New setup option is added: --install-mlsdk-deps-with-pip For Linux/Windows x86 machines, PyPi packages of MLSDK repository for VGF backend may be used. This will eventually be the default. Reason it is not yet default is because of a limitation of model-converter to handle large models. The new option will decrease setup time, which can enable VGF backend testing in github. Co-authored-by: Per Held <[per.held@arm.com](mailto:per.held@arm.com)>, Ryan O'Shea <[ryan.oshea3@arm.com](mailto:ryan.oshea3@arm.com)> cc @freddan80 @per @zingo @oscarandersson8218 @digantdesai Signed-off-by: Måns Nilsson <mans.nilsson@arm.com> Co-authored-by: Per Held <per.held@arm.com> Co-authored-by: Per Held <per.held@arm.com>, Ryan O'Shea <ryan.oshea3@arm.com>
diff --git a/backends/arm/CMakeLists.txt b/backends/arm/CMakeLists.txt
@@ -48,17 +48,44 @@ endif()
 
 # VGF backend builds
 if(EXECUTORCH_BUILD_VGF)
-
-  # include libvgf
-  set(LIBVGF_PATH
-      "${EXECUTORCH_ROOT}/examples/arm/ethos-u-scratch/ml-sdk-for-vulkan-manifest/sw/vgf-lib/"
-  )
-
   set(VULKAN_THIRD_PARTY_PATH ${EXECUTORCH_ROOT}/backends/vulkan/third-party)
   set(VULKAN_HEADERS_PATH ${VULKAN_THIRD_PARTY_PATH}/Vulkan-Headers/include)
   set(VOLK_HEADERS_PATH ${VULKAN_THIRD_PARTY_PATH}/volk)
 
-  set(LIBVGF_STATIC "${LIBVGF_PATH}/build/src/libvgf.a")
+  if(APPLE
+     OR CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)$"
+     OR EXISTS
+        "${EXECUTORCH_ROOT}/examples/arm/ethos-u-scratch/ml-sdk-for-vulkan-manifest/"
+  )
+    message(STATUS "libvgf sourced from local scratch tree")
+
+    # Legacy layout: libvgf sourced from local scratch tree
+    set(LIBVGF_PATH
+        "${EXECUTORCH_ROOT}/examples/arm/ethos-u-scratch/ml-sdk-for-vulkan-manifest/sw/vgf-lib/"
+    )
+    set(LIBVGF_STATIC "${LIBVGF_PATH}/build/src/libvgf.a")
+  else()
+    message(STATUS "libvgf installed from pip package")
+
+    set(Python3_FIND_VIRTUALENV FIRST)
+    if(EXECUTORCH_ROOT AND EXISTS "${EXECUTORCH_ROOT}/env")
+      set(Python3_EXECUTABLE "${EXECUTORCH_ROOT}/env/bin/python3")
+    endif()
+
+    find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
+    # Prefer arch-specific site-packages if present, else pure
+    set(_vgf_site_arch "${Python3_SITEARCH}/vgf_lib/binaries")
+    set(_vgf_site_pure "${Python3_SITELIB}/vgf_lib/binaries")
+    if(EXISTS "${_vgf_site_arch}")
+      set(LIBVGF_PATH "${_vgf_site_arch}")
+    else()
+      set(LIBVGF_PATH "${_vgf_site_pure}")
+    endif()
+
+    set(LIBVGF_STATIC "${LIBVGF_PATH}/lib/libvgf.a")
+  endif()
+
   set(LIBVGF_INCLUDE "${LIBVGF_PATH}/include/")
 
   add_library(vgf STATIC IMPORTED)
diff --git a/backends/arm/requirements-arm-vgf.txt b/backends/arm/requirements-arm-vgf.txt
@@ -0,0 +1,8 @@
+# Copyright 2025 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+ai_ml_emulation_layer_for_vulkan == 0.7.0
+ai_ml_sdk_model_converter == 0.7.0
+ai_ml_sdk_vgf_library == 0.7.0
diff --git a/backends/arm/scripts/mlsdk_utils.sh b/backends/arm/scripts/mlsdk_utils.sh
@@ -205,7 +205,51 @@ function setup_path_emulation_layer() {
     model_emulation_layer_path="$(cd "${mlsdk_manifest_dir}/sw/emulation-layer/" && pwd)"
     prepend_env_in_setup_path LD_LIBRARY_PATH "${model_emulation_layer_path}/deploy/lib"
     prepend_env_in_setup_path DYLD_LIBRARY_PATH "${model_emulation_layer_path}/deploy/lib"
+    prepend_env_in_setup_path VK_LAYER_PATH "${model_emulation_layer_path}/deploy/share/vulkan/explicit_layer.d"
     prepend_env_in_setup_path VK_INSTANCE_LAYERS VK_LAYER_ML_Tensor_Emulation
     prepend_env_in_setup_path VK_INSTANCE_LAYERS VK_LAYER_ML_Graph_Emulation
-    prepend_env_in_setup_path VK_LAYER_PATH "${model_emulation_layer_path}/deploy/share/vulkan/explicit_layer.d"
+}
+
+function setup_path_emulation_layer_from_pip() {
+    if ! command -v emulation_layer >/dev/null 2>&1; then
+        echo "[mlsdk_utils] 'emulation_layer' command not found; skipping pip emulation layer path setup"
+        return
+    fi
+
+    local output
+    if ! output=$(emulation_layer 2>/dev/null); then
+        echo "[mlsdk_utils] Failed to query emulation_layer environment; skipping"
+        return
+    fi
+
+    local exports
+    exports=$(echo "$output" | grep '^export ' || true)
+
+    local ld_line
+    ld_line=$(echo "$exports" | grep 'LD_LIBRARY_PATH=' || true)
+    if [[ -n "${ld_line}" ]]; then
+        local ld_value=${ld_line#export LD_LIBRARY_PATH=}
+        ld_value=${ld_value%%:\$LD_LIBRARY_PATH*}
+        if [[ -n "${ld_value}" ]]; then
+            prepend_env_in_setup_path LD_LIBRARY_PATH "${ld_value}"
+        fi
+    fi
+
+    local vk_add_line
+    vk_add_line=$(echo "$exports" | grep 'VK_ADD_LAYER_PATH=' || true)
+    if [[ -n "${vk_add_line}" ]]; then
+        local vk_add_value=${vk_add_line#export VK_ADD_LAYER_PATH=}
+        if [[ -n "${vk_add_value}" ]]; then
+            prepend_env_in_setup_path VK_ADD_LAYER_PATH "${vk_add_value}"
+        fi
+    fi
+
+    local vk_instance_line
+    vk_instance_line=$(echo "$exports" | grep 'VK_INSTANCE_LAYERS=' || true)
+    if [[ -n "${vk_instance_line}" ]]; then
+        local vk_instance_value=${vk_instance_line#export VK_INSTANCE_LAYERS=}
+        if [[ -n "${vk_instance_value}" ]]; then
+            prepend_env_in_setup_path VK_INSTANCE_LAYERS "${vk_instance_value}"
+        fi
+    fi
 }
diff --git a/backends/arm/scripts/run_vkml.sh b/backends/arm/scripts/run_vkml.sh
@@ -50,9 +50,14 @@ if [[ -z ${model} ]]; then echo "Model name needs to be provided"; exit 1; fi
 
 source ${setup_path_script}
 
-# basic checks before we get started
-hash ${converter} \
-    || { echo "Could not find ${converter} on PATH, ${_setup_msg}"; exit 1; }
+if ! command -v "${converter}" >/dev/null 2>&1; then
+    if command -v model_converter >/dev/null 2>&1; then
+        converter="model_converter"
+    fi
+fi
+
+command -v "${converter}" >/dev/null 2>&1 \
+    || { echo "Could not find a model converter executable (tried model-converter, model_converter). ${_setup_msg}"; exit 1; }
 
 
 
diff --git a/backends/arm/test/runner_utils.py b/backends/arm/test/runner_utils.py
@@ -31,6 +31,7 @@
 from executorch.backends.arm.tosa.compile_spec import TosaCompileSpec
 from executorch.backends.arm.tosa.specification import Tosa_1_00, TosaSpecification
 from executorch.backends.arm.vgf import VgfCompileSpec
+from executorch.backends.arm.vgf.model_converter import find_model_converter_binary
 from executorch.exir import ExecutorchProgramManager, ExportedProgram
 from executorch.exir.lowered_backend_module import LoweredBackendModule
 from torch.fx.node import Node
@@ -678,11 +679,15 @@ def corstone320_installed() -> bool:
 
 
 def model_converter_installed() -> bool:
-    cmd = ["model-converter", "--version"]
+    model_converter = find_model_converter_binary()
+    if model_converter is None:
+        return False
+
     try:
-        _run_cmd(cmd, check=True)
-    except:
+        _run_cmd([model_converter, "--version"], check=True)
+    except Exception:
         return False
+
     return True
 
 
diff --git a/backends/arm/vgf/backend.py b/backends/arm/vgf/backend.py
@@ -17,13 +17,24 @@
 import tempfile
 from typing import final, List
 
-from executorch.backends.arm.tosa.backend import (
+from executorch.backends.arm.tosa.backend import (  # type: ignore[import-not-found]
     arm_get_first_delegation_tag,
     TOSABackend,
 )
-from executorch.backends.arm.vgf.compile_spec import VgfCompileSpec
-from executorch.exir.backend.backend_details import BackendDetails, PreprocessResult
-from executorch.exir.backend.compile_spec_schema import CompileSpec
+
+from executorch.backends.arm.vgf.compile_spec import (  # type: ignore[import-not-found]
+    VgfCompileSpec,
+)
+from executorch.backends.arm.vgf.model_converter import (  # type: ignore[import-not-found]
+    require_model_converter_binary,
+)
+from executorch.exir.backend.backend_details import (  # type: ignore[import-not-found]
+    BackendDetails,
+    PreprocessResult,
+)
+from executorch.exir.backend.compile_spec_schema import (  # type: ignore[import-not-found]
+    CompileSpec,
+)
 from torch.export.exported_program import ExportedProgram
 
 # debug functionality
@@ -96,9 +107,10 @@ def vgf_compile(
             f.write(tosa_flatbuffer)
 
         additional_flags = " ".join(compile_flags)
+        converter_binary = require_model_converter_binary()
         vgf_path = tosa_path + ".vgf"
         conversion_command = (
-            f"model-converter {additional_flags} -i {tosa_path} -o {vgf_path}"
+            f"{converter_binary} {additional_flags} -i {tosa_path} -o {vgf_path}"
         )
         try:
             subprocess.run(
diff --git a/backends/arm/vgf/model_converter.py b/backends/arm/vgf/model_converter.py
@@ -0,0 +1,34 @@
+# Copyright 2025 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import annotations
+
+from shutil import which
+from typing import Optional
+
+MODEL_CONVERTER_BINARY = "model-converter"
+_MODEL_CONVERTER_FALLBACK_BINARY = "model_converter"
+
+
+def find_model_converter_binary() -> Optional[str]:
+    """Return the name of the first model converter executable on PATH."""
+
+    for candidate in (MODEL_CONVERTER_BINARY, _MODEL_CONVERTER_FALLBACK_BINARY):
+        if which(candidate):
+            return candidate
+    return None
+
+
+def require_model_converter_binary() -> str:
+    """Return a usable model converter executable or raise a helpful error."""
+
+    binary = find_model_converter_binary()
+    if binary is None:
+        tried = ", ".join((MODEL_CONVERTER_BINARY, _MODEL_CONVERTER_FALLBACK_BINARY))
+        raise RuntimeError(
+            "Unable to locate a model converter executable. "
+            f"Tried: {tried}. Ensure the Model Converter is installed and on PATH."
+        )
+    return binary
diff --git a/examples/arm/setup.sh b/examples/arm/setup.sh
@@ -26,6 +26,7 @@ enable_model_converter=0   # model-converter tool for VGF output
 enable_vgf_lib=0  # vgf reader - runtime backend dependency
 enable_emulation_layer=0  # Vulkan layer driver - emulates Vulkan ML extensions
 enable_vulkan_sdk=0  # Download and export Vulkan SDK required by emulation layer
+enable_mlsdk_pip_install=0  # This is a temporary option that will soon be the default
 
 # Figure out if setup.sh was called or sourced and save it into "is_script_sourced"
 (return 0 2>/dev/null) && is_script_sourced=1 || is_script_sourced=0
@@ -51,6 +52,7 @@ OPTION_LIST=(
   "--enable-emulation-layer Enable MLSDK Vulkan emulation layer"
   "--disable-ethos-u-deps Do not setup what is needed for Ethos-U"
   "--enable-mlsdk-deps Setup what is needed for MLSDK"
+  "--install-mlsdk-deps-with-pip Use MLSDK PyPi package instead of building from source"
   "--mlsdk-manifest-url URL to the MLSDK manifest for vulkan."
   "--help Display help"
 )
@@ -140,6 +142,10 @@ function check_options() {
                 enable_vela=0
                 shift
                 ;;
+            --install-mlsdk-deps-with-pip)
+                enable_mlsdk_pip_install=1
+                shift
+                ;;
             --enable-mlsdk-deps)
                 enable_model_converter=1
                 enable_vgf_lib=1
@@ -176,12 +182,22 @@ function setup_ethos_u_tools() {
     CMAKE_POLICY_VERSION_MINIMUM=3.5 BUILD_PYBIND=1 pip install --no-dependencies -r $et_dir/backends/arm/requirements-arm-ethos-u.txt
 }
 
+function setup_mlsdk_dependencies() {
+    log_step "mlsdk" "Installing MLSDK dependencies from pip"
+    pip install -r $et_dir/backends/arm/requirements-arm-vgf.txt
+}
+
 function create_setup_path(){
     cd "${root_dir}"
 
     clear_setup_path
     log_step "path" "Generating setup path scripts at ${setup_path_script}"
 
+    local use_mlsdk_pip=0
+    if use_mlsdk_pip_package; then
+        use_mlsdk_pip=1
+    fi
+
     if [[ "${enable_fvps}" -eq 1 ]]; then
         setup_path_fvp
     fi
@@ -194,19 +210,48 @@ function create_setup_path(){
         setup_path_vulkan
     fi
 
-    if [[ "${enable_model_converter}" -eq 1 ]]; then
+    if [[ "${enable_model_converter}" -eq 1 && "${use_mlsdk_pip}" -eq 0 ]]; then
         setup_path_model_converter
     fi
 
-    if [[ "${enable_vgf_lib}" -eq 1 ]]; then
+    if [[ "${enable_vgf_lib}" -eq 1 && "${use_mlsdk_pip}" -eq 0 ]]; then
         setup_path_vgf_lib
     fi
 
     if [[ "${enable_emulation_layer}" -eq 1 ]]; then
-        setup_path_emulation_layer
+        if [[ "${use_mlsdk_pip}" -eq 0 ]]; then
+            setup_path_emulation_layer
+        else
+            setup_path_emulation_layer_from_pip
+        fi
+    fi
+
+   log_step "path" "Update PATH by sourcing ${setup_path_script}.{sh|fish}"
+}
+
+function use_mlsdk_pip_package() {
+    os=$(uname -s)
+    arch=$(uname -m)
+
+    if [[ "${enable_mlsdk_pip_install}" -eq 0 ]]; then
+        return 1
+    fi
+
+    if [[ "$os" == "Darwin" ]]; then
+        if [[ "${enable_mlsdk_pip_install}" -eq 1 ]]; then
+            log_step "mlsdk" "[error] MLSDK pip install not yet supported on MacOS"
+            exit 1
+        fi
+    fi
+
+    if [[ "$arch" == "arm64" || "$arch" == "aarch64" ]]; then
+        if [[ "${enable_mlsdk_pip_install}" -eq 1 ]]; then
+            log_step "mlsdk" "[error] MLSDK pip install not yet supported on aarch64"
+            exit 1
+        fi
     fi
 
-    log_step "path" "Update PATH by sourcing ${setup_path_script}.{sh|fish}"
+    return 0
 }
 
 
@@ -224,6 +269,7 @@ if [[ $is_script_sourced -eq 0 ]]; then
     source $et_dir/backends/arm/scripts/fvp_utils.sh
     source $et_dir/backends/arm/scripts/toolchain_utils.sh
     source $et_dir/backends/arm/scripts/vulkan_utils.sh
+    source $et_dir/backends/arm/scripts/mlsdk_utils.sh
 
     log_step "main" "Checking platform and OS"
     check_platform_support
@@ -239,8 +285,12 @@ if [[ $is_script_sourced -eq 0 ]]; then
         mlsdk_manifest_dir="${root_dir}/${mlsdk_manifest_dir}"
     fi
 
-    log_step "options" "root=${root_dir}, target-toolchain=${target_toolchain:-<default>}, mlsdk-dir=${mlsdk_manifest_dir}"
-    log_step "options" "ethos-u: fvps=${enable_fvps}, toolchain=${enable_baremetal_toolchain}, vela=${enable_vela} | mlsdk: model-converter=${enable_model_converter}, vgf-lib=${enable_vgf_lib}, emu-layer=${enable_emulation_layer}, vulkan-sdk=${enable_vulkan_sdk}"
+    log_step "options" \
+             "root=${root_dir}, target-toolchain=${target_toolchain:-<default>}, mlsdk-dir=${mlsdk_manifest_dir}"
+    log_step "options" \
+             "ethos-u: fvps=${enable_fvps}, toolchain=${enable_baremetal_toolchain}, vela=${enable_vela} | " \
+             "mlsdk: model-converter=${enable_model_converter}, vgf-lib=${enable_vgf_lib}, " \
+                    "emu-layer=${enable_emulation_layer}, vulkan-sdk=${enable_vulkan_sdk}"
 
     # Setup toolchain
     if [[ "${enable_baremetal_toolchain}" -eq 1 ]]; then
@@ -267,13 +317,18 @@ if [[ $is_script_sourced -eq 0 ]]; then
     if [[ "${enable_model_converter}" -eq 1 || \
           "${enable_vgf_lib}" -eq 1 || \
           "${enable_emulation_layer}" -eq 1 ]]; then
-        log_step "mlsdk" "Configuring MLSDK components (model-converter=${enable_model_converter}, vgf-lib=${enable_vgf_lib}, emu-layer=${enable_emulation_layer})"
-        source $et_dir/backends/arm/scripts/mlsdk_utils.sh
-        setup_mlsdk "${root_dir}" \
-                    "${mlsdk_manifest_dir}" \
-                    "${enable_model_converter}" \
-                    "${enable_vgf_lib}" \
-                    "${enable_emulation_layer}"
+        log_step "mlsdk" "Configuring MLSDK components (model-converter=${enable_model_converter}, " \
+                         "vgf-lib=${enable_vgf_lib}, emu-layer=${enable_emulation_layer})"
+        if use_mlsdk_pip_package; then
+            setup_mlsdk_dependencies
+        else
+            log_step "mlsdk" "Installing MLSDK dependencies from source"
+            setup_mlsdk ${root_dir} \
+                        ${mlsdk_manifest_dir} \
+                        ${enable_model_converter} \
+                        ${enable_vgf_lib} \
+                        ${enable_emulation_layer}
+        fi
     fi
 
     # Create the setup_path.sh used to create the PATH variable for shell
