Openvino backend for Executorch to enable inference on Intel CPUs, GPUs, NPUs

.ci/scripts/setup-openvino.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+# shellcheck source=/dev/null
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+git clone https://github.com/openvinotoolkit/openvino.git
+cd openvino && git checkout releases/2025/1
+git submodule update --init --recursive
+sudo ./install_build_dependencies.sh
+mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON=ON
+make -j$(nproc)
+
+cd ..
+cmake --install build --prefix dist
+
+source dist/setupvars.sh
+cd ../backends/openvino
+pip install -r requirements.txt
+cd scripts
+./openvino_build.sh --enable_python

.ci/scripts/test_openvino.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+# shellcheck source=/dev/null
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+source openvino/dist/setupvars.sh
+cd backends/openvino/tests
+python test_runner.py --test_type ops
+python test_runner.py --test_type models

.github/workflows/pull.yml

@@ -736,3 +736,25 @@ jobs:
         conda activate "${CONDA_ENV}"
 
         # placeholder for mediatek to add more tests
+
+  test-openvino-linux:
+    name: test-openvino-linux
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      fail-fast: false
+    with:
+      runner: linux.2xlarge
+      docker-image: executorch-ubuntu-22.04-gcc9
+      submodules: 'true'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 90
+      script: |
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-openvino.sh
+        PYTHON_EXECUTABLE=python bash .ci/scripts/test_openvino.sh

.lintrunner.toml

@@ -299,12 +299,14 @@ include_patterns = [
     # TODO(https://github.com/pytorch/executorch/issues/7441): Gradually start enabling all folders.
     # 'backends/**/*.py',
     'backends/arm/**/*.py',
+    'backends/openvino/**/*.py',
     'build/**/*.py',
     'codegen/**/*.py',
     # 'devtools/**/*.py',
     'devtools/visualization/**/*.py',
     'docs/**/*.py',
     # 'examples/**/*.py',
+    'examples/openvino/**/*.py',
     # 'exir/**/*.py',
     # 'extension/**/*.py',
     'kernels/**/*.py',

CMakeLists.txt

@@ -204,6 +204,8 @@ option(EXECUTORCH_BUILD_MPS "Build the MPS backend" OFF)
 
 option(EXECUTORCH_BUILD_NEURON "Build the backends/mediatek directory" OFF)
 
+option(EXECUTORCH_BUILD_OPENVINO "Build the Openvino backend" OFF)
+
 option(EXECUTORCH_BUILD_PYBIND "Build the Python Bindings" OFF)
 
 option(EXECUTORCH_BUILD_QNN "Build the Qualcomm backend" OFF)
@@ -715,6 +717,10 @@ if(EXECUTORCH_BUILD_NEURON)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/backends/mediatek)
 endif()
 
+if(EXECUTORCH_BUILD_OPENVINO)
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/backends/openvino)
+endif()
+
 if(EXECUTORCH_BUILD_QNN)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/backends/qualcomm)
 endif()
@@ -817,6 +823,10 @@ if(EXECUTORCH_BUILD_PYBIND)
     list(APPEND _dep_libs mpsdelegate)
   endif()
 
+  if(EXECUTORCH_BUILD_OPENVINO)
+    list(APPEND _dep_libs openvino_backend)
+  endif()
+
   if(EXECUTORCH_BUILD_XNNPACK)
     # need to explicitly specify XNNPACK and microkernels-prod
     # here otherwise uses XNNPACK and microkernel-prod symbols from libtorch_cpu

README.md

@@ -29,6 +29,7 @@ Platform Support:
   - Arm
   - Cadence
   - MediaTek
+  - OpenVINO
   - Qualcomm
   - Vulkan
   - XNNPACK

backends/openvino/CMakeLists.txt

@@ -0,0 +1,75 @@
+# Copyright (c) Intel Corporation
+#
+# Licensed under the BSD License (the "License"); you may not use this file
+# except in compliance with the License. See the license file found in the
+# LICENSE file in the root directory of this source tree.
+
+# Set minimum required CMake version
+cmake_minimum_required(VERSION 3.19)
+
+# Set project name
+project(openvino_backend_project)
+
+# Set C++ standard
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Ensure compile_commands.json is generated
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+# Set up EXECUTORCH_ROOT if not already set
+if(NOT EXECUTORCH_ROOT)
+  set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..)
+endif()
+
+# Define common include directories
+set(COMMON_INCLUDE_DIRS ${EXECUTORCH_ROOT}/..)
+
+# Include utility CMake scripts from ExecuteTorch
+include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake)
+
+# Find OpenVINO libraries
+find_package(OpenVINO REQUIRED)
+
+# Define OpenVINO backend as a static library
+add_library(openvino_backend STATIC .)
+
+# Enable exceptions and RTTI for OpenVINO backend
+target_compile_options(openvino_backend PRIVATE -frtti -fexceptions)
+
+# Include Executorch directories
+target_include_directories(openvino_backend PUBLIC ${COMMON_INCLUDE_DIRS})
+
+# Link OpenVINO and ExecuteTorch core libraries
+target_link_libraries(openvino_backend PRIVATE openvino::runtime executorch_core)
+
+# Add source files for OpenVINO backend
+target_sources(openvino_backend PRIVATE ${CMAKE_CURRENT_LIST_DIR}/runtime/OpenvinoBackend.cpp)
+
+target_link_options_shared_lib(openvino_backend)
+
+if(EXECUTORCH_BUILD_OPENVINO_EXECUTOR_RUNNER)
+    # Build executor runner binary for openvino backend
+    list(APPEND openvino_executor_runner_libs openvino_backend executorch)
+
+    set(_openvino_executor_runner__srcs
+        ${EXECUTORCH_ROOT}/examples/portable/executor_runner/executor_runner.cpp
+        ${EXECUTORCH_ROOT}/extension/data_loader/file_data_loader.cpp
+        ${EXECUTORCH_ROOT}/extension/evalue_util/print_evalue.cpp
+        ${EXECUTORCH_ROOT}/extension/runner_util/inputs.cpp
+        ${EXECUTORCH_ROOT}/extension/runner_util/inputs_portable.cpp
+        )
+    add_executable(openvino_executor_runner ${_openvino_executor_runner__srcs})
+
+    list(APPEND openvino_executor_runner_libs)
+
+    target_link_libraries(
+      openvino_executor_runner gflags portable_ops_lib ${openvino_executor_runner_libs}
+    )
+    target_compile_options(openvino_executor_runner PUBLIC ${_common_compile_options})
+endif()
+
+
+
+# Install OpenVINO backend library to the lib directory
+install(TARGETS openvino_backend DESTINATION lib)

backends/openvino/README.md

@@ -0,0 +1,89 @@
+# OpenVINO Backend for ExecuTorch
+The OpenVINO backend enables optimized execution of deep learning models on Intel hardware, leveraging Intel's [OpenVINO toolkit](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/overview.html) for inference acceleration.
+
+## Supported Hardware
+
+OpenVINO backend supports the following hardware:
+
+- Intel CPUs
+- Intel integrated GPUs
+- Intel discrete GPUs
+- Intel NPUs
+
+For more information on the supported hardware, please refer to [OpenVINO System Requirements](https://docs.openvino.ai/2025/about-openvino/release-notes-openvino/system-requirements.html) page.
+
+## Directory Structure
+
+```
+executorch
+├── backends
+│   └── openvino
+│       ├── runtime
+│           ├── OpenvinoBackend.cpp
+│           └── OpenvinoBackend.h
+│       ├── scripts
+│           └── openvino_build.sh
+│       ├── tests
+│       ├── CMakeLists.txt
+│       ├── README.md
+│       ├── __init__.py
+│       ├── partitioner.py
+│       ├── preprocess.py
+│       └── requirements.txt
+└── examples
+    └── openvino
+        ├── aot_optimize_and_infer.py
+        └── README.md
+```
+
+## Build Instructions
+
+### Prerequisites
+
+Before you begin, ensure you have openvino installed and configured on your system:
+
+```bash
+git clone https://github.com/openvinotoolkit/openvino.git
+cd openvino && git checkout releases/2025/1
+git submodule update --init --recursive
+sudo ./install_build_dependencies.sh
+mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON=ON
+make -j$(nproc)
+
+cd ..
+cmake --install build --prefix <your_preferred_install_location>
+cd <your_preferred_install_location>
+source setupvars.sh
+```
+Note: The OpenVINO backend is not yet supported with the current OpenVINO release packages. It is recommended to build from source. The instructions for using OpenVINO release packages will be added soon.
+For more information about OpenVINO build, refer to the [OpenVINO Build Instructions](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build_linux.md).
+
+### Setup
+
+Follow the steps below to setup your build environment:
+
+1. **Setup ExecuTorch Environment**: Refer to the [Environment Setup](https://pytorch.org/executorch/stable/getting-started-setup#environment-setup) guide for detailed instructions on setting up the ExecuTorch environment.
+
+2. **Setup OpenVINO Backend Environment**
+- Install the dependent libs. Ensure that you are inside `executorch/backends/openvino/` directory
+   ```bash
+   pip install -r requirements.txt
+   ```
+  Note: To achieve optimal performance with NNCF quantization, you should install the latest development version of NNCF (version 2.16.0.dev0+191b53d9 or higher).
+3. Navigate to `scripts/` directory.
+
+4. **Build OpenVINO Backend C++ Libraries and Executor Runner**: Once the prerequisites are in place, run the `openvino_build.sh` script to start the build process. By default, OpenVINO backend will be built under `cmake-out/backends/openvino/` as `libopenvino_backend.a`
+
+   ```bash
+   ./openvino_build.sh
+   ```
+   **Build OpenVINO Backend Python Package with Pybindings**: To build and install the OpenVINO backend Python package with Python bindings, run the `openvino_build.sh` script with the `--enable_python` argument. This will compile and install the ExecuTorch Python package with the OpenVINO backend into your Python environment. This option will also enable python bindings required to execute OpenVINO backend tests and `export_and_infer_openvino.py` script inside `executorch/examples/openvino` folder.
+
+   ```bash
+   ./openvino_build.sh --enable_python
+   ```
+
+### Run
+
+Please refer to [README.md](../../examples/openvino/README.md) for instructions on running examples of various of models with openvino backend.

backends/openvino/__init__.py

@@ -0,0 +1,5 @@
+from .partitioner import OpenvinoPartitioner
+from .preprocess import OpenvinoBackend
+from .quantizer.quantizer import OpenVINOQuantizer
+
+__all__ = ["OpenvinoBackend", "OpenvinoPartitioner", "OpenVINOQuantizer"]