Skip to content

Commit

Permalink
Add all relevant testcases for Arm Ethos-U85 (#5346)
Browse files Browse the repository at this point in the history
Summary:
Add separate tests for Ethos-U85 to all backend operator tests.
Updated ethos-u-vela version to support more operators.

Signed-off-by: Per Åstrand <[per.astrand@arm.com](mailto:per.astrand@arm.com)>
Signed-off-by: Tom Allsop <[tom.allsop@arm.com](mailto:tom.allsop@arm.com)>

Pull Request resolved: #5346

Reviewed By: manuelcandales

Differential Revision: D62875027

Pulled By: digantdesai

fbshipit-source-id: 3bf238d81957258ee93ae235d575beff8a575191
  • Loading branch information
zingo authored and facebook-github-bot committed Sep 24, 2024
1 parent 5a984cc commit f4728f4
Show file tree
Hide file tree
Showing 29 changed files with 637 additions and 135 deletions.
4 changes: 2 additions & 2 deletions backends/arm/arm_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ def __init__(self):
def ethosu_compile_spec(
self,
config: str,
system_config: Optional[str] = None,
memory_mode: Optional[str] = None,
system_config: str,
memory_mode: str,
extra_flags: Optional[str] = None,
config_ini: Optional[str] = "Arm/vela.ini",
) -> "ArmCompileSpecBuilder":
Expand Down
15 changes: 15 additions & 0 deletions backends/arm/test/models/test_mobilenet_v2_arm.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,3 +102,18 @@ def test_mv2_u55_BI(self):
tester.run_method_and_compare_outputs(
atol=1.0, qtol=1, inputs=self.model_inputs
)

def test_mv2_u85_BI(self):
(
ArmTester(
self.mv2,
example_inputs=self.model_inputs,
compile_spec=common.get_u85_compile_spec(permute_memory_to_nhwc=True),
)
.quantize()
.export()
.to_edge(config=self._edge_compile_config)
.check(list(self.operators_after_quantization))
.partition()
.to_executorch()
)
39 changes: 33 additions & 6 deletions backends/arm/test/ops/test_add.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from executorch.backends.arm.test import common
from executorch.backends.arm.test.tester.arm_tester import ArmTester
from executorch.exir import EdgeCompileConfig
from executorch.exir.backend.compile_spec_schema import CompileSpec
from parameterized import parameterized


Expand Down Expand Up @@ -92,16 +93,17 @@ def _test_add_tosa_BI_pipeline(
.run_method_and_compare_outputs(inputs=test_data, qtol=1)
)

def _test_add_u55_BI_pipeline(
def _test_add_ethos_BI_pipeline(
self,
module: torch.nn.Module,
compile_spec: CompileSpec,
test_data: Tuple[torch.Tensor],
):
tester = (
ArmTester(
module,
example_inputs=test_data,
compile_spec=common.get_u55_compile_spec(permute_memory_to_nhwc=True),
compile_spec=compile_spec,
)
.quantize()
.export()
Expand All @@ -114,8 +116,7 @@ def _test_add_u55_BI_pipeline(
.serialize()
)

if common.is_option_enabled("corstone300"):
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
return tester

@parameterized.expand(Add.test_parameters)
def test_add_tosa_MI(self, test_data: torch.Tensor):
Expand All @@ -130,7 +131,22 @@ def test_add_tosa_BI(self, test_data: torch.Tensor):
@parameterized.expand(Add.test_parameters)
def test_add_u55_BI(self, test_data: torch.Tensor):
test_data = (test_data,)
self._test_add_u55_BI_pipeline(self.Add(), test_data)
tester = self._test_add_ethos_BI_pipeline(
self.Add(),
common.get_u55_compile_spec(permute_memory_to_nhwc=True),
test_data,
)
if common.is_option_enabled("corstone300"):
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)

@parameterized.expand(Add.test_parameters)
def test_add_u85_BI(self, test_data: torch.Tensor):
test_data = (test_data,)
self._test_add_ethos_BI_pipeline(
self.Add(),
common.get_u85_compile_spec(permute_memory_to_nhwc=True),
test_data,
)

@parameterized.expand(Add2.test_parameters)
def test_add2_tosa_MI(self, operand1: torch.Tensor, operand2: torch.Tensor):
Expand All @@ -145,4 +161,15 @@ def test_add2_tosa_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
@parameterized.expand(Add2.test_parameters)
def test_add2_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
test_data = (operand1, operand2)
self._test_add_u55_BI_pipeline(self.Add2(), test_data)
tester = self._test_add_ethos_BI_pipeline(
self.Add2(), common.get_u55_compile_spec(), test_data
)
if common.is_option_enabled("corstone300"):
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)

@parameterized.expand(Add2.test_parameters)
def test_add2_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
test_data = (operand1, operand2)
self._test_add_ethos_BI_pipeline(
self.Add2(), common.get_u85_compile_spec(), test_data
)
30 changes: 25 additions & 5 deletions backends/arm/test/ops/test_avg_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import torch
from executorch.backends.arm.test import common
from executorch.backends.arm.test.tester.arm_tester import ArmTester
from executorch.exir.backend.backend_details import CompileSpec
from parameterized import parameterized

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -86,14 +87,17 @@ def _test_avgpool2d_tosa_BI_pipeline(
.run_method_and_compare_outputs(inputs=test_data, qtol=1)
)

def _test_avgpool2d_tosa_u55_BI_pipeline(
self, module: torch.nn.Module, test_data: Tuple[torch.tensor]
def _test_avgpool2d_tosa_ethos_BI_pipeline(
self,
module: torch.nn.Module,
compile_spec: CompileSpec,
test_data: Tuple[torch.tensor],
):
(
ArmTester(
module,
example_inputs=test_data,
compile_spec=common.get_u55_compile_spec(permute_memory_to_nhwc=True),
compile_spec=compile_spec,
)
.quantize()
.export()
Expand Down Expand Up @@ -141,6 +145,22 @@ def test_avgpool2d_tosa_u55_BI(
test_data: torch.Tensor,
model_params: int | Tuple[int, int],
):
self._test_avgpool2d_tosa_u55_BI_pipeline(
self.AvgPool2d(*model_params), (test_data,)
self._test_avgpool2d_tosa_ethos_BI_pipeline(
self.AvgPool2d(*model_params),
common.get_u55_compile_spec(permute_memory_to_nhwc=True),
(test_data,),
)

@parameterized.expand(test_data_suite)
@unittest.expectedFailure
def test_avgpool2d_tosa_u85_BI(
self,
test_name: str,
test_data: torch.Tensor,
model_params: int | Tuple[int, int],
):
self._test_avgpool2d_tosa_ethos_BI_pipeline(
self.AvgPool2d(*model_params),
common.get_u85_compile_spec(permute_memory_to_nhwc=True),
(test_data,),
)
21 changes: 17 additions & 4 deletions backends/arm/test/ops/test_bmm.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import torch
from executorch.backends.arm.test import common
from executorch.backends.arm.test.tester.arm_tester import ArmTester
from executorch.exir.backend.compile_spec_schema import CompileSpec
from parameterized import parameterized

torch.manual_seed(1)
Expand Down Expand Up @@ -83,14 +84,17 @@ def _test_bmm_tosa_BI_pipeline(
.run_method_and_compare_outputs(inputs=test_data)
)

def _test_bmm_u55_BI_pipeline(
self, module: torch.nn.Module, test_data: Tuple[torch.Tensor, ...]
def _test_bmm_ethosu_BI_pipeline(
self,
module: torch.nn.Module,
compile_spec: CompileSpec,
test_data: Tuple[torch.Tensor, ...],
):
(
ArmTester(
module,
example_inputs=test_data,
compile_spec=common.get_u55_compile_spec(),
compile_spec=compile_spec,
)
.quantize()
.export()
Expand Down Expand Up @@ -132,4 +136,13 @@ def test_bmm_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
@unittest.expectedFailure
def test_bmm_single_input_u55_BI(self, operand1: torch.Tensor):
test_data = (operand1,)
self._test_bmm_u55_BI_pipeline(self.BMMSingleInput(), test_data)
self._test_bmm_ethosu_BI_pipeline(
self.BMMSingleInput(), common.get_u55_compile_spec(), test_data
)

@parameterized.expand(BMMSingleInput.test_parameters)
def test_bmm_single_input_u85_BI(self, operand1: torch.Tensor):
test_data = (operand1,)
self._test_bmm_ethosu_BI_pipeline(
self.BMMSingleInput(), common.get_u85_compile_spec(), test_data
)
23 changes: 17 additions & 6 deletions backends/arm/test/ops/test_cat.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from executorch.backends.arm.test import common

from executorch.backends.arm.test.tester.arm_tester import ArmTester
from executorch.exir.backend.compile_spec_schema import CompileSpec
from parameterized import parameterized


Expand Down Expand Up @@ -89,14 +90,17 @@ def _test_cat_tosa_BI_pipeline(
.run_method_and_compare_outputs(inputs=test_data, qtol=1)
)

def _test_cat_u55_BI_pipeline(
self, module: torch.nn.Module, test_data: Tuple[tuple[torch.Tensor, ...], int]
def _test_cat_ethosu_BI_pipeline(
self,
module: torch.nn.Module,
compile_spec: CompileSpec,
test_data: Tuple[tuple[torch.Tensor, ...], int],
):
(
ArmTester(
module,
example_inputs=test_data,
compile_spec=common.get_u55_compile_spec(),
compile_spec=compile_spec,
)
.quantize()
.export()
Expand Down Expand Up @@ -125,9 +129,16 @@ def test_cat_tosa_BI(self, operands: tuple[torch.Tensor, ...], dim: int):
test_data = (operands, dim)
self._test_cat_tosa_BI_pipeline(self.Cat(), test_data)

# TODO: Remove @unittest.expectedFailure when this issue is fixed in Regor
@parameterized.expand(Cat.test_parameters)
@unittest.expectedFailure
def test_cat_u55_BI(self, operands: tuple[torch.Tensor, ...], dim: int):
test_data = (operands, dim)
self._test_cat_u55_BI_pipeline(self.Cat(), test_data)
self._test_cat_ethosu_BI_pipeline(
self.Cat(), common.get_u55_compile_spec(), test_data
)

@parameterized.expand(Cat.test_parameters)
def test_cat_u85_BI(self, operands: tuple[torch.Tensor, ...], dim: int):
test_data = (operands, dim)
self._test_cat_ethosu_BI_pipeline(
self.Cat(), common.get_u85_compile_spec(), test_data
)
33 changes: 26 additions & 7 deletions backends/arm/test/ops/test_clone.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
from executorch.backends.arm.test.tester.arm_tester import ArmTester

from executorch.backends.xnnpack.test.tester.tester import Quantize

from executorch.exir.backend.compile_spec_schema import CompileSpec
from parameterized import parameterized


Expand Down Expand Up @@ -76,16 +78,15 @@ def _test_clone_tosa_BI_pipeline(
.run_method_and_compare_outputs(inputs=test_data, qtol=1)
)

def _test_clone_tosa_u55_pipeline(
self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
def _test_clone_tosa_ethos_pipeline(
self,
compile_spec: list[CompileSpec],
module: torch.nn.Module,
test_data: Tuple[torch.Tensor],
):
quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config())
(
ArmTester(
module,
example_inputs=test_data,
compile_spec=common.get_u55_compile_spec(),
)
ArmTester(module, example_inputs=test_data, compile_spec=compile_spec)
.quantize(Quantize(quantizer, get_symmetric_quantization_config()))
.export()
.check_count({"torch.ops.aten.clone.default": 1})
Expand All @@ -95,6 +96,20 @@ def _test_clone_tosa_u55_pipeline(
.to_executorch()
)

def _test_clone_tosa_u55_pipeline(
self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
):
self._test_clone_tosa_ethos_pipeline(
common.get_u55_compile_spec(), module, test_data
)

def _test_clone_tosa_u85_pipeline(
self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
):
self._test_clone_tosa_ethos_pipeline(
common.get_u85_compile_spec(), module, test_data
)

@parameterized.expand(Clone.test_parameters)
def test_clone_tosa_MI(self, test_tensor: torch.Tensor):
self._test_clone_tosa_MI_pipeline(self.Clone(), (test_tensor,))
Expand All @@ -106,3 +121,7 @@ def test_clone_tosa_BI(self, test_tensor: torch.Tensor):
@parameterized.expand(Clone.test_parameters)
def test_clone_u55_BI(self, test_tensor: torch.Tensor):
self._test_clone_tosa_u55_pipeline(self.Clone(), (test_tensor,))

@parameterized.expand(Clone.test_parameters)
def test_clone_u85_BI(self, test_tensor: torch.Tensor):
self._test_clone_tosa_u85_pipeline(self.Clone(), (test_tensor,))
24 changes: 20 additions & 4 deletions backends/arm/test/ops/test_conv.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from executorch.backends.arm.test import common

from executorch.backends.arm.test.tester.arm_tester import ArmTester
from executorch.exir.backend.compile_spec_schema import CompileSpec
from parameterized import parameterized


Expand Down Expand Up @@ -297,14 +298,17 @@ def _test_conv2d_tosa_BI_pipeline(
.run_method_and_compare_outputs(inputs=test_data, qtol=1)
)

def _test_conv2d_u55_BI_pipeline(
self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
def _test_conv2d_ethosu_BI_pipeline(
self,
compile_spec: CompileSpec,
module: torch.nn.Module,
test_data: Tuple[torch.Tensor],
):
(
ArmTester(
module,
example_inputs=test_data,
compile_spec=common.get_u55_compile_spec(permute_memory_to_nhwc=True),
compile_spec=compile_spec,
)
.quantize()
.export()
Expand All @@ -325,4 +329,16 @@ def test_conv2d_tosa_BI(self, test_name, model):

@parameterized.expand(testsuite_u55)
def test_conv2d_u55_BI(self, test_name, model):
self._test_conv2d_u55_BI_pipeline(model, model.get_inputs())
self._test_conv2d_ethosu_BI_pipeline(
common.get_u55_compile_spec(permute_memory_to_nhwc=True),
model,
model.get_inputs(),
)

@parameterized.expand(testsuite_u55)
def test_conv2d_u85_BI(self, test_name, model):
self._test_conv2d_ethosu_BI_pipeline(
common.get_u85_compile_spec(permute_memory_to_nhwc=True),
model,
model.get_inputs(),
)
Loading

0 comments on commit f4728f4

Please sign in to comment.