Arm backend: Fix flaky sigmoid 16/32 bit tests (#9537)

- Change constant test data to random generators.
- Tests on Ethos-U55 are meant to xfail as int 16 tables are currently
not supported.
- For other tests, add flaky marker. Remove increased qtol, since the
inaccuracies only show up sporadically.

Signed-off-by: Erik Lundell <erik.lundell@arm.com>

Author: Erik-Lundell

backends/arm/test/common.py

@@ -231,6 +231,7 @@ def parametrize(
     arg_name: str,
     test_data: dict[str, Any],
     xfails: dict[str, xfail_type] | None = None,
+    strict: bool = True,
 ):
     """
     Custom version of pytest.mark.parametrize with some syntatic sugar and added xfail functionality
@@ -261,7 +262,9 @@ def decorator_func(func):
                 pytest_param = pytest.param(
                     test_parameters,
                     id=id,
-                    marks=pytest.mark.xfail(reason=reason, raises=raises, strict=True),
+                    marks=pytest.mark.xfail(
+                        reason=reason, raises=raises, strict=strict
+                    ),
                 )
             else:
                 pytest_param = pytest.param(test_parameters, id=id)

backends/arm/test/ops/test_sigmoid_16bit.py

@@ -3,6 +3,8 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import pytest
+
 import torch
 from executorch.backends.arm.quantizer.arm_quantizer import (
     get_symmetric_quantization_config,
@@ -48,12 +50,12 @@ def get_16bit_sigmoid_quantizer(tosa_str: str):
 
 input_t = tuple[torch.Tensor]
 test_data_suite = {
-    "ones": (torch.ones(10, 10, 10),),
-    "rand": (torch.rand(10, 10) - 0.5,),
-    "rand_4d": (torch.rand(1, 1, 5, 10),),
-    "randn_pos": (torch.randn(10) + 10,),
-    "randn_neg": (torch.randn(10) - 10,),
-    "ramp": (torch.arange(-16, 16, 0.02),),
+    "ones": lambda: torch.ones(10, 10, 10),
+    "rand": lambda: torch.rand(10, 10) - 0.5,
+    "rand_4d": lambda: torch.rand(1, 1, 5, 10),
+    "randn_pos": lambda: torch.randn(10) + 10,
+    "randn_neg": lambda: torch.randn(10) - 10,
+    "ramp": lambda: torch.arange(-16, 16, 0.02),
 }
 
 
@@ -79,8 +81,11 @@ def forward(self, x):
 
 
 @common.parametrize("test_data", test_data_suite)
+@pytest.mark.flaky(reruns=5)
 def test_sigmoid_tosa_BI(test_data):
-    pipeline = TosaPipelineBI(Sigmoid(), test_data, Sigmoid.aten_op, Sigmoid.exir_op)
+    pipeline = TosaPipelineBI(
+        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op
+    )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
 
@@ -89,12 +94,13 @@ def test_sigmoid_tosa_BI(test_data):
     "test_data",
     test_data_suite,
     xfails={
-        "ramp": "AssertionError: Output 0 does not match reference output. Passes with qtol=2. MLETORCH-787"
+        "ramp": "AssertionError: Output 0 does not match reference output. MLETORCH-787"
     },
 )
+@pytest.mark.flaky(reruns=5)
 def test_sigmoid_add_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
-        SigmoidAddSigmoid(), test_data, Sigmoid.aten_op, Sigmoid.exir_op
+        SigmoidAddSigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op
     )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
@@ -107,13 +113,18 @@ def test_sigmoid_add_sigmoid_tosa_BI(test_data):
         "ones": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "rand": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "rand_4d": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
+        "randn_pos": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
+        "randn_neg": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "ramp": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
     },
+    # int16 tables are not supported, but some tests happen to pass regardless.
+    # Set them to xfail but strict=False -> ok if they pass.
+    strict=False,
 )
 @common.XfailIfNoCorstone300
 def test_sigmoid_tosa_u55(test_data):
     pipeline = EthosU55PipelineBI(
-        Sigmoid(), test_data, Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
+        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
     )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI+u55"))
     pipeline.run()
@@ -127,14 +138,18 @@ def test_sigmoid_tosa_u55(test_data):
         "rand": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "rand_4d": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "randn_neg": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
-        "ramp": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
+        "randn_pos": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
+        "ramp": "AsssertionError: Output 0 does not match reference output. MLBEDSW-9770",
     },
+    # int16 tables are not supported, but some tests happen to pass regardless.
+    # Set them to xfail but strict=False -> ok if they pass.
+    strict=False,
 )
 @common.XfailIfNoCorstone300
 def test_sigmoid_add_sigmoid_tosa_u55(test_data):
     pipeline = EthosU55PipelineBI(
         SigmoidAddSigmoid(),
-        test_data,
+        (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
         run_on_fvp=True,
@@ -144,10 +159,11 @@ def test_sigmoid_add_sigmoid_tosa_u55(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
+@pytest.mark.flaky(reruns=5)
 @common.XfailIfNoCorstone320
 def test_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(
-        Sigmoid(), test_data, Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
+        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
     )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
@@ -160,11 +176,12 @@ def test_sigmoid_tosa_u85(test_data):
         "ramp": "AssertionError: Output 0 does not match reference output.",
     },
 )
+@pytest.mark.flaky(reruns=5)
 @common.XfailIfNoCorstone320
 def test_sigmoid_add_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(
         SigmoidAddSigmoid(),
-        test_data,
+        (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
         run_on_fvp=True,

backends/arm/test/ops/test_sigmoid_32bit.py

@@ -3,6 +3,7 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import pytest
 import torch
 from executorch.backends.arm.quantizer.arm_quantizer import TOSAQuantizer
 from executorch.backends.arm.quantizer.quantization_config import QuantizationConfig
@@ -52,7 +53,7 @@ def _get_32_bit_quant_config():
     return qconfig
 
 
-def get_16bit_sigmoid_quantizer(tosa_str: str):
+def get_32bit_sigmoid_quantizer(tosa_str: str):
     tosa_spec = common.TosaSpecification.create_from_string(tosa_str)
     quantizer = TOSAQuantizer(tosa_spec)
     quantizer.set_global(_get_32_bit_quant_config())
@@ -65,12 +66,12 @@ def get_16bit_sigmoid_quantizer(tosa_str: str):
 
 input_t = tuple[torch.Tensor]
 test_data_suite = {
-    "ones": (torch.ones(10, 10, 10),),
-    "rand": (torch.rand(10, 10) - 0.5,),
-    "rand_4d": (torch.rand(1, 10, 10, 10),),
-    "randn_pos": (torch.randn(10) + 10,),
-    "randn_neg": (torch.randn(10) - 10,),
-    "ramp": (torch.arange(-16, 16, 0.2),),
+    "ones": lambda: torch.ones(10, 10, 10),
+    "rand": lambda: torch.rand(10, 10) - 0.5,
+    "rand_4d": lambda: torch.rand(1, 10, 10, 10),
+    "randn_pos": lambda: torch.randn(10) + 10,
+    "randn_neg": lambda: torch.randn(10) - 10,
+    "ramp": lambda: torch.arange(-16, 16, 0.2),
 }
 
 
@@ -96,28 +97,28 @@ def forward(self, x):
 
 
 @common.parametrize("test_data", test_data_suite)
+@pytest.mark.flaky(reruns=5)
 def test_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
         Sigmoid(),
-        test_data,
+        (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
     )
-    pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
+    pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
 
 
 @common.parametrize("test_data", test_data_suite)
+@pytest.mark.flaky(reruns=5)
 def test_sigmoid_add_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
         SigmoidAddSigmoid(),
-        test_data,
+        (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
     )
-    pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
-    pipeline.change_args("run_method_and_compare_outputs", test_data, qtol=1)
-
+    pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
 
 
@@ -129,16 +130,19 @@ def test_sigmoid_add_sigmoid_tosa_BI(test_data):
         "rand": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "rand_4d": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "randn_pos": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
+        "randn_neg": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "ramp": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
     },
+    # int16 tables are not supported, but some tests happen to pass regardless.
+    # Set them to xfail but strict=False -> ok if they pass.
+    strict=False,
 )
 @common.XfailIfNoCorstone300
 def test_sigmoid_tosa_u55(test_data):
     pipeline = EthosU55PipelineBI(
-        Sigmoid(), test_data, Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
+        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
     )
-    pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI+u55"))
-    pipeline.change_args("run_method_and_compare_outputs", test_data, qtol=1)
+    pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI+u55"))
     pipeline.run()
 
 
@@ -153,29 +157,31 @@ def test_sigmoid_tosa_u55(test_data):
         "randn_neg": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "ramp": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
     },
+    # int16 tables are not supported, but some tests happen to pass regardless.
+    # Set them to xfail but strict=False -> ok if they pass.
+    strict=False,
 )
 @common.XfailIfNoCorstone300
 def test_sigmoid_add_sigmoid_tosa_u55(test_data):
     pipeline = EthosU55PipelineBI(
         SigmoidAddSigmoid(),
-        test_data,
+        (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
         run_on_fvp=True,
     )
-    pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI+u55"))
-    pipeline.change_args("run_method_and_compare_outputs", test_data, qtol=1)
+    pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI+u55"))
     pipeline.run()
 
 
 @common.parametrize("test_data", test_data_suite)
+@pytest.mark.flaky(reruns=5)
 @common.XfailIfNoCorstone320
 def test_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(
-        Sigmoid(), test_data, Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
+        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
     )
-    pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
-    pipeline.change_args("run_method_and_compare_outputs", test_data, qtol=1)
+    pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
 
 
@@ -186,15 +192,15 @@ def test_sigmoid_tosa_u85(test_data):
         "ramp": "AssertionError: Output 0 does not match reference output.",
     },
 )
+@pytest.mark.flaky(reruns=5)
 @common.XfailIfNoCorstone320
 def test_sigmoid_add_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(
         SigmoidAddSigmoid(),
-        test_data,
+        (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
         run_on_fvp=True,
     )
-    pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
-    pipeline.change_args("run_method_and_compare_outputs", test_data, qtol=1)
+    pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()