Arm test: Pass inputs to run_method_and_compare_outputs (#3750)

oscarandersson8218 · facebook-github-bot · commit da9b92df7c05 · 2024-06-14T08:13:20.000-07:00
Summary: Pass inputs to ArmTester.run_method_and_compare_outputs to make sure that the test data is from the same distribution as the calibration data. Change-Id: I690b9a22c1e12465364998bdaddd4903bcc1c18f Pull Request resolved: #3750 Reviewed By: kirklandsign Differential Revision: D58137923 Pulled By: digantdesai fbshipit-source-id: be335489824855c7ac8cb6aed77c2f723276191d
diff --git a/backends/arm/test/models/test_mobilenet_v2_arm.py b/backends/arm/test/models/test_mobilenet_v2_arm.py
@@ -60,7 +60,7 @@ def test_mv2_tosa_MI(self):
             .check(list(self.all_operators))
             .partition()
             .to_executorch()
-            .run_method_and_compare_outputs()
+            .run_method_and_compare_outputs(inputs=self.model_inputs)
         )
 
     def test_mv2_tosa_BI(self):
diff --git a/backends/arm/test/ops/test_add.py b/backends/arm/test/ops/test_add.py
@@ -71,7 +71,7 @@ def _test_add_tosa_MI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs()
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_add_tosa_BI_pipeline(
@@ -91,7 +91,7 @@ def _test_add_tosa_BI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs(qtol=1)
+            .run_method_and_compare_outputs(inputs=test_data, qtol=1)
         )
 
     def _test_add_u55_BI_pipeline(
diff --git a/backends/arm/test/ops/test_avg_pool.py b/backends/arm/test/ops/test_avg_pool.py
@@ -60,7 +60,7 @@ def _test_avgpool2d_tosa_MI_pipeline(
             .check_not(["executorch_exir_dialects_edge__ops_aten_avg_pool2d_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs()
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_avgpool2d_tosa_BI_pipeline(
@@ -81,7 +81,7 @@ def _test_avgpool2d_tosa_BI_pipeline(
             .check_not(["executorch_exir_dialects_edge__ops_aten_avg_pool2d_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs(qtol=1)
+            .run_method_and_compare_outputs(inputs=test_data, qtol=1)
         )
 
     def _test_avgpool2d_tosa_u55_BI_pipeline(
diff --git a/backends/arm/test/ops/test_batch_norm.py b/backends/arm/test/ops/test_batch_norm.py
@@ -552,7 +552,7 @@ def _test_batchnorm2d_tosa_MI_pipeline(
                 ]
             )
             .to_executorch()
-            .run_method_and_compare_outputs(test_data)
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_batchnorm2d_no_stats_tosa_MI_pipeline(
@@ -581,7 +581,7 @@ def _test_batchnorm2d_no_stats_tosa_MI_pipeline(
                 ]
             )
             .to_executorch()
-            .run_method_and_compare_outputs(test_data)
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_batchnorm2d_tosa_BI_pipeline(
@@ -613,7 +613,7 @@ def _test_batchnorm2d_tosa_BI_pipeline(
                 ]
             )
             .to_executorch()
-            .run_method_and_compare_outputs(test_data)
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_batchnorm2d_u55_BI_pipeline(
diff --git a/backends/arm/test/ops/test_clone.py b/backends/arm/test/ops/test_clone.py
@@ -47,7 +47,7 @@ def _test_clone_tosa_MI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs(qtol=1)
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_clone_tosa_BI_pipeline(
@@ -66,7 +66,7 @@ def _test_clone_tosa_BI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs(qtol=1)
+            .run_method_and_compare_outputs(inputs=test_data, qtol=1)
         )
 
     def _test_clone_tosa_u55_pipeline(
diff --git a/backends/arm/test/ops/test_conv.py b/backends/arm/test/ops/test_conv.py
@@ -256,7 +256,7 @@ def _test_conv2d_tosa_MI_pipeline(
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .check_not(["executorch_exir_dialects_edge__ops_aten_convolution_default"])
             .to_executorch()
-            .run_method_and_compare_outputs()
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_conv2d_tosa_BI_pipeline(
@@ -277,7 +277,7 @@ def _test_conv2d_tosa_BI_pipeline(
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .check_not(["executorch_exir_dialects_edge__ops_aten_convolution_default"])
             .to_executorch()
-            .run_method_and_compare_outputs(qtol=1)
+            .run_method_and_compare_outputs(inputs=test_data, qtol=1)
         )
 
     def _test_conv2d_u55_BI_pipeline(
diff --git a/backends/arm/test/ops/test_conv_combos.py b/backends/arm/test/ops/test_conv_combos.py
@@ -169,7 +169,7 @@ def _test_conv_combo_tosa_MI_pipeline(
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .check_not(list(module.edge_op_list))
             .to_executorch()
-            .run_method_and_compare_outputs()
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_conv_combo_tosa_BI_pipeline(
@@ -192,7 +192,9 @@ def _test_conv_combo_tosa_BI_pipeline(
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .check_not(list(module.edge_op_list))
             .to_executorch()
-            .run_method_and_compare_outputs(atol=atol, rtol=rtol, qtol=1)
+            .run_method_and_compare_outputs(
+                inputs=test_data, atol=atol, rtol=rtol, qtol=1
+            )
         )
 
     def _test_conv_combo_u55_BI_pipeline(
diff --git a/backends/arm/test/ops/test_depthwise_conv.py b/backends/arm/test/ops/test_depthwise_conv.py
@@ -142,7 +142,7 @@ def _test_dw_conv2d_tosa_MI_pipeline(
             .check_not(["executorch_exir_dialects_edge__ops_aten_convolution_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs()
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_dw_conv2d_tosa_BI_pipeline(
@@ -161,7 +161,7 @@ def _test_dw_conv2d_tosa_BI_pipeline(
             .check_not(["executorch_exir_dialects_edge__ops_aten_convolution_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs(qtol=1)
+            .run_method_and_compare_outputs(inputs=test_data, qtol=1)
         )
 
     def _test_dw_conv2d_u55_BI_pipeline(
diff --git a/backends/arm/test/ops/test_div.py b/backends/arm/test/ops/test_div.py
@@ -117,7 +117,7 @@ def _test_div_tosa_MI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs(test_data)
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_div_tosa_BI_pipeline(
@@ -137,7 +137,7 @@ def _test_div_tosa_BI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs(test_data)
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_div_u55_BI_pipeline(
diff --git a/backends/arm/test/ops/test_linear.py b/backends/arm/test/ops/test_linear.py
@@ -129,7 +129,7 @@ def _test_linear_tosa_MI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs()
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_linear_tosa_BI_pipeline(
@@ -149,7 +149,7 @@ def _test_linear_tosa_BI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs(qtol=True)
+            .run_method_and_compare_outputs(inputs=test_data, qtol=True)
         )
 
     def _test_linear_tosa_u55_BI_pipeline(
diff --git a/backends/arm/test/ops/test_mean_dim.py b/backends/arm/test/ops/test_mean_dim.py
@@ -65,7 +65,7 @@ def _test_meandim_tosa_MI_pipeline(
             .check_not(["executorch_exir_dialects_edge__ops_aten_mean_dim"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs()
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_meandim_tosa_BI_pipeline(
@@ -86,7 +86,7 @@ def _test_meandim_tosa_BI_pipeline(
             .check_not(["executorch_exir_dialects_edge__ops_aten_mean_dim"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs(qtol=1)
+            .run_method_and_compare_outputs(inputs=test_data, qtol=1)
         )
 
     def _test_meandim_tosa_u55_BI_pipeline(
diff --git a/backends/arm/test/ops/test_softmax.py b/backends/arm/test/ops/test_softmax.py
@@ -53,7 +53,7 @@ def _test_softmax_tosa_MI_pipeline(
             .check_not(["executorch_exir_dialects_edge__ops_aten__softmax_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs()
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_softmax_tosa_BI_pipeline(
@@ -74,7 +74,7 @@ def _test_softmax_tosa_BI_pipeline(
             .check_not(["executorch_exir_dialects_edge__ops_aten__softmax_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs(qtol=1)
+            .run_method_and_compare_outputs(inputs=test_data, qtol=1)
         )
 
     def _test_softmax_tosa_u55_BI_pipeline(
diff --git a/backends/arm/test/ops/test_view.py b/backends/arm/test/ops/test_view.py
@@ -44,7 +44,7 @@ def _test_view_tosa_MI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs(qtol=1)
+            .run_method_and_compare_outputs(inputs=test_data)
         )
 
     def _test_view_tosa_BI_pipeline(
@@ -63,7 +63,7 @@ def _test_view_tosa_BI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
-            .run_method_and_compare_outputs(qtol=1)
+            .run_method_and_compare_outputs(inputs=test_data, qtol=1)
         )
 
     def _test_view_u55_BI_pipeline(

Original file line number	Diff line number	Diff line change
`@@ -60,7 +60,7 @@ def test_mv2_tosa_MI(self):`
`60`	`60`	`.check(list(self.all_operators))`
`61`	`61`	`.partition()`
`62`	`62`	`.to_executorch()`
`63`		`- .run_method_and_compare_outputs()`
	`63`	`+ .run_method_and_compare_outputs(inputs=self.model_inputs)`
`64`	`64`	`)`
`65`	`65`
`66`	`66`	`def test_mv2_tosa_BI(self):`
Original file line number	Diff line number	Diff line change
`@@ -552,7 +552,7 @@ def _test_batchnorm2d_tosa_MI_pipeline(`
`552`	`552`	`]`
`553`	`553`	`)`
`554`	`554`	`.to_executorch()`
`555`		`- .run_method_and_compare_outputs(test_data)`
	`555`	`+ .run_method_and_compare_outputs(inputs=test_data)`
`556`	`556`	`)`
`557`	`557`
`558`	`558`	`def _test_batchnorm2d_no_stats_tosa_MI_pipeline(`
`@@ -581,7 +581,7 @@ def _test_batchnorm2d_no_stats_tosa_MI_pipeline(`
`581`	`581`	`]`
`582`	`582`	`)`
`583`	`583`	`.to_executorch()`
`584`		`- .run_method_and_compare_outputs(test_data)`
	`584`	`+ .run_method_and_compare_outputs(inputs=test_data)`
`585`	`585`	`)`
`586`	`586`
`587`	`587`	`def _test_batchnorm2d_tosa_BI_pipeline(`
`@@ -613,7 +613,7 @@ def _test_batchnorm2d_tosa_BI_pipeline(`
`613`	`613`	`]`
`614`	`614`	`)`
`615`	`615`	`.to_executorch()`
`616`		`- .run_method_and_compare_outputs(test_data)`
	`616`	`+ .run_method_and_compare_outputs(inputs=test_data)`
`617`	`617`	`)`
`618`	`618`
`619`	`619`	`def _test_batchnorm2d_u55_BI_pipeline(`