[mono][interp] Tweaks to x86 call convention handling (#88466)

First, we no longer assert that the call convetion is cdecl on the slowpath. The pinvoke trampoline on x86 restores the saved stack pointer so it doesn't matter if the pinvoke callee pops or not the stack. Therefore both stdcall and cdecl calls should be supported with the same path. Untested.

Second, we use the fastpath (which uses the C call convention) only if the pinvoke signature is also cdecl, otherwise we have cconv mismatch.

Author: BrzVlad

src/mono/mono/mini/interp/transform.c

@@ -3787,10 +3787,20 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target
 			if (native && !method->dynamic)
 				icall_sig = interp_get_icall_sig (csignature);
 #endif
+			gboolean default_cconv = TRUE;
+#ifdef TARGET_X86
+#ifdef TARGET_WIN32
+			// Platform that we don't actively support ?
+			default_cconv = csignature->call_convention == MONO_CALL_C;
+#else
+			default_cconv = csignature->call_convention == MONO_CALL_DEFAULT || csignature->call_convention == MONO_CALL_C;
+#endif
+#endif
+
 			// FIXME calli receives both the args offset and sometimes another arg for the frame pointer,
 			// therefore some args are in the param area, while the fp is not. We should differentiate for
 			// this, probably once we will have an explicit param area where we copy arguments.
-			if (icall_sig != MINT_ICALLSIG_MAX) {
+			if (icall_sig != MINT_ICALLSIG_MAX && default_cconv) {
 				interp_add_ins (td, MINT_CALLI_NAT_FAST);
 				interp_ins_set_dreg (td->last_ins, dreg);
 				interp_ins_set_sregs2 (td->last_ins, fp_sreg, MINT_CALL_ARGS_SREG);
@@ -3804,10 +3814,6 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target
 				td->last_ins->data [0] = get_data_item_index (td, (void *)csignature);
 			} else if (native) {
 				interp_add_ins (td, MINT_CALLI_NAT);
-#ifdef TARGET_X86
-				/* Windows not tested/supported yet */
-				g_assertf (csignature->call_convention == MONO_CALL_DEFAULT || csignature->call_convention == MONO_CALL_C, "Interpreter supports only cdecl pinvoke on x86");
-#endif
 
 				InterpMethod *imethod = NULL;
 				/*