[wasm] add a total wasm bytes quota to the jiterpreter; adjust parameters (#80266)

* Adjust jiterpreter parameters to avoid making S.T.J AOT tests slower
* Add a total wasm bytes quota to the jiterpreter to avoid generating way too much code in degenerate cases

Author: kg

src/mono/mono/utils/options-def.h

@@ -107,17 +107,20 @@ DEFINE_BOOL(jiterpreter_dump_traces, "jiterpreter-dump-traces", FALSE, "Dump the
 // Currently reduces performance significantly :(
 DEFINE_BOOL(jiterpreter_use_constants, "jiterpreter-use-constants", FALSE, "Use runtime imports for pointer constants")
 // any trace that doesn't have at least this many meaningful (non-nop) opcodes in it will be rejected
-DEFINE_INT(jiterpreter_minimum_trace_length, "jiterpreter-minimum-trace-length", 8, "Reject traces shorter than this number of meaningful opcodes")
+DEFINE_INT(jiterpreter_minimum_trace_length, "jiterpreter-minimum-trace-length", 10, "Reject traces shorter than this number of meaningful opcodes")
 // once a trace entry point is inserted, we only actually JIT code for it once it's been hit this many times
 DEFINE_INT(jiterpreter_minimum_trace_hit_count, "jiterpreter-minimum-trace-hit-count", 5000, "JIT trace entry points once they are hit this many times")
 // After a do_jit_call call site is hit this many times, we will queue it to be jitted
-DEFINE_INT(jiterpreter_jit_call_trampoline_hit_count, "jiterpreter-jit-call-hit-count", 2000, "Queue specialized do_jit_call trampoline for JIT after this many hits")
+DEFINE_INT(jiterpreter_jit_call_trampoline_hit_count, "jiterpreter-jit-call-hit-count", 1000, "Queue specialized do_jit_call trampoline for JIT after this many hits")
 // After a do_jit_call call site is hit this many times without being jitted, we will flush the JIT queue
-DEFINE_INT(jiterpreter_jit_call_queue_flush_threshold, "jiterpreter-jit-call-queue-flush-threshold", 8000, "Flush the do_jit_call JIT queue after an unJITted call site has this many hits")
+DEFINE_INT(jiterpreter_jit_call_queue_flush_threshold, "jiterpreter-jit-call-queue-flush-threshold", 6000, "Flush the do_jit_call JIT queue after an unJITted call site has this many hits")
 // After a generic interp_entry wrapper is hit this many times, we will queue it to be jitted
-DEFINE_INT(jiterpreter_interp_entry_trampoline_hit_count, "jiterpreter-interp-entry-hit-count", 250, "Queue specialized interp_entry wrapper for JIT after this many hits")
+DEFINE_INT(jiterpreter_interp_entry_trampoline_hit_count, "jiterpreter-interp-entry-hit-count", 1000, "Queue specialized interp_entry wrapper for JIT after this many hits")
 // After a generic interp_entry wrapper is hit this many times without being jitted, we will flush the JIT queue
-DEFINE_INT(jiterpreter_interp_entry_queue_flush_threshold, "jiterpreter-interp-entry-queue-flush-threshold", 1000, "Flush the interp_entry JIT queue after an unJITted call site has this many hits")
+DEFINE_INT(jiterpreter_interp_entry_queue_flush_threshold, "jiterpreter-interp-entry-queue-flush-threshold", 3000, "Flush the interp_entry JIT queue after an unJITted call site has this many hits")
+// In degenerate cases the jiterpreter could end up generating lots of WASM, so shut off jitting once it reaches this limit
+// Each wasm byte likely maps to multiple bytes of native code, so it's important for this limit not to be too high
+DEFINE_INT(jiterpreter_wasm_bytes_limit, "jiterpreter-wasm-bytes-limit", 6 * 1024 * 1024, "Disable jiterpreter code generation once this many bytes of WASM have been generated")
 #endif // HOST_BROWSER
 
 /* Cleanup */

src/mono/wasm/runtime/jiterpreter-interp-entry.ts

@@ -207,6 +207,12 @@ function flush_wasm_entry_trampoline_jit_queue () {
         trampBuilder = builder = new WasmBuilder(constantSlots);
     else
         builder.clear(constantSlots);
+
+    if (builder.options.wasmBytesLimit <= counters.bytesGenerated) {
+        jitQueue.length = 0;
+        return;
+    }
+
     const started = _now();
     let compileStarted = 0;
     let rejected = true, threw = false;
@@ -321,6 +327,7 @@ function flush_wasm_entry_trampoline_jit_queue () {
         const buffer = builder.getArrayView();
         if (trace > 0)
             console.log(`jit queue generated ${buffer.length} byte(s) of wasm`);
+        counters.bytesGenerated += buffer.length;
         const traceModule = new WebAssembly.Module(buffer);
 
         const imports : any = {

src/mono/wasm/runtime/jiterpreter-jit-call.ts

@@ -167,6 +167,7 @@ function getIsWasmEhSupported () : boolean {
         for (let i = 0; i < doJitCall16.length; i += 2)
             bytes[i / 2] = parseInt(doJitCall16.substring(i, i + 2), 16);
 
+        counters.bytesGenerated += bytes.length;
         doJitCallModule = new WebAssembly.Module(bytes);
         wasmEhSupported = true;
     } catch (exc) {
@@ -248,6 +249,11 @@ export function mono_interp_flush_jitcall_queue () : void {
     else
         builder.clear(0);
 
+    if (builder.options.wasmBytesLimit <= counters.bytesGenerated) {
+        jitQueue.length = 0;
+        return;
+    }
+
     if (builder.options.enableWasmEh) {
         if (!getIsWasmEhSupported()) {
             // The user requested to enable wasm EH but it's not supported, so turn the option back off
@@ -350,6 +356,7 @@ export function mono_interp_flush_jitcall_queue () : void {
         const buffer = builder.getArrayView();
         if (trace > 0)
             console.log(`do_jit_call queue flush generated ${buffer.length} byte(s) of wasm`);
+        counters.bytesGenerated += buffer.length;
         const traceModule = new WebAssembly.Module(buffer);
 
         const imports : any = {

src/mono/wasm/runtime/jiterpreter-support.ts

@@ -650,7 +650,8 @@ export const counters = {
     tracesCompiled: 0,
     entryWrappersCompiled: 0,
     jitCallsCompiled: 0,
-    failures: 0
+    failures: 0,
+    bytesGenerated: 0
 };
 
 export const _now = (globalThis.performance && globalThis.performance.now)
@@ -906,6 +907,8 @@ export type JiterpreterOptions = {
     jitCallFlushThreshold: number;
     interpEntryHitCount: number;
     interpEntryFlushThreshold: number;
+    // Maximum total number of wasm bytes to generate
+    wasmBytesLimit: number;
 }
 
 const optionNames : { [jsName: string] : string } = {
@@ -927,6 +930,7 @@ const optionNames : { [jsName: string] : string } = {
     "jitCallFlushThreshold": "jiterpreter-jit-call-queue-flush-threshold",
     "interpEntryHitCount": "jiterpreter-interp-entry-hit-count",
     "interpEntryFlushThreshold": "jiterpreter-interp-entry-queue-flush-threshold",
+    "wasmBytesLimit": "jiterpreter-wasm-bytes-limit",
 };
 
 let optionsVersion = -1;

src/mono/wasm/runtime/jiterpreter.ts

@@ -585,6 +585,7 @@ function generate_wasm (
         const buffer = builder.getArrayView();
         if (trace > 0)
             console.log(`${traceName} generated ${buffer.length} byte(s) of wasm`);
+        counters.bytesGenerated += buffer.length;
         const traceModule = new WebAssembly.Module(buffer);
 
         const imports : any = {
@@ -2936,6 +2937,8 @@ export function mono_interp_tier_prepare_jiterpreter (
     // FIXME: We shouldn't need this check
     if (!mostRecentOptions.enableTraces)
         return JITERPRETER_NOT_JITTED;
+    else if (mostRecentOptions.wasmBytesLimit <= counters.bytesGenerated)
+        return JITERPRETER_NOT_JITTED;
 
     let info = traceInfo[<any>ip];
 
@@ -2983,7 +2986,7 @@ export function jiterpreter_dump_stats (b?: boolean) {
     if (!mostRecentOptions.enableStats && (b !== undefined))
         return;
 
-    console.log(`// jiterpreter produced ${counters.tracesCompiled} traces from ${counters.traceCandidates} candidates (${(counters.tracesCompiled / counters.traceCandidates * 100).toFixed(1)}%), ${counters.jitCallsCompiled} jit_call trampolines, and ${counters.entryWrappersCompiled} interp_entry wrappers`);
+    console.log(`// generated: ${counters.bytesGenerated} wasm bytes; ${counters.tracesCompiled} traces (${counters.traceCandidates} candidates, ${(counters.tracesCompiled / counters.traceCandidates * 100).toFixed(1)}%); ${counters.jitCallsCompiled} jit_calls; ${counters.entryWrappersCompiled} interp_entries`);
     console.log(`// time spent: ${elapsedTimes.generation | 0}ms generating, ${elapsedTimes.compilation | 0}ms compiling wasm`);
     if (mostRecentOptions.countBailouts) {
         for (let i = 0; i < BailoutReasonNames.length; i++) {