Fuzzer: Do not run other VMs if the Binaryen interpreter hits a VM limitation (#6483)

The VM limitation might be an OOM (which can change due to opts) or an
atomic wait (which can hang on proper VMs with support). We already
avoided running VMs on the optimized wasm in this case, but we still
ran them on the original wasm, which this changes, mainly to avoid that
atomic wait situation.

Author: kripken

scripts/fuzz_opt.py

@@ -947,30 +947,39 @@ def can_compare_to_others(self):
 
     def handle_pair(self, input, before_wasm, after_wasm, opts):
         global ignored_vm_runs
-        ignored_before = ignored_vm_runs
 
         before = self.run_vms(before_wasm)
 
-        # if the binaryen interpreter hit a host limitation on the original
-        # testcase, or for some other reason we need to ignore this, then stop
-        # (otherwise, a host limitation on say allocations may be hit in the
-        # 'before' but not in the 'after' as optimizations may remove it).
-        if before[self.bynterpreter] == IGNORE:
-            # the ignoring should have been noted during run_vms()
-            assert(ignored_vm_runs > ignored_before)
-            return
-
         after = self.run_vms(after_wasm)
         self.compare_before_and_after(before, after)
 
     def run_vms(self, wasm):
+        ignored_before = ignored_vm_runs
+
         # vm_results will map vms to their results
         vm_results = {}
         for vm in self.vms:
             if vm.can_run(wasm):
                 print(f'[CompareVMs] running {vm.name}')
                 vm_results[vm] = fix_output(vm.run(wasm))
 
+                # If the binaryen interpreter hit a host limitation then do not
+                # run other VMs, as that is risky: the host limitation may be an
+                # an OOM which could be very costly (lots of swapping, and the
+                # OOM may change after opts that remove allocations etc.), or it
+                # might be an atomic wait which other VMs implement fully (and
+                # the wait might be very long). In general host limitations
+                # should be rare (which can be verified by looking at the
+                # details of how many things we ended up ignoring), and when we
+                # see one we are in a situation that we can't fuzz properly.
+                if vm == self.bynterpreter and vm_results[vm] == IGNORE:
+                    print('(ignored, so not running other VMs)')
+
+                    # the ignoring should have been noted during run_vms()
+                    assert(ignored_vm_runs > ignored_before)
+
+                    return vm_results
+
         # compare between the vms on this specific input
         first_vm = None
         for vm in vm_results.keys():