Fuzzer: Do not run other VMs if the Binaryen interpreter hits a VM limitation

scripts/fuzz_opt.py

@@ -947,30 +947,39 @@ def can_compare_to_others(self):
 
     def handle_pair(self, input, before_wasm, after_wasm, opts):
         global ignored_vm_runs
-        ignored_before = ignored_vm_runs
 
         before = self.run_vms(before_wasm)
 
-        # if the binaryen interpreter hit a host limitation on the original
-        # testcase, or for some other reason we need to ignore this, then stop
-        # (otherwise, a host limitation on say allocations may be hit in the
-        # 'before' but not in the 'after' as optimizations may remove it).
-        if before[self.bynterpreter] == IGNORE:
-            # the ignoring should have been noted during run_vms()
-            assert(ignored_vm_runs > ignored_before)
-            return
-
         after = self.run_vms(after_wasm)
         self.compare_before_and_after(before, after)
 
     def run_vms(self, wasm):
+        ignored_before = ignored_vm_runs
+
         # vm_results will map vms to their results
         vm_results = {}
         for vm in self.vms:
             if vm.can_run(wasm):
                 print(f'[CompareVMs] running {vm.name}')
                 vm_results[vm] = fix_output(vm.run(wasm))
 
+                # If the binaryen interpreter hit a host limitation then do not
+                # run other VMs, as that is risky: the host limitation may be an
+                # an OOM which could be very costly (lots of swapping, and the
+                # OOM may change after opts that remove allocations etc.), or it
+                # might be an atomic wait which other VMs implement fully (and
+                # the wait might be very long). In general host limitations
+                # should be rare (which can be verified by looking at the
+                # details of how many things we ended up ignoring), and when we
+                # see one we are in a situation that we can't fuzz properly.
+                if vm == self.bynterpreter and vm_results[vm] == IGNORE:
+                    print('(ignored, so not running other VMs)')
+
+                    # the ignoring should have been noted during run_vms()
+                    assert(ignored_vm_runs > ignored_before)
+
+                    return vm_results
+
         # compare between the vms on this specific input
         first_vm = None
         for vm in vm_results.keys():