Validate heap types

Validate that GC is enabled if struct and array types are used by the module and
validate that both GC and isorecursive types are enabled when nontrivial rec
groups are used.

This fixes a fuzz bug in #5239 where initial contents included a rec group but
the fuzzer disabled GC. Since the resulting module passed validation, the rec
groups made it into the binary output, making the type section malformed.

Author: tlively

src/wasm/wasm-validator.cpp

@@ -3461,6 +3461,25 @@ static void validateTags(Module& module, ValidationInfo& info) {
   }
 }
 
+static void validateTypes(Module& module, ValidationInfo& info) {
+  for (auto type : ModuleUtils::collectHeapTypes(module)) {
+    if (type.getRecGroup().size() > 1) {
+      info.shouldBeTrue(module.features.hasGC() &&
+                          getTypeSystem() == TypeSystem::Isorecursive,
+                        type,
+                        "Recursion groups require GC [--enable-gc] and "
+                        "isorecursive types [--hybrid]");
+    }
+
+    if (!module.features.hasGC()) {
+      info.shouldBeTrue(
+        !type.isStruct(), type, "Struct types require GC [--enable-gc]");
+      info.shouldBeTrue(
+        !type.isArray(), type, "Array types require GC [--enable-gc]");
+    }
+  }
+}
+
 static void validateModule(Module& module, ValidationInfo& info) {
   // start
   if (module.start.is()) {
@@ -3505,6 +3524,7 @@ bool WasmValidator::validate(Module& module, Flags flags) {
     validateDataSegments(module, info);
     validateTables(module, info);
     validateTags(module, info);
+    validateTypes(module, info);
     validateModule(module, info);
     validateFeatures(module, info);
   }

test/lit/validation/gc-types-no-gc.wast

@@ -0,0 +1,25 @@
+;; Test that using GC types without GC is a validation error.
+
+;; RUN: not wasm-opt --hybrid -all --disable-gc %s 2>&1 | filecheck %s
+
+;; CHECK: Recursion groups require GC [--enable-gc] and isorecursive types [--hybrid]
+;; CHECK: Struct types require GC [--enable-gc]
+;; CHECK: Array types require GC [--enable-gc]
+
+(module
+  (rec
+    (type $f1 (func))
+    (type $f2 (func))
+  )
+
+  (type $struct (struct))
+  (type $array (array i32))
+
+  (func $test1 (type $f1)
+    (unreachable)
+  )
+
+  (func $test2 (param (ref $struct) (ref $array))
+    (unreachable)
+  )
+)