Add support for (ref ...) / (ref null ...) constructs (#2562)

This large patch adds function references support for those parts of the
wabt library, which already present in the code. The current code is
designed for an older (outdated) variant of the function references
proposal.

Key changes:
- Var variables can optionally store type data. This way there is no
need for a Var/Type field pair in many cases.
- CallRef now needs to explicitly provide its reference type (no auto
detection)
- Tracking local reference initialization (non nullable refs must be set
before using them)
- Supporting type equality comparisons in the validator and in the
interpreter
- Adds a code for reading/writing 33 bit integers (leb format)
- Remove EndTypeSection in shared validator, types can be validated
earlier
- Improve named reference resolving in the text parser

Co-authored-by: Zoltan Herczeg <hzmester@freemail.hu>

Author: zherczeg

include/wabt/binary-reader-logging.h

@@ -181,7 +181,7 @@ class BinaryReaderLogging : public BinaryReaderDelegate {
   Result OnCatchExpr(Index tag_index) override;
   Result OnCatchAllExpr() override;
   Result OnCallIndirectExpr(Index sig_index, Index table_index) override;
-  Result OnCallRefExpr() override;
+  Result OnCallRefExpr(Type sig_type) override;
   Result OnCompareExpr(Opcode opcode) override;
   Result OnConvertExpr(Opcode opcode) override;
   Result OnDelegateExpr(Index depth) override;

include/wabt/binary-reader-nop.h

@@ -250,7 +250,7 @@ class BinaryReaderNop : public BinaryReaderDelegate {
   Result OnCallIndirectExpr(Index sig_index, Index table_index) override {
     return Result::Ok;
   }
-  Result OnCallRefExpr() override { return Result::Ok; }
+  Result OnCallRefExpr(Type sig_type) override { return Result::Ok; }
   Result OnCatchExpr(Index tag_index) override { return Result::Ok; }
   Result OnCatchAllExpr() override { return Result::Ok; }
   Result OnCompareExpr(Opcode opcode) override { return Result::Ok; }

include/wabt/binary-reader.h

@@ -255,7 +255,7 @@ class BinaryReaderDelegate {
                                Index default_target_depth) = 0;
   virtual Result OnCallExpr(Index func_index) = 0;
   virtual Result OnCallIndirectExpr(Index sig_index, Index table_index) = 0;
-  virtual Result OnCallRefExpr() = 0;
+  virtual Result OnCallRefExpr(Type sig_type) = 0;
   virtual Result OnCatchExpr(Index tag_index) = 0;
   virtual Result OnCatchAllExpr() = 0;
   virtual Result OnCompareExpr(Opcode opcode) = 0;

include/wabt/interp/interp-inl.h

@@ -42,7 +42,7 @@ inline bool FuncType::classof(const ExternType* type) {
 }
 
 inline FuncType::FuncType(ValueTypes params, ValueTypes results)
-    : ExternType(ExternKind::Func), params(params), results(results) {}
+    : ExternType(ExternKind::Func), params(params), results(results), func_types(nullptr) {}
 
 //// TableType ////
 // static

include/wabt/interp/interp.h

@@ -184,6 +184,10 @@ struct FuncType : ExternType {
 
   ValueTypes params;
   ValueTypes results;
+  // When params or results contain references, the referenced
+  // types are also needed for type equality comparisons.
+  // An example for these comparisons is import validation.
+  std::vector<FuncType>* func_types;
 };
 
 struct TableType : ExternType {

include/wabt/ir.h

@@ -36,24 +36,43 @@ namespace wabt {
 
 struct Module;
 
-enum class VarType {
+// VarType (16 bit) and the opt_type_ (16 bit)
+// fields of Var forms a 32 bit field.
+enum class VarType : uint16_t {
   Index,
   Name,
 };
 
 struct Var {
+  // Var can represent variables or types.
+
+  // Represent a variable:
+  //   has_opt_type() is false
+  //   Only used by wast-parser
+
+  // Represent a type:
+  //   has_opt_type() is true, is_index() is true
+  //   type can be get by to_type()
+  //   Binary reader only constructs this variant
+
+  // Represent both a variable and a type:
+  //   has_opt_type() is true, is_name() is true
+  //   A reference, which index is unknown
+  //   Only used by wast-parser
+
   explicit Var();
   explicit Var(Index index, const Location& loc);
   explicit Var(std::string_view name, const Location& loc);
+  explicit Var(Type type, const Location& loc);
   Var(Var&&);
   Var(const Var&);
   Var& operator=(const Var&);
   Var& operator=(Var&&);
   ~Var();
 
-  VarType type() const { return type_; }
   bool is_index() const { return type_ == VarType::Index; }
   bool is_name() const { return type_ == VarType::Name; }
+  bool has_opt_type() const { return opt_type_ < 0; }
 
   Index index() const {
     assert(is_index());
@@ -63,17 +82,25 @@ struct Var {
     assert(is_name());
     return name_;
   }
+  Type::Enum opt_type() const {
+    assert(has_opt_type());
+    return static_cast<Type::Enum>(opt_type_);
+  }
 
   void set_index(Index);
   void set_name(std::string&&);
   void set_name(std::string_view);
+  void set_opt_type(Type::Enum);
+  Type to_type() const;
 
   Location loc;
 
  private:
   void Destroy();
 
   VarType type_;
+  // Can be set to Type::Enum types, Type::Any represent no optional type.
+  int16_t opt_type_;
   union {
     Index index_;
     std::string name_;
@@ -155,6 +182,7 @@ struct Const {
   }
   void set_funcref() { From<uintptr_t>(Type::FuncRef, 0); }
   void set_externref(uintptr_t x) { From(Type::ExternRef, x); }
+  void set_extern(uintptr_t x) { From(Type(Type::ExternRef, Type::ReferenceNonNull), x); }
   void set_null(Type type) { From<uintptr_t>(type, kRefNullBits); }
 
   bool is_expected_nan(int lane = 0) const {
@@ -537,10 +565,10 @@ using MemoryCopyExpr = MemoryBinaryExpr<ExprType::MemoryCopy>;
 template <ExprType TypeEnum>
 class RefTypeExpr : public ExprMixin<TypeEnum> {
  public:
-  RefTypeExpr(Type type, const Location& loc = Location())
+  RefTypeExpr(Var type, const Location& loc = Location())
       : ExprMixin<TypeEnum>(loc), type(type) {}
 
-  Type type;
+  Var type;
 };
 
 using RefNullExpr = RefTypeExpr<ExprType::RefNull>;
@@ -662,8 +690,8 @@ using MemoryInitExpr = MemoryVarExpr<ExprType::MemoryInit>;
 
 class SelectExpr : public ExprMixin<ExprType::Select> {
  public:
-  SelectExpr(TypeVector type, const Location& loc = Location())
-      : ExprMixin<ExprType::Select>(loc), result_type(type) {}
+  SelectExpr(const Location& loc = Location())
+      : ExprMixin<ExprType::Select>(loc) {}
   TypeVector result_type;
 };
 
@@ -727,9 +755,7 @@ class CallRefExpr : public ExprMixin<ExprType::CallRef> {
   explicit CallRefExpr(const Location& loc = Location())
       : ExprMixin<ExprType::CallRef>(loc) {}
 
-  // This field is setup only during Validate phase,
-  // so keep that in mind when you use it.
-  Var function_type_index;
+  Var sig_type;
 };
 
 template <ExprType TypeEnum>

include/wabt/shared-validator.h

@@ -43,6 +43,7 @@ struct ValidateOptions {
 class SharedValidator {
  public:
   WABT_DISALLOW_COPY_AND_ASSIGN(SharedValidator);
+  using FuncType = TypeChecker::FuncType;
   SharedValidator(Errors*, const ValidateOptions& options);
 
   // TODO: Move into SharedValidator?
@@ -72,7 +73,6 @@ class SharedValidator {
                     Index type_index);
   Result OnStructType(const Location&, Index field_count, TypeMut* fields);
   Result OnArrayType(const Location&, TypeMut field);
-  Result EndTypeSection();
 
   Result OnFunction(const Location&, Var sig_var);
   Result OnTable(const Location&, Type elem_type, const Limits&);
@@ -141,7 +141,7 @@ class SharedValidator {
   Result EndBrTable(const Location&);
   Result OnCall(const Location&, Var func_var);
   Result OnCallIndirect(const Location&, Var sig_var, Var table_var);
-  Result OnCallRef(const Location&, Index* function_type_index);
+  Result OnCallRef(const Location&, Var function_type_var);
   Result OnCatch(const Location&, Var tag_var, bool is_catch_all);
   Result OnCompare(const Location&, Opcode);
   Result OnConst(const Location&, Type);
@@ -178,7 +178,7 @@ class SharedValidator {
   Result OnNop(const Location&);
   Result OnRefFunc(const Location&, Var func_var);
   Result OnRefIsNull(const Location&);
-  Result OnRefNull(const Location&, Type type);
+  Result OnRefNull(const Location&, Var func_type_var);
   Result OnRethrow(const Location&, Var depth);
   Result OnReturnCall(const Location&, Var func_var);
   Result OnReturnCallIndirect(const Location&, Var sig_var, Var table_var);
@@ -221,18 +221,6 @@ class SharedValidator {
   Result OnUnreachable(const Location&);
 
  private:
-  struct FuncType {
-    FuncType() = default;
-    FuncType(const TypeVector& params,
-             const TypeVector& results,
-             Index type_index)
-        : params(params), results(results), type_index(type_index) {}
-
-    TypeVector params;
-    TypeVector results;
-    Index type_index;
-  };
-
   struct StructType {
     StructType() = default;
     StructType(const TypeMutVector& fields) : fields(fields) {}
@@ -289,6 +277,13 @@ class SharedValidator {
     Index end;
   };
 
+  struct LocalReferenceMap {
+    Type type;
+    // An index for a single bit value, which represents that
+    // the corresponding local reference has been set before.
+    size_t local_ref_is_set;
+  };
+
   bool ValidInitOpcode(Opcode opcode) const;
   Result CheckInstr(Opcode opcode, const Location& loc);
   Result CheckType(const Location&,
@@ -336,6 +331,10 @@ class SharedValidator {
 
   TypeVector ToTypeVector(Index count, const Type* types);
 
+  void SaveLocalRefs();
+  void RestoreLocalRefs(Result result);
+  void IgnoreLocalRefs();
+
   ValidateOptions options_;
   Errors* errors_;
   TypeChecker typechecker_;  // TODO: Move into SharedValidator.
@@ -361,6 +360,8 @@ class SharedValidator {
   // Includes parameters, since this is only used for validating
   // local.{get,set,tee} instructions.
   std::vector<LocalDecl> locals_;
+  std::map<Index, LocalReferenceMap> local_refs_map_;
+  std::vector<bool> local_ref_is_set_;
 
   std::set<std::string> export_names_;  // Used to check for duplicates.
   std::set<Index> declared_funcs_;      // TODO: optimize?

include/wabt/token.def

@@ -55,6 +55,7 @@ WABT_TOKEN(Module, "module")
 WABT_TOKEN(Mut, "mut")
 WABT_TOKEN(NanArithmetic, "nan:arithmetic")
 WABT_TOKEN(NanCanonical, "nan:canonical")
+WABT_TOKEN(Null, "null")
 WABT_TOKEN(Offset, "offset")
 WABT_TOKEN(Output, "output")
 WABT_TOKEN(PageSize, "pagesize")

include/wabt/type-checker.h

@@ -20,6 +20,7 @@
 #include <functional>
 #include <type_traits>
 #include <vector>
+#include <map>
 
 #include "wabt/common.h"
 #include "wabt/feature.h"
@@ -31,6 +32,18 @@ class TypeChecker {
  public:
   using ErrorCallback = std::function<void(const char* msg)>;
 
+  struct FuncType {
+    FuncType() = default;
+    FuncType(const TypeVector& params,
+             const TypeVector& results,
+             Index type_index)
+        : params(params), results(results), type_index(type_index) {}
+
+    TypeVector params;
+    TypeVector results;
+    Index type_index;
+  };
+
   struct Label {
     Label(LabelType,
           const TypeVector& param_types,
@@ -46,9 +59,11 @@ class TypeChecker {
     TypeVector result_types;
     size_t type_stack_limit;
     bool unreachable;
+    std::vector<bool> local_ref_is_set_;
   };
 
-  explicit TypeChecker(const Features& features) : features_(features) {}
+  explicit TypeChecker(const Features& features, std::map<Index, FuncType>& func_types)
+    : features_(features), func_types_(func_types) {}
 
   void set_error_callback(const ErrorCallback& error_callback) {
     error_callback_ = error_callback;
@@ -80,7 +95,7 @@ class TypeChecker {
   Result OnCallIndirect(const TypeVector& param_types,
                         const TypeVector& result_types,
                         const Limits& table_limits);
-  Result OnIndexedFuncRef(Index* out_index);
+  Result OnCallRef(Type);
   Result OnReturnCall(const TypeVector& param_types,
                       const TypeVector& result_types);
   Result OnReturnCallIndirect(const TypeVector& param_types,
@@ -115,7 +130,7 @@ class TypeChecker {
   Result OnTableGrow(Type elem_type, const Limits& limits);
   Result OnTableSize(const Limits& limits);
   Result OnTableFill(Type elem_type, const Limits& limits);
-  Result OnRefFuncExpr(Index func_type, bool force_generic_funcref);
+  Result OnRefFuncExpr(Index func_type);
   Result OnRefNullExpr(Type type);
   Result OnRefIsNullExpr();
   Result OnRethrow(Index depth);
@@ -141,7 +156,7 @@ class TypeChecker {
   Result BeginInitExpr(Type type);
   Result EndInitExpr();
 
-  static Result CheckType(Type actual, Type expected);
+  Result CheckType(Type actual, Type expected);
 
  private:
   void WABT_PRINTF_FORMAT(2, 3) PrintError(const char* fmt, ...);
@@ -210,6 +225,7 @@ class TypeChecker {
   // to represent "any".
   TypeVector* br_table_sig_ = nullptr;
   Features features_;
+  std::map<Index, FuncType>& func_types_;
 };
 
 }  // namespace wabt