Merge branch 'master' into tc-fast-int

build.sh

@@ -43,7 +43,7 @@ WAVE="src/modules/wave/*.v3"
 WASI="src/modules/wasi/*.v3"
 WASI_X86_64_LINUX="src/modules/wasi/x86-64-linux/*.v3"
 WALI="src/modules/wali/*.v3"
-WALI_X86_64_LINUX="src/modules/wali/x86-64-linux/*.v3"
+WALI_X86_64_LINUX="src/modules/wali/x86-64-linux/*.v3 $VIRGIL_LIB/wasm-linux/*.v3"
 MODULES="src/modules/*.v3"
 
 PROGRAM=$1

ideas/Reflect.v3

@@ -0,0 +1,70 @@
+enum Kind {
+	I32, I64, F32, F64, V128, REF
+}
+
+type externref;
+type moduleref;
+type instanceref;
+type funcref;
+type frameref;
+type anyref;
+
+import "wizeng:reflect" component wizeng_reflect {
+	// reflection
+	def get_own_module() -> moduleref;
+	def get_own_instance() -> instanceref;
+	def get_comodule() -> moduleref;
+	def get_coinstance() -> instanceref;
+	def get_instance_func(module: moduleref, func: u31) -> funcref;
+
+	def eq_externref(a: externref, b: externref) -> bool;
+	def eq_anyref(a: anyref, b: anyref) -> bool;
+	def eq_module(a: moduleref, b: moduleref) -> bool;
+	def eq_instance(a: instanceref, b: instanceref) -> bool;
+	def eq_func(a: funcref, b: funcref) -> bool;
+}
+
+import "wizeng:reflect:module" component wizeng_reflect_module {
+	def get_type_count(module: moduleref) -> u31;
+	def get_func_count(module: moduleref) -> u31;
+	def get_memory_count(module: moduleref) -> u31;
+	def get_table_count(module: moduleref) -> u31;
+}
+
+import "wizeng:reflect:func" component wizeng_reflect_func {
+	def get_instance(func: funcref) -> instanceref;
+	def get_name(func: funcref, name_buf: u32/*Pointer*/);
+	def get_param_arity(func: funcref) -> u31;
+	def get_param_kind(func: funcref) -> Kind;
+	def get_result_arity(func: funcref) -> u31;
+	def get_result_kind(func: funcref) -> Kind;
+}
+
+import "wizeng:instrument" component wizeng_instrument {
+	def insert_global_probe(callback: funcref);
+	def insert_local_probe(func: funcref, pc: u31, callback: funcref);
+	def remove_local_probe(func: funcref, pc: u31, callback: funcref);
+}
+
+import "wizeng:reflect:frame" component wizeng_reflect_frame {
+	def get_module(accessor: frameref, func: funcref) -> moduleref;
+	def get_instance(accessor: frameref, func: funcref) -> instanceref;
+	def get_func(accessor: frameref) -> funcref;
+	def get_pc(accessor: frameref) -> u31;
+	def get_stp(accessor: frameref) -> u31;
+	def get_depth(accessor: frameref) -> u31;
+	def get_caller(accessor: frameref) -> frameref;
+
+	def get_local_count(accessor: frameref) -> u31;
+	def get_operand_count(accessor: frameref) -> u31;
+
+	def get_operand_i(accessor: frameref, d: int) -> u32;
+	def get_operand_l(accessor: frameref, d: int) -> u64;
+	def get_operand_f(accessor: frameref, d: int) -> float;
+	def get_operand_d(accessor: frameref, d: int) -> double;
+
+	def set_operand_i(accessor: frameref, d: int, val: u32);
+	def set_operand_l(accessor: frameref, d: int, val: u64);
+	def set_operand_f(accessor: frameref, d: int, val: float);
+	def set_operand_d(accessor: frameref, d: int, val: double);
+}

issues.txt

@@ -13,6 +13,7 @@ Prio	Component	Issue
 	spc		instr-combine for cmp+branch
 	spc             unit-test SpcState
 	spc		unit-test regalloc
+	spc		inlining of direct calls / whamm probes
 	interpreter	implement atomics
         interpreter     compress ctlxfer entries (4 byte cases)
 	spc		implement deoptimization for inserting probes when on-stack
@@ -26,16 +27,18 @@ Prio	Component	Issue
 
 	wizeng		repl for loading and running modules
 	validator	typecheck whole signatures at a time
+	validator	prettier trace output
 	spec		is it legal to have 0-count funcs and no code section?
 	parse		Tracing of indexes/offsets might be wrong
 	parse		Simplify BinParser state machine
-	util		Improve objdump utility
+	util		Improve objdump utility, add control rendering
 	util		Dump module into text format
         wasi            Implement x86-64-linux wasi
 
 	ext		extended constants
 	ext		memory64
 	ext		threads
+	ext		custom-page-sizes
 
         test            Add testing of --trace-interpreter and --trace-calls
 	test		Add fuzzy-matching of stacktraces for CRASH opcodes
@@ -51,20 +54,23 @@ Prio	Component	Issue
 	fast-int	tune: register assignment
 	fast-int	tune: handler order, alignment
 	fast-int	tune: inline some extended LEB cases?
-        fast-int        tune: hot-swap bytecode implementations
+        fast-int        tune: hot-swap bytecode implementations, e.g. memory64
 
 
 Monitors + trace
 --------------------------------
-	CallsMonitor should report source-names of callees
 	build general-purpose "what is this program doing" monitor
+	CallsMonitor should report source-names of callees
 	TablesMonitor should report read/write/calls through tables
 	TrapsMonitor shows state of program at trap points (default?)
 	VirgilMonitor hooks into sections of Virgil-generated Wasm
 	ExceptionMonitor tracks try/throw/catch
 	MallocMonitor tracks malloc/free behavior
            - tries to find memory leaks
 	   - perf metrics: total usage, average usage, distribution of alloc sizes, free distance
+	   - heatmap of allocation sites, memory ranges
+	   - timeseries of memory consumption (bytes, objects)
+	   - lifetime analysis (time between allocate and free)
 	MemoryMonitor records/reports at multiple granularities
 	   - read/write count
 	   - strides (FFT somehow ?)
@@ -82,7 +88,7 @@ Monitors + trace
 	  - wasm: traps, throw/catch, memory access, suspend/resume, host calls, frame unwind
 	  - engine: gc, compile, load, verify
         LoopMonitor	track and display loop count distribution, improve display
-        BranchMonitor   print full CFG and block nesting, better display of taken/not taken
+	ConstArgs track runtime function argument uniqueness to detect specialization/inlining opportunities
 
 
 Test Coverage
@@ -147,10 +153,6 @@ Instantiator
 Instance
 	evalInitExpr of f32, f64, FUNCREF, EXTERNREF, global
 
-HashMap<JawString>
-	cache hit
-	balance
-
 CodeValidator
 	.checkArgs of null
 	.readLocals exceeds maximum
@@ -219,6 +221,7 @@ Inline WasmObject.values array
 
 Done
 --------------------------------
+        BranchMonitor   print full CFG and block nesting (done in ControlMonitor)
 	ext		gc
 	ext		function-references
 	wave		Run monitor reports on System.error

src/engine/BinParser.v3

@@ -16,6 +16,7 @@ class BinParser(extensions: Extension.set, limits: Limits, filename: string) ext
 	private var parser: WasmParser;
 	private var decoder: DataReader;
 	private var fsm: BpFsm;
+	private var seen_section = -1;
 	private var seen_sections = Array<bool>.new(BpSection.count);
 	private var declared_func_count: u32;
 	private var func_body_cursor: int;
@@ -59,13 +60,7 @@ class BinParser(extensions: Extension.set, limits: Limits, filename: string) ext
 		var start = d.pos;
 		decoder = parser.decoder = d;
 //		parser.debugBuffer();
-		if (kind != BpSection.Unknown) {
-			if (seen_sections[kind.code]) {
-				if (!extensions.REPEAT_SECTIONS) return err.rel(d, d.pos-1).DuplicateSection(kind);
-			} else {
-				seen_sections[kind.code] = true;
-			}
-		}
+		enforceSectionOrder(kind, d.abs_of(d.pos-1));
 		err.section = kind;
 		match (kind) {
 			Unknown => readCustomSection(size);
@@ -121,7 +116,7 @@ class BinParser(extensions: Extension.set, limits: Limits, filename: string) ext
 			_ => return err.at(d).InvalidSectionCode(kind.code);
 		}
 		if (Trace.binparse) {
-			Trace.OUT.put2("======== end section code %d (%s) ========", kind.code, kind.name).outln();
+			Trace.OUT.put2("======== end section code %d (%s) ========", kind.code, kind.name).ln();
 		}
 
 		err.section = BpSection.Unknown;
@@ -130,16 +125,22 @@ class BinParser(extensions: Extension.set, limits: Limits, filename: string) ext
 			err.at(d).IncompleteSection(kind.code, read, size);
 		}
 	}
+	def enforceSectionOrder(kind: BpSection, abspos: int) {
+		if (kind == BpSection.Unknown) return;
+		if (!extensions.REPEAT_SECTIONS) {
+			// enforce section ordering and uniqueness
+			if (seen_sections[kind.code]) return err.abs(abspos).DuplicateSection(kind);
+			else if (kind.tag < seen_section) return err.abs(abspos).OutOfOrderSection(kind);
+		}
+		seen_sections[kind.code] = true;
+		seen_section = kind.tag;
+	}
 	def beginCodeSection(abspos: int, count: u32) {
 		var kind = BpSection.Code;
-		if (seen_sections[kind.code]) {
-			if (!extensions.REPEAT_SECTIONS) return err.abs(abspos - 1).DuplicateSection(kind);
-		} else {
-			seen_sections[kind.code] = true;
-			if (count != declared_func_count) {
-				// TODO: count >= remaining_func_count ?
-				err.abs(abspos).FuncCountMismatch(declared_func_count, count);
-			}
+		enforceSectionOrder(kind, abspos);
+		if (count != declared_func_count) {
+			// TODO: count >= remaining_func_count ?
+			err.abs(abspos).FuncCountMismatch(declared_func_count, count);
 		}
 		if (tiering != null) tiering.onCodeSectionStart(module, count, 0); // TODO: size
 	}
@@ -158,7 +159,7 @@ class BinParser(extensions: Extension.set, limits: Limits, filename: string) ext
 		for (i < count) {
 			if (err.error()) break;
 			err.index = i;
-			if (Trace.binparse) Trace.OUT.put2("-->%s #%d", thing_name, i).outln();
+			if (Trace.binparse) Trace.OUT.put2("-->%s #%d", thing_name, i).ln();
 			read(i);
 		}
 	}
@@ -179,7 +180,7 @@ class BinParser(extensions: Extension.set, limits: Limits, filename: string) ext
 		module.explicit_data_count = count;
 	}
 	def decodeFunctionBody(index: u32, size: u32, decoder: DataReader) {
-		if (Trace.binparse) Trace.OUT.put1("-->body #%d", index).outln();
+		if (Trace.binparse) Trace.OUT.put1("-->body #%d", index).ln();
 		parser.decoder = decoder;
 		var start = decoder.pos;
 		var f: FuncDecl, i = func_body_cursor;
@@ -307,7 +308,14 @@ class BinParser(extensions: Extension.set, limits: Limits, filename: string) ext
 		var shared = (flags & BpMemoryFlag.Shared.mask) != 0;
 		if (shared && !has_max) err.rel(decoder, pt).ExpectedSharedMemoryMaximum();
 		var indexType = if((flags & BpMemoryFlag.Is64.mask) != 0, ValueType.I64, ValueType.I32);
-		return MemLimits(initial, max, shared, indexType);
+
+		var log2_pageSize: u5 = BpConstants.log2_WASM_PAGE_SIZE;
+		var has_page_size = (flags & BpMemoryFlag.HasPageSize.mask) != 0;
+		if (has_page_size) {
+			log2_pageSize = u5.!(parser.readU32("pagesize (log2)", limits.max_log2_page_size));
+		}
+
+		return MemLimits(initial, max, shared, indexType, log2_pageSize);
 	}
 	def readElemType() -> ValueType.Ref {
 		return parser.readRefType(true);
@@ -464,7 +472,7 @@ class BinParser(extensions: Extension.set, limits: Limits, filename: string) ext
 			BpImportExportKind.Memory.code => {
 				reserveOne(pt, "memories", module.memories, limits.max_num_memories);
 				var l = readMemLimits();
-				decl = MemoryDecl.new(l.initial, l.max, l.shared, l.indexType);
+				decl = MemoryDecl.new(l.initial, l.max, l.shared, l.indexType, l.log2_pageSize);
 			}
 			BpImportExportKind.Global.code => {
 				reserveOne(pt, "globals", module.globals, limits.max_num_globals);
@@ -490,7 +498,7 @@ class BinParser(extensions: Extension.set, limits: Limits, filename: string) ext
 	}
 	def readMemoryDecl(index: int) {
 		var l = readMemLimits();
-		var decl = MemoryDecl.new(l.initial, l.max, l.shared, l.indexType);
+		var decl = MemoryDecl.new(l.initial, l.max, l.shared, l.indexType, l.log2_pageSize);
 		module.addDecl(decl);
 	}
 	def readTagDecl(index: int) {
@@ -749,11 +757,11 @@ class BpFsm extends WasmParser {
 		if (Trace.binparse) {
 // TODO: this is debug code, not tracing code
 //			Trace.OUT.put3("@+%d [%d bytes] BpFsm.state=%s", decoder.abspos(),
-//				decoder.available(), state.name).outln();
+//				decoder.available(), state.name).ln();
 		}
 		match (state) {
 			MAGIC => {
-				if (Trace.binparse) Trace.OUT.puts("======== begin module ========").outln();
+				if (Trace.binparse) Trace.OUT.puts("======== begin module ========").ln();
 				var pt = decoder.pos;
 				var magic = decoder.read_u32();
 				if (eof) return if(!retry(), err.at(decoder).EOFMagicWord());
@@ -775,7 +783,7 @@ class BpFsm extends WasmParser {
 				}
 				current_section = checkSectionCode(b);
 				if (Trace.binparse) {
-					Trace.OUT.put2("======== begin section code %d (%s) ========", current_section.code, current_section.name).outln();
+					Trace.OUT.put2("======== begin section code %d (%s) ========", current_section.code, current_section.name).ln();
 				}
 				state = BpFsmState.SECTION;
 			}
@@ -994,7 +1002,7 @@ type BrOnCastImm(flags: u8, depth: u31, ht1: int, ht2: int) #unboxed {
 	def null1() -> bool { return (flags & 1) != 0; }
 	def null2() -> bool { return (flags & 2) != 0; }
 }
-type MemLimits(initial: u64, max: Max, shared: bool, indexType: ValueType) #unboxed { }
+type MemLimits(initial: u64, max: Max, shared: bool, indexType: ValueType, log2_pageSize: u5) #unboxed { }
 type TableLimits() #unboxed { }
 
 type BlockTypeCode(code: int, index: int) #unboxed {

src/engine/BpConstants.v3

@@ -70,6 +70,7 @@ enum BpMemoryFlag(mask: byte) {
 	HasMax(0x01),
 	Shared(0x02),
 	Is64(0x04),
+	HasPageSize(0x08),
 }
 // Table flag masks.
 enum BpTableFlag(mask: byte) {
@@ -108,6 +109,8 @@ type BpCatchCode {
 component BpConstants {
 	def WASM_MAGIC = 0x6d736100u;
 	def WASM_VERSION = 0x01u;
+	def WASM_PAGE_SIZE = 65536u31;
+	def log2_WASM_PAGE_SIZE = 16u5;
 	def MEMARG_INDEX_FLAG = byte.view(0x40u);
 	def valid_section = Array<bool>.new(256);
 	def enum_section = Array<BpSection>.new(256);
@@ -209,6 +212,8 @@ component BpConstants {
 	def renderMemoryFlags(buf: StringBuilder, b: byte) -> StringBuilder {
 		if ((b & BpMemoryFlag.HasMax.mask) != 0) buf.puts("has_max");
 		if ((b & BpMemoryFlag.Shared.mask) != 0) buf.puts(" shared");
+		if ((b & BpMemoryFlag.Is64.mask) != 0) buf.puts(" is64");
+		if ((b & BpMemoryFlag.HasPageSize.mask) != 0) buf.puts(" has_page_size");
 		return buf;
 	}
 	def renderAttribute(buf: StringBuilder, b: byte) -> StringBuilder {

src/engine/Canon.v3

@@ -105,13 +105,13 @@ class HeapTypeCache {
 	// Canonicalize a group in {heaptypes[recgrp_start...(recgrp_start + count)]} and update the
 	// vector if necessary.
 	def doGroup(heaptypes: Vector<HeapTypeDecl>, recgrp_start: int, count: int) {
-		if (Trace.canon) Trace.OUT.put2("canon_group(%d, %d)", recgrp_start, count).outln();
+		if (Trace.canon) Trace.OUT.put2("canon_group(%d, %d)", recgrp_start, count).ln();
 		var g = RecGroup.new(count), max = recgrp_start + count;
 		for (i = recgrp_start; i < max; i++) g.add(heaptypes[i]);
 		var result = g.finish(this);
 		for (i = recgrp_start; i < max; i++) {
 			var r = result[i - recgrp_start];
-			if (Trace.canon) Trace.OUT.put2("  heaptypes[%d] = %q", i, r.render).outln();
+			if (Trace.canon) Trace.OUT.put2("  heaptypes[%d] = %q", i, r.render).ln();
 			heaptypes[i] = r;
 			r.heaptype_index = i;
 		}
@@ -161,13 +161,13 @@ class RecGroup(length: int) {
 	// Finish is idempotent: repeated calls return the same (cached) result.
 	def finish(cache: HeapTypeCache) -> Array<HeapTypeDecl> {
 		if (done != null) {
-			if (Trace.canon) Trace.OUT.puts("  recgrp done").outln();
+			if (Trace.canon) Trace.OUT.puts("  recgrp done").ln();
 			return done;
 		}
 		if (cache != null) {
 			var prev = cache.recGroups[this];
 			if (prev != null) {
-				if (Trace.canon) Trace.OUT.puts("  recgrp cached").outln();
+				if (Trace.canon) Trace.OUT.puts("  recgrp cached").ln();
 				return done = prev.done;
 			}
 		}
@@ -197,7 +197,7 @@ class RecGroup(length: int) {
 					}
 				}
 			}
-			if (Trace.canon) Trace.OUT.put3("%q hash=%d open=%z", d.render, d.hash, d != decls[i]).outln();
+			if (Trace.canon) Trace.OUT.put3("%q hash=%d open=%z", d.render, d.hash, d != decls[i]).ln();
 			d.recgrp_index = i;
 			done[i] = d;
 		}
@@ -214,7 +214,7 @@ class RecGroup(length: int) {
 				}
 			}
 			if (d.canon_id < 0 && cache != null) d.canon_id = ++cache.uid;
-			if (Trace.canon) Trace.OUT.put1("%q", d.render).outln();
+			if (Trace.canon) Trace.OUT.put1("%q", d.render).ln();
 		}
 		// cache this recursion group if a cache has been supplied
 		if (cache != null) cache.recGroups[this] = this;