SinglePassCompiler.v3

// Copyright 2022 Ben L. Titzer. All rights reserved.
// See LICENSE for details of Apache 2.0 license.

// Describes the register and frame configuration for the single-pass compiler.
class SpcExecEnv {
	// Frame information.
	var frameSize: int;
	var vsp_slot: MasmAddr;
	var vfp_slot: MasmAddr;
	var pc_slot: MasmAddr;
	var instance_slot: MasmAddr;
	var inlined_instance_slot: MasmAddr;
	var wasm_func_slot: MasmAddr;
	var mem0_base_slot: MasmAddr;
	var inlined_mem0_base_slot: MasmAddr;
	var accessor_slot: MasmAddr;

	// Register information.
	var sp: Reg;
	var func_arg: Reg;
	var vsp: Reg;
	var vfp: Reg;
	var mem0_base: Reg;
	var instance: Reg;
	var runtime_arg0: Reg;
	var runtime_arg1: Reg;
	var runtime_arg2: Reg;
	var runtime_arg3: Reg;
	var runtime_arg4: Reg;
	var runtime_ret0: Reg;
	var runtime_ret1: Reg;
	var ret_throw: Reg;
	var scratch: Reg;
}

def INITIAL_VALUE_STACK_SIZE = 16;
def OUT = Trace.OUT;
def regRefCounts = Array<int>.new(128); // used for paranoid checking of regalloc state
def TMP_SLOT = 1000000000;

// Expose constants outside this file.
component SpcConsts {
	// Abstract values tracked during single-pass compilation.
	def NO_REG = Reg(0);
	def IS_STORED: byte = 0x01;
	def IS_CONST: byte = 0x02;
	def IN_REG: byte = 0x04;
	def TAG_STORED: byte = 0x08;
	def KIND_MASK: byte = 0xF0;
	def KIND_I32: byte = kindToFlags(ValueKind.I32);
	def KIND_I64: byte = kindToFlags(ValueKind.I64);
	def KIND_F32: byte = kindToFlags(ValueKind.F32);
	def KIND_F64: byte = kindToFlags(ValueKind.F64);
	def KIND_V128: byte = kindToFlags(ValueKind.V128);
	def KIND_REF: byte = kindToFlags(ValueKind.REF);
	def KIND_REF_U64: byte = kindToFlags(ValueKind.REF_U64);
	def kinds: Array<ValueKind> = [ValueKind.I32, ValueKind.I64, ValueKind.F32, ValueKind.F64,
					ValueKind.V128, ValueKind.REF, ValueKind.REF_U64];
	def kindToFlags(kind: ValueKind) -> byte {
		return byte.view(kind.tag) << 4;
	}
}

// Shorten constants inside this file.
def NO_REG = SpcConsts.NO_REG;
def IS_STORED = SpcConsts.IS_STORED;
def IS_CONST = SpcConsts.IS_CONST;
def IN_REG = SpcConsts.IN_REG;
def TAG_STORED = SpcConsts.TAG_STORED;
def KIND_MASK = SpcConsts.KIND_MASK;
def KIND_I32 = SpcConsts.KIND_I32;
def KIND_I64 = SpcConsts.KIND_I64;
def KIND_F32 = SpcConsts.KIND_F32;
def KIND_F64 = SpcConsts.KIND_F64;
def KIND_V128 = SpcConsts.KIND_V128;
def KIND_REF = SpcConsts.KIND_REF;
def KIND_REF_U64 = SpcConsts.KIND_REF_U64;

// Compiles Wasm bytecode to machine code in a single pass via a MacroAssembler.
class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAlloc, extensions: Extension.set, limits: Limits) extends BytecodeVisitor {
	def instrTracer = if(Trace.compiler, InstrTracer.new());
	def config = masm.regConfig;
	def regs = xenv;
	def frame = xenv;
	def resolver = SpcMoveResolver.new(masm);
	def unrefs = Array<(Reg, int)>.new(config.regSet.regs.length);
	def probeSpillMode = if(SpcTuning.probeCallFreesRegs, SpillMode.SAVE_AND_FREE_REGS, SpillMode.SAVE_ONLY);
	def runtimeSpillMode = if(SpcTuning.runtimeCallFreesRegs, SpillMode.SAVE_AND_FREE_REGS, SpillMode.SAVE_ONLY);
	var err: ErrorGen;
	var num_unrefs = 0;

	// Abstract state of the value stack
	def state = SpcState.new(regAlloc);
	// Other state
	def trap_labels = Vector<(TrapReason, MasmLabel, Array<SpcFrame>)>.new();

	var it = BytecodeIterator.new();

	// Frame state (refers to the fields in the top SpcFrame in SpcState)
	var module: Module;
	var func: FuncDecl;
	var sig: SigDecl;
	var num_locals: int;
	var local_base_sp: u31; // can use a Range for 0-indexing instead of from offset
	var ctl_base_sp: u31;   // index of the RETURN control in ctl_stack for the current frame

	var success = true;
	var osr_pc: int;
	var osr_offset: int;
	var osr_state: Array<SpcVal>;
	var osr_loop_label: MasmLabel;
	var osr_entry_label: MasmLabel;
	var ret_label: MasmLabel;
	var last_probe = 0;
	var skip_to_end: bool;
	var whamm_config: WhammInlineConfig;
	var frames_reconstructed = false;
	// XXX: hack
	var handler_dest_info = Vector<SpcHandlerInfo>.new();

	// tracks the last masm writer offset to generate instruction trace for each bytecode.
	var codegen_offset: u64 = 0;

	var intrinsified_read_probe: MemoryReadProbe = null;

	new() {
		masm.unimplemented = unsupported;
		masm.newTrapLabel = newTrapLabel; // trap labels are per-pc
	}

	def gen(module: Module, func: FuncDecl, err: ErrorGen) -> bool {
		this.osr_pc = -1;
		this.err = err;
		return Metrics.spc_time_us.run(gen0, (module, func));
	}
	def genOsr(module: Module, func: FuncDecl, pc: int, err: ErrorGen) -> MasmLabel {
		this.osr_pc = pc;
		this.err = err;
		var ok = Metrics.spc_time_us.run(gen0, (module, func));
		return if(ok, osr_entry_label);
	}
	private def gen0(module: Module, func: FuncDecl) -> bool {
		if (Trace.compiler) OUT.put1("==== begin compile: %q ========================", func.render(module.names, _)).ln();
		var before_code_bytes = masm.curCodeBytes();
		var before_data_bytes = masm.curDataBytes();

		// Reset internal state.
		regAlloc.clear();
		trap_labels.resize(0);
		success = true;
		osr_offset = -1;
		osr_state = null;
		handler_dest_info.clear();
		handler_dest_info.resize(func.handlers.handler_dests.length);

		// Reset frame state
		this.module = null;
		this.func = null;
		this.sig = null;
		this.num_locals = 0;
		this.local_base_sp = 0;

		// Initialize parameters, locals, and first control stack entry.
		var end_label = masm.newLabel(func.cur_bytecode.length);
		state.reset(func.sig, end_label);

		// Push initial frame for top-level function
		state.frame_stack.clear();
		var initial_frame = SpcFrame.new(func, module, 0, 0, func.num_slots(), 0, masm.newLabel(func.cur_bytecode.length));
		pushSpcFrame(initial_frame);

		// Emit prologue, which allocates the frame and initializes various registers.
		emitPrologue();

		// Visit all local declarations.
		it.dispatchLocalDecls(this);

		if (!FeatureDisable.frameVariables && func.frame_var_tags != null) {
			for (t in func.frame_var_tags) state.push(tagToKindFlags(t) | IS_CONST, NO_REG, 0);
		}

		// Emit function entry probe, if any.
		if (!FeatureDisable.entryProbes && func.entry_probed) {
			var probe = Instrumentation.getLocalProbe(module, func.func_index, 0);
			withReconstructedInlinedFrames(fun =>
				emitProbe0(0, probe));
		}

		masm.current_fid = func.func_index;
		// Emit instructions.
		while (it.more() && success) {
			if (Trace.compiler) traceOpcodeAndStack(false);
			last_probe = 0;
			masm.source_loc = it.pc;
			it.dispatch(this);
			unrefRegs();
			if (Trace.compiler && Trace.asm) {
				OUT.puts("JIT code: ");
				masm.printCodeBytes(OUT, before_code_bytes, masm.curCodeBytes());
				before_code_bytes = masm.curCodeBytes();
				OUT.ln();
			}
			if (Debug.compiler) checkRegAlloc();
			it.next();
			if (skip_to_end) doSkipToEndOfBlock();
		}

		// Emit trap labels.
		for (i < trap_labels.length) {
			var e = trap_labels[i];
			var reason = e.0;
			var label = e.1;
			var frames = e.2;

			masm.bindLabel(label);

			if (frames.length > 1) {
				unrefRegs();
				emitReconstructStackFrames(frames);
			} else {
				masm.emit_mov_m_i(xenv.pc_slot, label.create_pos);
			}
			masm.emit_jump_to_trap_at(reason);
		}

		// Emit handler stubs.
		if (Trace.exception) Trace.OUT.put1("Generating %d handler stubs", handler_dest_info.length).ln();
		var handlers = func.handlers;
		for (i < handler_dest_info.length) {
			// Dummy destinations do not need a stub.
			if (handlers.handler_dests[i].is_dummy) {
				if (Trace.compiler) Trace.OUT.put1("  handler stub #%d: DUMMY", i).ln();
				continue;
			}
			var info = handler_dest_info[i];
			handlers.handler_dests[i].dest_label = info.dest_label;
			handlers.handler_dests[i].stub_label = info.stub_label;

			// TODO: impl this; this is only null when not implemented on SPC side
			// - delegate
			if (info.stub_label == null) continue;
			masm.bindLabel(info.stub_label);
			X86_64MasmLabel.!(info.stub_label).label.pos -= int.view(before_code_bytes);
			if (Trace.compiler) {
				Trace.OUT.put1("  handler stub #%d:", i).ln();
				Trace.OUT.put1("    stub offset=+%d", X86_64MasmLabel.!(info.stub_label).label.pos).ln();
				Trace.OUT.put1("    end of func=%s", if(info.func_end, "true", "false")).ln();
			}

			if (info.func_end) {
				if (Trace.compiler) Trace.OUT.put1("    moving %d values to ($vfp)", func.sig.results.length).ln();
				for (to_slot < u32.!(func.sig.results.length)) {
					var from_slot = to_slot + func.num_locals;
					var sv = info.merge_state[to_slot];
					var fv = SpcVal(sv.kindFlags(IS_STORED), NO_REG, sv.const);
					var tv = SpcVal(typeToKindFlags(func.sig.results[to_slot]) | IS_STORED, NO_REG, 0);
					resolver.addMove((to_slot, tv), (from_slot, fv));
				}
			} else {
				for (slot < u32.!(info.merge_state.length)) {
					var sv = info.merge_state[slot];
					if (!sv.inReg()) continue;
					var fv = SpcVal(sv.kindFlags(IS_STORED), NO_REG, sv.const);
					var tv = SpcVal(sv.kindFlags(IN_REG), sv.reg, sv.const);
					resolver.addMove((slot, tv), (slot, fv));
				}
			}
			emit_reload_regs();
			resolver.emitMoves();

			masm.emit_br(info.dest_label);
		}

		// Emit OSR entry.
		if (osr_state != null) {
			osr_entry_label = masm.newLabel(osr_pc);
			emitOsrEntry(osr_entry_label, osr_state);
		}

		if (success) {
			// Metric collection
			Metrics.spc_in_bytes.val += u32.view(func.orig_bytecode.length);
			Metrics.spc_code_bytes.val += (masm.curCodeBytes() - before_code_bytes);
			Metrics.spc_data_bytes.val += (masm.curDataBytes() - before_data_bytes);
			Metrics.spc_functions.val++;
		}
		return success;
	}
	def doSkipToEndOfBlock() {
		skip_to_end = false;
		var height = 0;
		while (it.more() && success) {
			var opcode = it.current();
			match (opcode) {
				BLOCK, LOOP, TRY, TRY_TABLE, IF => height++;
				ELSE, CATCH, CATCH_ALL => if (height == 0) return;
				END, DELEGATE => if (height-- == 0) return;
				_ => ;
			}
			if (Trace.compiler) traceOpcodeUnreachable(true);
			it.next();
		}
	}
	def checkRegAlloc() {
		if (Trace.compiler) {
			OUT.puts("checkRegAlloc ");
			regAlloc.render(Trace.OUT, config.regSet);
			state.trace();
		}
		for (i < regRefCounts.length) regRefCounts[i] = 0;
		for (i < state.sp) {
			var sv = state.state[i];
			if (sv.inReg())  {
				var ri = sv.reg.index;
				assert1(ri != 0, "state[%d].inReg but reg == 0", i);
				assert1(ri < config.regSet.length, "state[%d], has invalid register", i);
				var pool = config.poolMap.regToPool[ri];
				assert1(pool >= 0 && pool < config.poolMap.numRegPools, "state[%d] has invalid register pool", i);
				var poolk = config.poolMap.kindToPool[sv.kind().tag];
				assert1(pool == poolk, "state[%d] differs on pool membership", i);
				regRefCounts[ri]++;
			}
		}
		for (i = 1; i < config.regSet.length; i++) {
			var r = Reg(byte.view(i));
			var buf = StringBuilder.new().puts("{");
			regAlloc.forEachAssignment(r, appendSlots(buf, _));
			var slots = buf.puts("}").toString();
			if (regRefCounts[i] == 0) {
				assert2(regAlloc.isFree(r), "%s should be free, got slots = %s", config.regSet.getName(r), slots);
			} else {
				assert2(!regAlloc.isFree(r), "%s should be allocated, got slots = %s", config.regSet.getName(r), slots);
			}
		}
	}
	def appendSlots(buf: StringBuilder, slot: int) -> StringBuilder {
		if (buf.length > 1) buf.csp();
		return buf.putd(slot);
	}
	def assert1<T>(cond: bool, msg: string, param: T) {
		if (!cond) bailout(Strings.format1(msg, param));
	}
	def assert2<T, U>(cond: bool, msg: string, p1: T, p2: U) {
		if (!cond) bailout(Strings.format2(msg, p1, p2));
	}
	def assert3<T, U, V>(cond: bool, msg: string, p1: T, p2: U, p3: V) {
		if (!cond) bailout(Strings.format3(msg, p1, p2, p3));
	}
	def emitPrologue() {
		// Allocate stack frame
		masm.emit_subw_r_i(regs.sp, frame.frameSize);

		// Spill VSP
		emit_spill_vsp(regs.vsp); // XXX: track VSP-spilled state
		// Spill wf: WasmFunction
		masm.emit_mov_m_r(ValueKind.REF, frame.wasm_func_slot, regs.func_arg);
		// Load wf.instance and spill
		masm.emit_v3_WasmFunction_instance_r_r(regs.instance, regs.func_arg);
		masm.emit_mov_m_r(ValueKind.REF, frame.instance_slot, regs.instance);
		// Clear FrameAccessor
		masm.emit_mov_m_l(frame.accessor_slot, 0); // XXX: value kind
		// Clear inlined whamm instance
		if (SpcTuning.inlineWhammProbes && SpcTuning.intrinsifyWhammProbe) {
			masm.emit_mov_m_l(frame.inlined_instance_slot, 0);
		}

		// Compute VFP = VSP - sig.params.length * SLOT_SIZE
		masm.emit_mov_r_r(ValueKind.REF, regs.vfp, regs.vsp); // XXX: use 3-addr adjustment of VFP
		masm.emit_subw_r_i(regs.vfp, sig.params.length * masm.valuerep.slot_size);
		// XXX: skip spilling of VFP
		masm.emit_mov_m_r(ValueKind.REF, frame.vfp_slot, regs.vfp);

		// Load instance.memories[0].start into MEM0_BASE and spill
		if (module.memories.length > 0) {
			// XXX: skip loading memory base if function doesn't access memory
			masm.emit_v3_Instance_memories_r_r(regs.mem0_base, regs.instance);
			masm.emit_v3_Array_elem_r_ri(ValueKind.REF, regs.mem0_base, regs.mem0_base, 0);
			masm.emit_v3_Memory_start_r_r(regs.mem0_base, regs.mem0_base);
			masm.emit_mov_m_r(ValueKind.REF, frame.mem0_base_slot, regs.mem0_base);
		}
	}
	def visitLocalDecl(count: u32, vtc: ValueTypeCode) {
		var vt = vtc.toAbstractValueType(module);
		var sp = state.sp;
		state.addLocals(count, vt);
		if (!SpcTuning.eagerTagLocals || !masm.valuerep.tagged) return;
		// eagerly tag locals
		var code = ValueTypes.kind(vt).code;
		for (i < count) masm.emit_mov_m_i(masm.tagAddr(sp + i), code);
	}
	def visitOp(opcode: Opcode) {
		bailout(Strings.format1("unsupported opcode: %s", opcode.name));
	}

	def visitProbe(orig_op: Opcode) {
		last_probe = it.pc;
		if (orig_op != Opcode.LOOP && orig_op != Opcode.END) emitProbe();
	}
	def emitProbe() {
		if (last_probe == 0) return;
		var probe = Instrumentation.getLocalProbe(module, func.func_index, last_probe);
		last_probe = 0;
		withReconstructedInlinedFrames(fun =>
			emitProbe0(it.pc, probe));
		if (Trace.compiler) traceOpcodeAndStack(true);
	}
	def emitProbe0(pc: int, probe: Probe) {
		// Check for intrinsified probes.
		var spillMode = this.probeSpillMode;
		match (probe) { // TODO: emit code for multiple intrinsified probes.
			null => ;
			x: MemoryReadProbe => if (SpcTuning.intrinsifyMemoryProbes && x.size <= 8) {
				intrinsified_read_probe = x;
				return;
			}
			x: CountProbe => if (SpcTuning.intrinsifyCountProbe) { // TODO: check for subclass override
				var tmp = allocTmp(ValueKind.REF);
				masm.emit_increment_CountProbe(tmp, x, 1);
				return;
			}
			x: CountMoreProbe => if (SpcTuning.intrinsifyCountProbe) { // TODO: check for subclass override
				var tmp = allocTmp(ValueKind.REF);
				masm.emit_increment_CountProbe(tmp, x.c, x.increment);
				return;
			}
			x: OperandProbe_i_v => if (SpcTuning.intrinsifyOperandProbe) {
				state.emitSaveAll(resolver, probeSpillMode);
				emit_compute_vsp(regs.scratch, state.sp);
				emit_spill_vsp(regs.scratch);
				masm.emit_store_curstack_vsp(regs.scratch);
				var value_reg = masm.getV3ParamReg(ValueKind.I32, 1);
				// XXX: Factor out loading slot into specific reg
				var sv = state.peek();
				if (sv.reg != value_reg) {
					if (sv.inReg()) {
						masm.emit_mov_r_r(sv.kind(), value_reg, sv.reg);
					} else if (sv.isConst()) {
						masm.emit_mov_r_k(sv.kind(), value_reg, sv.const);
					} else {
						masm.emit_mov_r_s(sv.kind(), value_reg, state.sp - 1);
					}
				}
				masm.emit_call_OperandProbe_i_v_fire(x, value_reg);
				emit_reload_regs();
				if (!probeSpillMode.free_regs) state.emitRestoreAll(resolver);
				return;
			}
			x: ExternalDebuggerBreakpointProbe => {
				masm.emit_debugger_breakpoint();
				return;
			}
			x: WhammProbe => if (SpcTuning.intrinsifyWhammProbe && WasmFunction.?(x.func)) {
				emitWhammProbe(x);
				return;
			}
			x: TracePointProbe => {
				// Tracepoints are used to help debug the JIT. Saving but not freeing
				// the registers has less effect on the surrounding code, making it
				// less likely to hide bugs.
				spillMode = SpillMode.SAVE_ONLY;
			}
		}
		// spill everything
		state.emitSaveAll(resolver, spillMode);
		// compute VSP for potential frame access
		emit_compute_vsp(regs.vsp, state.sp);
		emit_spill_vsp(regs.vsp);
		masm.emit_store_curstack_vsp(regs.vsp);
		// load stack
		masm.emit_get_curstack(regs.runtime_arg0);
		// store %sp
		masm.emit_v3_set_X86_64Stack_rsp_r_r(regs.runtime_arg0, regs.sp);
		masm.emit_push_X86_64Stack_rsp_r_r(regs.runtime_arg0);
		// reload WasmFunction
		masm.emit_mov_r_m(ValueKind.REF, regs.runtime_arg1, frame.wasm_func_slot);
		// load PC
		masm.emit_mov_r_i(regs.runtime_arg2, pc);
		// call runtime
		masm.emit_call_runtime_Probe_instr();
		// restore {stack.rsp}
		masm.emit_get_curstack(regs.scratch);
		masm.emit_pop_X86_64Stack_rsp_r_r(regs.scratch);
		emit_reload_regs();
		if (!spillMode.free_regs) state.emitRestoreAll(resolver);
	}

	// saves the overhead of using a runtime call by directly invoking the wasm function associated with the whamm probe
	def emitWhammProbe(probe: WhammProbe) {
		if (Trace.compiler) Trace.OUT.puts("emitting whamm probe\n");
		// set up args and push to frame slots.
		var whamm_sig = probe.sig;
		var orig_sp = state.sp;
		var callee_func = WasmFunction.!(probe.func);
		def inline_decision = shouldInline(callee_func.decl) && SpcTuning.inlineWhammProbes; // TODO move to shouldInline
		var swap_instance = false;
		var swap_membase = false;

		if (inline_decision) {
			probe.checkSwap();
			swap_instance = probe.swap_instance;
			swap_membase = probe.swap_membase;

			if (swap_instance) {
				masm.emit_mov_r_Instance(regs.scratch, callee_func.instance);
				masm.emit_mov_m_r(ValueKind.REF, frame.inlined_instance_slot, regs.scratch);
			}
			if (swap_membase) {
				if (callee_func.instance.memories.length > 0) {
					var membase = callee_func.instance.memories[0].getMemBase64();
					masm.emit_mov_r_l(regs.mem0_base, i64.view(membase));
				}
				masm.emit_mov_m_r(ValueKind.REF, frame.inlined_mem0_base_slot, regs.mem0_base);
			}
		} else {
			state.emitSaveAll(resolver, probeSpillMode);
		}

		for (i < whamm_sig.length) {
			var slot_tag_addr = masm.tagAddr(state.sp + u32.view(i));
			var slot_addr = masm.slotAddr(state.sp + u32.view(i));
			var kind: byte;
			match(whamm_sig[i]) {
				FrameAccessor => {
					if (inline_decision) state.emitSaveAll(resolver, probeSpillMode); // spill entire value stack.
					masm.emit_call_runtime_getFrameAccessorMetaRef();
					emit_reload_regs();
					if (inline_decision && !probeSpillMode.free_regs) state.emitRestoreAll(resolver);

					// move result to mem slot or reg, depending on inlining
					if (inline_decision) {
						var reg = allocRegTos(ValueKind.REF);
						masm.emit_mov_r_r(ValueKind.REF, reg, xenv.runtime_ret0);
						state.push(KIND_REF | IN_REG, reg, 0);
					} else {
						masm.emit_mov_m_r(ValueKind.REF, slot_addr, xenv.runtime_ret0);
					}
					kind = ValueKind.REF.code;
				}
				Val(val) => {
					match (val) {
						I31(v) => {
							if (inline_decision) {
								var reg = allocRegTos(ValueKind.REF);
								masm.emit_mov_r_i(reg, i32.view(v) << 1);
								state.push(KIND_REF | IN_REG, reg, 0);
							} else {
								masm.emit_mov_m_d(slot_addr, u64.view(v) << 1);
							}
							kind = ValueKind.REF.code;
						}
						I32(v) => {
							if (inline_decision) {
								state.push(KIND_I32 | IS_CONST, NO_REG, i32.view(v));
							} else {
								masm.emit_mov_m_d(slot_addr, v);
							}
							kind = ValueKind.I32.code;
						}
						I64(v) => {
							if (inline_decision) {
								var reg = allocRegTos(ValueKind.I64);
								masm.emit_mov_r_l(reg, i64.view(v));
								state.push(KIND_I64 | IN_REG, reg, 0);
							} else {
								masm.emit_mov_m_d(slot_addr, v);
							}
							kind = ValueKind.I64.code;
						}
						F32(v) => {
							if (inline_decision) {
								var reg = allocRegTos(ValueKind.F32);
								masm.emit_mov_r_f32(reg, v);
								state.push(KIND_F32 | IN_REG, reg, 0);
							} else {
								masm.emit_mov_m_d(slot_addr, v);
							}
							kind = ValueKind.F32.code;
						}
						F64(v) => {
							if (inline_decision) {
								var reg = allocRegTos(ValueKind.F64);
								masm.emit_mov_r_d64(reg, v);
								state.push(KIND_F64 | IN_REG, reg, 0);
							} else {
								masm.emit_mov_m_d(slot_addr, v);
							}
							kind = ValueKind.F64.code;
						}
						V128(l, h) => {
							if (inline_decision) {
								var reg = allocRegTos(ValueKind.V128);
								masm.emit_mov_r_q(reg, l, h);
								state.push(KIND_V128 | IN_REG, reg, 0);
							} else {
								masm.emit_mov_m_d(slot_addr, l);
								masm.emit_mov_m_d(slot_addr.plus(8), h);
							}
							kind = ValueKind.V128.code;
						}
						Ref(v) => {
							if (inline_decision) {
								var reg = allocRegTos(ValueKind.REF);
								masm.emit_mov_r_Object(reg, v);
								state.push(KIND_REF | IN_REG, reg, 0);
							} else {
								masm.emit_mov_r_Object(regs.scratch, v);
								masm.emit_mov_m_r(ValueKind.REF, slot_addr, regs.scratch);
							}
							kind = ValueKind.REF.code;
						}
						Cont(v) => {
							if (inline_decision) {
								var reg = allocRegTos(ValueKind.REF_U64);
								masm.emit_mov_r_Cont(reg, v);
								state.push(KIND_REF_U64 | IN_REG, reg, 0);
							} else {
								// TODO[stack-switching]: designated xmm scratch register
								// masm.emit_mov_r_Cont(regs.scratch, v);
								// asm.movdqu_m_s(slot_addr, regs.scratch)
							}
							kind = ValueKind.REF.code;
						}
					}
				}
				Operand(_, i) => {
					var index = orig_sp + u32.view(i) - 1;
					if (inline_decision) {
						visit_LOCAL_GET(u31.view(index - local_base_sp));
					} else {
						masm.emit_mov_m_m(state.state[index].kind(), slot_addr, masm.slotAddr(index));
					}
					kind = state.state[index].kind().code;
				}
				Local(_, i) => {
					if (inline_decision) {
						visit_LOCAL_GET(u31.view(i));
					} else {
						masm.emit_mov_m_m(state.state[u31.view(i)].kind(), slot_addr, masm.slotAddr(u32.view(i)));
					}
					kind = state.state[u31.view(i)].kind().code;
				}
				Null => System.error("whamm", "null whamm arg!");
			}
			if (!inline_decision) {
				masm.emit_mov_m_i(slot_tag_addr, kind);
			}
		}
		var whamm_instance = callee_func.instance;
		var func_id = callee_func.decl.func_index;
		var whamm_module = whamm_instance.module;
		var whamm_func_decl = callee_func.decl;
		if (inline_decision) {
			whamm_config = WhammInlineConfig(swap_membase, swap_instance, true);
			masm.emit_mov_m_r(ValueKind.REF, frame.vfp_slot, regs.vfp);
			emitInlinedCall(whamm_func_decl, probe);
			whamm_config = WhammInlineConfig(false, false, false);
			// Restore mem0_base after probe
			if (module.memories.length > 0) {
				masm.emit_mov_r_m(ValueKind.REF, regs.mem0_base, frame.mem0_base_slot);
			}
		} else {
			var vsp_reg = allocTmpFixed(ValueKind.REF, regs.vsp);
			var func_reg = allocTmpFixed(ValueKind.REF, regs.func_arg);
			var tmp = allocTmp(ValueKind.REF);

			// Load the target code/entrypoint.
			masm.emit_mov_r_Function(func_reg, whamm_instance.functions[func_id]);
			masm.emit_v3_WasmFunction_decl_r_r(tmp, func_reg);
			masm.emit_v3_FuncDecl_target_code_r_r(tmp, tmp);
			// adjust vsp_reg to compute the "true" VSP, accounting for args to WhammProbe's WasmFunction
			emit_compute_vsp(vsp_reg, state.sp + u32.view(whamm_sig.length));
			// Call to the entrypoint.
			masm.emit_call_r(tmp);
			emit_reload_regs();
			if (!probeSpillMode.free_regs) state.emitRestoreAll(resolver);
		}
	}

	def visit_CRASH_EXEC() {
		masm.emit_intentional_crash();
	}
	def visit_CRASH_COMPILER() {
		System.error("WizengError", "encountered crash-compiler opcode");
	}
	def visit_UNREACHABLE() {
		emitTrap(TrapReason.UNREACHABLE);
		setUnreachable();
	}
	def visit_NOP() {
		// emit nothing
	}
	def visit_BLOCK(btc: BlockTypeCode) {
		var pr = btc.toAbstractBlockType(module);
		state.pushBlock(pr.0, pr.1, masm.newLabel(it.pc));
	}
	def visit_LOOP(btc: BlockTypeCode) {
		var pr = btc.toAbstractBlockType(module);
		state.pushLoop(pr.0, pr.1, masm.newLabel(it.pc));
		var ctl_top = state.ctl_stack.peek();
		state.prepareLoop(resolver);
		masm.bindLabel(ctl_top.label);
		emitProbe();
		if (it.pc == osr_pc && !isInlined()) {
			osr_state = state.ctl_stack.peek().copyMerge();
			osr_loop_label = masm.newLabel(it.pc);
			masm.bindLabel(osr_loop_label);
		}
	}
	def visit_IF(btc: BlockTypeCode) {
		var pr = btc.toAbstractBlockType(module);
		var sv = pop();
		var ctl_top = state.pushIf(pr.0, pr.1, masm.newLabel(it.pc), masm.newLabel(it.pc));
		emitBrIf(sv, MasmBrCond.I32_ZERO, ctl_top.else_label, ctl_top, true, BrRepush.NONE);
	}
	def visit_ELSE() {
		var ctl_top = state.ctl_stack.peek();
		state.emitFallthru(resolver);
		masm.emit_br(ctl_top.label);
		masm.bindLabel(ctl_top.else_label);
		state.doElse();
		ctl_top.opcode = Opcode.ELSE.code;
		emitProbe();
	}
	def visit_TRY(btc: BlockTypeCode) {
		var pr = btc.toAbstractBlockType(module);
		state.pushBlock(pr.0, pr.1, masm.newLabel(it.pc));
		var ctl_top = state.ctl_stack.peek();
		ctl_top.catch_reset_state = Arrays.range(state.state, 0, int.view(state.sp));
	}
	def visit_THROW(tag_index: u31) {
		state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS);
		emit_compute_vsp(regs.vsp, state.sp);
		masm.emit_store_curstack_vsp(regs.vsp);
		masm.emit_get_curstack(regs.runtime_arg0);
		masm.emit_v3_set_X86_64Stack_rsp_r_r(regs.runtime_arg0, regs.sp);
		masm.emit_push_X86_64Stack_rsp_r_r(regs.runtime_arg0);
		emit_load_instance(regs.runtime_arg1);
		masm.emit_mov_r_i(regs.runtime_arg2, tag_index);
		masm.emit_call_runtime_op(Opcode.THROW);
		var tag = module.tags[tag_index];
		dropN(u32.!(tag.fields.length));
		setUnreachable();
	}
	def visit_THROW_REF() {
		state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS);
		emit_compute_vsp(regs.vsp, state.sp);
		masm.emit_store_curstack_vsp(regs.vsp);
		masm.emit_get_curstack(regs.runtime_arg0);
		masm.emit_v3_set_X86_64Stack_rsp_r_r(regs.runtime_arg0, regs.sp);
		masm.emit_push_X86_64Stack_rsp_r_r(regs.runtime_arg0);
		popFixedReg(regs.runtime_arg1);
		masm.emit_call_runtime_op(Opcode.THROW_REF);
		setUnreachable();
	}
	def visit_END() {
		var ctl_top = state.ctl_stack.peek();
		if (ctl_top.opcode == Opcode.LOOP.code) {
			state.ctl_stack.pop();
			if (!ctl_top.reachable) setUnreachable();
		} else if (ctl_top.opcode == Opcode.IF.code) {
			// simulate empty if-true block
			state.emitFallthru(resolver);
			masm.emit_br(ctl_top.label);
			masm.bindLabel(ctl_top.else_label);
			state.doElse();
			ctl_top.opcode = Opcode.ELSE.code;
			state.emitFallthru(resolver);
			masm.bindLabel(ctl_top.label);
			state.resetToMerge(ctl_top);
			state.ctl_stack.pop();
		} else if (ctl_top.opcode == Opcode.BLOCK.code || ctl_top.opcode == Opcode.ELSE.code) {
			state.emitFallthru(resolver);
			masm.bindLabel(ctl_top.label);
			state.resetToMerge(ctl_top);
			state.ctl_stack.pop();
		} else if (ctl_top.opcode == Opcode.RETURN.code) {
			state.emitFallthru(resolver);
			masm.bindLabel(ctl_top.label);
			state.resetToMerge(ctl_top);
			emitProbe();
			if (ctl_top.merge_count > 1) emitReturn(ctl_top);
			state.ctl_stack.pop();
			return;
		}
		emitProbe();
	}
	def visit_BR(depth: u31) {
		var target = state.getControl(depth);
		state.emitTransfer(target, resolver);
		masm.emit_br(target.label);
		setUnreachable();
	}
	def visit_BR_IF(depth: u31) {
		var target = state.getControl(depth);
		var sv = pop();
		emitBrIf(sv, MasmBrCond.I32_NONZERO, target.label, target, state.isTransferEmpty(target), BrRepush.NONE);
	}
	def visit_BR_ON_NULL(depth: u31) {
		var target = state.getControl(depth);
		var sv = pop();
		emitBrIf(sv, MasmBrCond.REF_NULL, target.label, target, state.isTransferEmpty(target), BrRepush.NOT_TAKEN);
	}
	def visit_BR_ON_NON_NULL(depth: u31) {
		var target = state.getControl(depth);
		var sv = pop();
		emitBrIf(sv, MasmBrCond.REF_NONNULL, target.label, target, state.isTransferEmpty(target), BrRepush.TAKEN);
	}
	def visit_BR_TABLE(depths: Range<u31>) {
		var sv = pop();
		emitBrTable(sv, depths);
		setUnreachable();
	}
	def visit_RETURN() {
		var target = state.ctl_stack.elems[ctl_base_sp];
		state.emitTransfer(target, resolver);
		masm.emit_br(ret_label);
		setUnreachable();
	}
	def visitCallDirect(op: Opcode, index: u31, tailCall: bool) {
		if (op == Opcode.CALL) {
			Metrics.spc_static_calls.val++;
			masm.emit_inc_metric(Metrics.spc_dynamic_calls);
		}
		var func = module.functions[index];

		// Try inlining for intra-module, non-tail calls
		if (!tailCall && shouldInline(func)) {
			if (Trace.compiler) Trace.OUT.put2("Inlining call to func #%d (%d bytes)", index, func.orig_bytecode.length).ln();
			if (op == Opcode.CALL) {
				Metrics.spc_static_inlined_calls.val++;
				masm.emit_inc_metric(Metrics.spc_dynamic_inlined_calls);
			}
			emitInlinedCall(func, null);
			return;
		}

		withReconstructedInlinedFrames(fun {
			var retpt = masm.newLabel(it.pc), wasmcall_label = masm.newLabel(it.pc);
			// Load the instance (which must happen before frame is unwound).
			var vsp_reg = allocTmpFixed(ValueKind.REF, regs.vsp);
			var func_reg = allocTmpFixed(ValueKind.REF, regs.func_arg);
			var tmp = allocTmp(ValueKind.REF);
			emit_load_instance(tmp);

			// Load the function, XXX: skip and compute function from instance + code on stack?
			masm.emit_v3_Instance_functions_r_r(func_reg, tmp);
			masm.emit_v3_Array_elem_r_ri(ValueKind.REF, func_reg, func_reg, func.func_index);

			emitCallToReg(func.sig, func_reg, vsp_reg, tmp, func.imp != null, tailCall);
		});
	}
	def emitInlinedCall(callee_func: FuncDecl, whamm: WhammProbe) {
		var sig = callee_func.sig;
		var params_count = u32.view(sig.params.length);
		var results_count = u32.view(sig.results.length);
		var orig_sp = state.sp;

		// Arguments are already on stack
		// Stack: [..., arg0, arg1, ..., argN] <- sp
		// We want callee's local 0 = arg0, so:
		var new_local_base_sp: u31 = u31.view(orig_sp - params_count);
		var new_ctl_base_sp = u31.view(state.ctl_stack.top);

		var num_locals = callee_func.num_slots();

		// Push a RETURN control for the inlined callee's function body.
		var end_label = masm.newLabel(callee_func.cur_bytecode.length);
		var func_body_ctl = state.pushFuncBody(sig.params, sig.results, end_label);

		var m: Module = module;

		// Whamm probe configuration
		if (whamm != null) {
			def whamm_sig = whamm.sig;
			def whamm_wf = WasmFunction.!(whamm.func);
			def whamm_instance = whamm_wf.instance;
			def whamm_func_decl = whamm_wf.decl;

			m = whamm_instance.module;
			new_local_base_sp = u31.view(state.sp) - u31.view(whamm_sig.length); // XXX
			func_body_ctl.val_stack_top = new_local_base_sp; // correct val_stack_top for whamm arg count 
		}

		// create merge state based on outer function's base sp given inlined function's results
		func_body_ctl.merge_state = state.getInMemoryMergeWithArgs(int.view(new_local_base_sp), sig.results);
		func_body_ctl.merge_count = 1;

		// Create and push frame for inlined function
		var callee_frame = SpcFrame.new(callee_func,
			m, new_local_base_sp, new_ctl_base_sp, num_locals, 0, masm.newLabel(callee_func.cur_bytecode.length));

		pushSpcFrame(callee_frame);

		// Emit function entry probe, if any.
		// XXX expensive because frame materialization required
		if (whamm == null && !FeatureDisable.entryProbes && func.entry_probed) {
			var probe = Instrumentation.getLocalProbe(module, callee_func.func_index, 0);
			withReconstructedInlinedFrames(fun =>
				emitProbe0(0, probe));
		}

		// Allocate callee's non-parameter locals
		it.dispatchLocalDecls(this);

		// Compile callee's bytecode
		if (Trace.compiler) Trace.OUT.puts("  Start inlined function body").ln();
		while (it.more() && success) {
			if (Trace.compiler) traceOpcodeAndStack(false);
			last_probe = 0;
			masm.source_loc = it.pc;
			masm.current_fid = func.func_index;
			it.dispatch(this);
			if (Trace.compiler && Trace.asm) {
				OUT.puts("JIT code: ");
				masm.printCodeBytes(OUT, codegen_offset, masm.curCodeBytes());
				codegen_offset = masm.curCodeBytes();
				OUT.ln();
			}
			unrefRegs();
			if (Debug.compiler) checkRegAlloc();
			it.next();
			if (skip_to_end) doSkipToEndOfBlock();
		}
		if (Trace.compiler) Trace.OUT.puts("  End inlined function body").ln();

		// Check if the inlined function is unreachable (e.g., ended with UNREACHABLE, RETURN, THROW)
		var inlined_reachable = state.ctl_stack.peek().reachable;

		// Restore caller context by popping frame
		popSpcFrame();  // Automatically restores cached fields

		// Note: Control stack cleanup (popping implicit BLOCK) is handled by visit_END

		// If inlined function is unreachable, no results to clean up
		if (!inlined_reachable) {
			if (Trace.compiler) {
				Trace.OUT.puts("  Inlined function unreachable, skipping result cleanup").ln();
				Trace.OUT.put3("    state.sp=%d, new_local_base_sp=%d, callee_slots=%d",
					state.sp, new_local_base_sp, state.sp - new_local_base_sp).ln();
			}
			// Drop all callee state (params + locals, no results)
			var callee_slots = state.sp - new_local_base_sp;
			if (callee_slots > 0) dropN(u32.view(callee_slots));
			if (Trace.compiler) Trace.OUT.put1("    After dropN: state.sp=%d", state.sp).ln();
			setUnreachable();
			return;
		}

		if (Trace.compiler) {
			Trace.OUT.put1("  Inlined call complete, sp=%d", state.sp).ln();
		}
	}
	def emitCallToReg(sig: SigDecl, func_reg: Reg, vsp_reg: Reg, tmp: Reg, checkHostCall: bool, tailCall: bool) {
		var retpt = masm.newLabel(it.pc), wasmcall_label = masm.newLabel(it.pc);
		// Handle the current stack state.
		if (tailCall) emitMoveTailCallArgs(sig); // transfer tail call args
		else state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS); // spill entire value stack

		// Compute the value stack pointer.
		emit_compute_vsp(vsp_reg, state.sp);
		if (checkHostCall) {
			// A call to imported function must first check for WasmFunction.
			masm.emit_br_r(func_reg, MasmBrCond.IS_WASM_FUNC, wasmcall_label);
			if (tailCall) {
				masm.emit_jump_HostCallStub(); // XXX: stub relies on func_arg and VSP
			} else {
				masm.emit_call_HostCallStub(); // XXX: stub relies on func_arg and VSP
				masm.emit_br(retpt);
			}
		}
		// Load the target code/entrypoint.
		masm.bindLabel(wasmcall_label);
		masm.emit_v3_WasmFunction_decl_r_r(tmp, func_reg);
		masm.emit_v3_FuncDecl_target_code_r_r(tmp, tmp);

		// Call or jump to the entrypoint.
		if (tailCall) {
			masm.emit_jump_r(tmp);
			setUnreachable();
		} else {
			masm.emit_call_r(tmp);
			masm.bindLabel(retpt);
			emit_reload_regs();
			state.popArgsAndPushResults(sig);
		}
	}
	def emitMoveTailCallArgs(sig: SigDecl) {
		var p = sig.params, count = u32.!(p.length);
		var base = state.sp - count;
		for (i < count) { // transfer args
			var tv = SpcVal(typeToKindFlags(p[i]) | IS_STORED | TAG_STORED, NO_REG, 0); // XXX: skip tag copy
			var fv = state.state[base + i];
			resolver.addMove((i, tv), (base + i, fv));
		}
		for (i < base) unrefSlot(i); // free all unused slots of frame below args
		resolver.emitMoves();
		state.sp = count;
		// adjust frame
		masm.emit_addw_r_i(regs.sp, frame.frameSize);
	}
	def visitCallIndirect(op: Opcode, sig_index: u31, table_index: u31, tailCall: bool) {
		var sig = SigDecl.!(module.heaptypes[sig_index]);
		var retpt = masm.newLabel(it.pc), wasmcall_label = masm.newLabel(it.pc);
		var vsp_reg = allocTmpFixed(ValueKind.REF, regs.vsp);
		var sv = popFixedReg(regs.func_arg);
		var func_reg = sv.reg;
		emit_load_instance(regs.instance);

		var table_reg = allocTmp(ValueKind.REF);
		var tmp_reg = allocTmp(ValueKind.REF);
		spillRegAndFree(regs.func_arg);

		// tmp1 = load table[table_index] from instance
		masm.emit_v3_Instance_tables_r_r(table_reg, regs.instance);
		masm.emit_v3_Array_elem_r_ri(ValueKind.REF, table_reg, table_reg, int.!(table_index));
		// bounds check
		masm.emit_v3_Table_ids_r_r(tmp_reg, table_reg);
		masm.emit_v3_Array_bounds_check_rr(tmp_reg, func_reg, newTrapLabel(TrapReason.FUNC_INVALID));
		// signature check
		masm.emit_v3_Array_elem_r_rr(ValueKind.I32, tmp_reg, tmp_reg, func_reg); // XXX: using ref reg as int
		masm.emit_breq_r_i(tmp_reg, -1, newTrapLabel(TrapReason.FUNC_INVALID));

		if (sig.final) {
			// Final signatures can do an exact signature check.
			masm.emit_brne_r_i(tmp_reg, sig.canon_id, newTrapLabel(TrapReason.FUNC_SIG_MISMATCH));
			// load from table
			masm.emit_v3_Table_funcs_r_r(tmp_reg, table_reg);
			masm.emit_v3_Array_elem_r_rr(ValueKind.REF, func_reg, tmp_reg, func_reg);
		} else {
			// Non-final signatures do a fast-path check and then fallback to a runtime call.
			var sigokpt = masm.newLabel(it.pc);
			masm.emit_breq_r_i(tmp_reg, sig.canon_id, sigokpt);
			state.emitSaveAll(resolver, SpillMode.SAVE_AND_REMEMBER_STORED); // XXX: no need to save all regs?
			var arg2 = regs.runtime_arg2;
			// load from table into runtime arg0
			masm.emit_v3_Table_funcs_r_r(arg2, table_reg);
			masm.emit_v3_Array_elem_r_rr(ValueKind.REF, arg2, arg2, func_reg);
			// call runtime
			emit_load_instance(regs.runtime_arg0);
			masm.emit_mov_r_i(regs.runtime_arg1, sig_index);
			masm.emit_call_runtime_checkFuncSigSubtyping(); // XXX: emit inline code for subtype check
			// result == null implies sig mismatch
			masm.emit_br_r(regs.runtime_ret0, MasmBrCond.REF_NULL, newTrapLabel(TrapReason.FUNC_SIG_MISMATCH));
			masm.emit_mov_r_r(ValueKind.REF, func_reg, regs.runtime_ret0);

			emit_reload_regs();
			state.emitRestoreAll(resolver);
			var end = masm.newLabel(it.pc);
			masm.emit_br(end);

			// load from table
			masm.bindLabel(sigokpt);
			masm.emit_v3_Table_funcs_r_r(tmp_reg, table_reg);
			masm.emit_v3_Array_elem_r_rr(ValueKind.REF, func_reg, tmp_reg, func_reg);
			masm.bindLabel(end);
		}

		emitCallToReg(sig, func_reg, vsp_reg, tmp_reg, true, tailCall);
	}
	def visitCallRef(op: Opcode, index: u31, tailCall: bool) {
		var sig = SigDecl.!(module.heaptypes[index]);
		var sv = state.peek();
		if (sv.isConst() && sv.const == 0) {
			emitTrap(TrapReason.NULL_DEREF);
			setUnreachable();
			return;
		}
		var vsp_reg = allocTmpFixed(ValueKind.REF, regs.vsp);
		sv = popFixedReg(regs.func_arg);
		var tmp = allocTmp(ValueKind.REF);
		var func_reg = sv.reg;

		emitCallToReg(sig, func_reg, vsp_reg, tmp, true, tailCall);
	}
	def visit_DROP() {
		dropN(1);
	}
	def visit_SELECT() {
		emitSelect(1);
	}
	def visit_SELECT_T(val_types: Range<ValueTypeCode>) {
		emitSelect(u32.!(val_types.length));
	}
	def visit_CATCH(tag_code: u31) {
		var tag = module.tags[tag_code];
		setupCatchBlock(Strings.format1("#%d", tag.tag_index), tag.fields);
	}
	def visit_CATCH_ALL() {
		setupCatchBlock("all", []);
	}
	private def setupCatchBlock(tag_display: string, tag_fields: Array<ValueType>) {
		var handler_id = getHandlerIndex(it.pc, 0);
		var dest_id = func.handlers.handler_dest_map[handler_id];
		var dest_label = masm.newLabel(it.pc);
		var ctl_top = state.ctl_stack.peek();
		// End the try or previous catch block with a xfer to the end.
		if (Trace.compiler) Trace.OUT.puts("    xfer to try end").ln();
		state.emitTransfer(ctl_top, resolver);
		masm.emit_br(ctl_top.label);
		// Load reset state from {try} block.
		var new_sp = int.view(ctl_top.val_stack_top) + tag_fields.length;
		var reset_state = Array<SpcVal>.new(new_sp);
		Arrays.copyInto(ctl_top.catch_reset_state, reset_state, 0);
		// Push tag fields onto stack.
		for (i < tag_fields.length) {
			var slot = int.view(ctl_top.val_stack_top) + i;
			reset_state[slot] = SpcVal(typeToKindFlags(tag_fields[i]) | TAG_STORED | IS_STORED, NO_REG, 0);
		}
		state.resetTo(u32.view(new_sp), reset_state);
		masm.bindLabel(dest_label);
		if (Trace.compiler) {
			Trace.OUT.put1("    catch tag[%s]", tag_display).ln();
			Trace.OUT.put2("    handler dest #%d recorded (handler=#%d)", dest_id, handler_id).ln();
		}
		handler_dest_info[dest_id] = SpcHandlerInfo(false, false, dest_label, masm.newLabel(it.pc), reset_state);
		// Reset reachability.
		ctl_top.reachable = true;
	}
	def visit_TRY_TABLE(btc: BlockTypeCode, catches: Range<BpCatchCode>) {
		// record reachable label positions for handler stub
		var pos = it.pc;
		for (i < catches.length) {
			var depth = catches[i].getLabel();
			var control = state.getControl(depth);
			buildHandlerDest(control, pos, i);
		}

		var pr = btc.toAbstractBlockType(module);
		state.pushBlock(pr.0, pr.1, masm.newLabel(it.pc));
	}
	// Given a control to xfer to, set up the {SpcHandlerInfo} and the merge.
	def buildHandlerDest(control: SpcControl, pc: int, inline_index: int) {
		if (Trace.compiler) Trace.OUT.put2("Building handler at pc=%d inline_index=%d", pc, inline_index).ln();
		var handler_id = getHandlerIndex(pc, inline_index);
		var dest_id = func.handlers.handler_dest_map[handler_id];
		var info = handler_dest_info[dest_id];

		// If target has not been reached via merge, populate with stack state which has everything
		// in memory. Otherwise, there will be a processed {merge_state} that we can use directly.
		if (control.merge_count == 0) {
			control.merge_state = state.getInMemoryMergeWithArgs(int.view(control.val_stack_top), labelArgs(control));
		}
		control.merge_count++;

		// {control != null} guarantees a non-dummy handler, in which case {dest_label == null} only when it's
		// not yet visited.
		if (info.dest_label == null) {
			if (Trace.compiler) {
				Trace.OUT.put2("    handler dest #%d recorded (handler=#%d)", dest_id, handler_id).ln();
			}
			handler_dest_info[dest_id] = SpcHandlerInfo(false, control.opcode == Opcode.RETURN.code, control.label, masm.newLabel(it.pc), control.merge_state);
		}
	}
	def buildSwitchHandler(pc: int, inline_index: int) {
		var handler_id = getHandlerIndex(pc, inline_index);
		var dest_id = func.handlers.handler_dest_map[handler_id];
		if (Trace.compiler) {
			Trace.OUT.put2("    dummy handler dest #%d recorded (handler=#%d)", dest_id, handler_id).ln();
		}
		handler_dest_info[dest_id] = SpcHandlerInfo(true, false, null, null, null);
	}
	def emitSelect(valcount: u32) {
		var sv = pop();
		var base: u32 = state.sp - valcount*2;
		if (sv.isConst()) {
			if (sv.const != 0) {
				// select a* b* T => a*
			} else {
				// select a* b* 0 => b*
				for (i < valcount) {
					var ti = base + i, fi = ti + valcount;
					var tv = state.state[ti], fv = state.state[fi];
					if (tv.sameValue(fv)) continue;
					unrefSlot(ti); // unref any register of dropped true values
					if (fv.inReg()) {
						regAlloc.reassign(fv.reg, int.!(fi), int.!(ti));
						state.state[ti] = SpcVal(tv.kindFlagsAndTag(fv.flags & (IN_REG | IS_CONST)), fv.reg, fv.const);
					} else {
						var tv = SpcVal(tv.kindFlagsAndTag(IS_STORED | (fv.flags & IS_CONST)), NO_REG, fv.const);
						resolver.addMove((ti, tv), (fi, fv));
						state.state[ti] = tv;
					}
				}
				resolver.emitMoves();
			}
			dropN(valcount);
			return;
		}
		// Compute moves for the true branch (happens before compare).
		for (i < valcount) {
			var ti = base + i;
			var fi = ti + valcount;
			var fv = state.state[fi];
			var tv = state.state[ti];
			if (tv.sameValue(fv)) {
				// no move necessary on either side
				state.state[fi] = tv;
				if (Trace.compiler) Trace.OUT.put1("    sp=%d: no move necessary", ti).ln();
			} else if (tv.reg != NO_REG && regAlloc.frequency(tv.reg) == 1) {
				// can reuse tv.reg
				if (Trace.compiler) Trace.OUT.put2("    sp=%d: reuse %s", ti, config.regSet.getName(tv.reg)).ln();
			} else {
				var reg = tryAllocReg(tv.kind(), ti);
				var prev_tv = tv;
				if (Trace.compiler) Trace.OUT.put3("    sp=%d: allocated %s, prev %s", ti, config.regSet.getName(reg), config.regSet.getName(prev_tv.reg)).ln();
				if (reg != NO_REG) {
					tv = SpcVal(tv.kindFlags(IN_REG), reg, 0);
					resolver.addMove((ti, tv), (ti, prev_tv));
				} else {
					tv = SpcVal(tv.kindFlags(IS_STORED), reg, 0);
					resolver.addMove((ti, tv), (ti, prev_tv));
				}
				tv = SpcVal(tv.flags | (prev_tv.flags & TAG_STORED), tv.reg, tv.const);
				if (prev_tv.reg != NO_REG) unrefLater(prev_tv.reg, int.!(ti));
			}
			state.state[ti] = tv;
		}
		resolver.emitMoves();
		var label = masm.newLabel(it.pc);
		var cond = MasmBrCond.I32_NONZERO;
		if (sv.inReg()) masm.emit_br_r(sv.reg, cond, label);
		else masm.emit_br_m(masm.slotAddr(state.sp), cond, label);
		// Compute moves for the false branch.
		for (i < valcount) {
			var ti = base + i;
			var fi = ti + valcount;
			var fv = state.state[fi];
			var tv = state.state[ti];
			resolver.addMove((ti, tv), (fi, fv));
		}
		resolver.emitMoves();
		masm.bindLabel(label);
		dropN(valcount);
	}
	def visit_LOCAL_GET(index: u31) {
		index = index + local_base_sp;
		var lv = state.get(index);
		if (lv.inReg()) {
			regAlloc.assign(lv.reg, int.!(state.sp));
			var isConst = lv.flags & IS_CONST;
			state.push(lv.kindFlags(isConst | IN_REG), lv.reg, lv.const);
		} else if (lv.isConst()) {
			state.push(lv.kindFlags(IS_CONST), NO_REG, lv.const);
		} else {
			var kind = lv.kind();
			var reg = allocRegTos(kind);
			masm.emit_mov_r_s(kind, reg, index);
			state.push(lv.kindFlags(IN_REG), reg, 0);
			state.state[index] = SpcVal(lv.flags | IN_REG, reg, 0);
			regAlloc.assign(reg, index);
		}
	}
	def visit_LOCAL_SET(index: u31) {
		index = index + local_base_sp;
		var lv = state.get(index);
		var sv = state.pop();
		if (sv.inReg()) {
			regAlloc.unassign(lv.reg, index); // unref existing register
			var isConst = sv.flags & IS_CONST;
			state.set(index, lv.kindFlagsAndTag(IN_REG | isConst), sv.reg, sv.const);
			regAlloc.reassign(sv.reg, int.!(state.sp), index);
		} else if (sv.isConst()) {
			regAlloc.unassign(lv.reg, index); // unref existing register
			state.set(index, lv.kindFlagsAndTag(IS_CONST), NO_REG, sv.const);
		} else {
			var kind = lv.kind();
			var reg: Reg;
			if (regAlloc.frequency(lv.reg) == 1) {
				reg = lv.reg; // used only once; can reuse
			} else {
				regAlloc.unassign(lv.reg, index); // unref existing register
				reg = allocReg(kind, index);           // and allocate a new one
			}
			masm.emit_mov_r_s(kind, reg, state.sp);
			state.set(index, lv.kindFlagsAndTag(IN_REG), reg, 0);
		}
	}
	def visit_LOCAL_TEE(index: u31) {
		index = index + local_base_sp;
		var lv = state.get(index);
		regAlloc.unassign(lv.reg, index); // unref existing register
		var sv = state.peek();
		if (sv.inReg()) {
			regAlloc.assign(sv.reg, index);
			var isConst = sv.flags & IS_CONST;
			state.set(index, lv.kindFlagsAndTag(IN_REG | isConst), sv.reg, sv.const);
		} else if (sv.isConst()) {
			state.set(index, lv.kindFlagsAndTag(IS_CONST), NO_REG, sv.const);
		} else {
			var tos = state.sp - 1;
			var kind = lv.kind();
			var reg = allocReg(kind, tos);
			regAlloc.assign(reg, index);
			masm.emit_mov_r_s(kind, reg, tos);
			state.set(index, lv.kindFlagsAndTag(IN_REG), reg, 0);
			state.overwrite(sv.kindFlagsAndTag(IN_REG | IS_STORED), reg, 0);
		}
	}
	def visit_GLOBAL_GET(index: u31) {
		var global = module.globals[index];
		if (!global.mutable && global.imp == null && InitExpr.I32.?(global.init)) {
			state.push(KIND_I32 | IS_CONST, NO_REG, InitExpr.I32.!(global.init).val);
			return;
		}
		var kind = ValueTypes.kind(global.valtype);
		if (SpcTuning.inlineGlobalAccess && kind != ValueKind.REF) {
			var tmp = allocTmp(ValueKind.REF);
			emit_load_instance(tmp);
			masm.emit_v3_Instance_globals_r_r(tmp, tmp);
			masm.emit_v3_Array_elem_r_ri(ValueKind.REF, tmp, tmp, int.!(index));
			var result = allocRegTos(kind);
			masm.emit_mov_r_m(kind, result, MasmAddr(tmp, masm.getOffsets().Global_low)); // TODO: unboxed value read
			state.push(SpcConsts.kindToFlags(kind) | IN_REG, result, 0);
		} else {
			emit_call_runtime_op1n(Opcode.GLOBAL_GET, index, 0, [global.valtype], false);
		}
	}
	def visit_GLOBAL_SET(index: u31) {
		var global = module.globals[index];
		var kind = ValueTypes.kind(global.valtype);
		if (SpcTuning.inlineGlobalAccess && kind != ValueKind.REF) {
			var tmp = allocTmp(ValueKind.REF);
			emit_load_instance(tmp);
			masm.emit_v3_Instance_globals_r_r(tmp, tmp);
			masm.emit_v3_Array_elem_r_ri(ValueKind.REF, tmp, tmp, int.!(index));
			var reg = popReg().reg; // XXX: improve global.set for constants
			masm.emit_mov_m_r(kind, MasmAddr(tmp, masm.getOffsets().Global_low), reg); // TODO: unboxed value write
		} else {
			emit_call_runtime_op1n(Opcode.GLOBAL_SET, index, 1, ValueTypes.NONE, false);
		}
	}
	def visit_TABLE_GET(index: u31) {
		var table = module.tables[index];
		emit_call_runtime_op1n(Opcode.TABLE_GET, index, 1, [table.elemtype], true);
	}
	def visit_TABLE_SET(index: u31) {
		var table = module.tables[index];
		emit_call_runtime_op1n(Opcode.TABLE_SET, index, 2, ValueTypes.NONE, true);
	}

	def visit_I32_LOAD(imm: MemArg) { emitLoad(ValueKind.I32, imm, masm.emit_load_r_r_r_i); }
	def visit_I64_LOAD(imm: MemArg) { emitLoad(ValueKind.I64, imm, masm.emit_load_r_r_r_i); }
	def visit_F32_LOAD(imm: MemArg) { emitLoad(ValueKind.F32, imm, masm.emit_load_r_r_r_i); }
	def visit_F64_LOAD(imm: MemArg) { emitLoad(ValueKind.F64, imm, masm.emit_load_r_r_r_i); }
	def visit_I32_LOAD8_S(imm: MemArg) { emitLoad(ValueKind.I32, imm, masm.emit_loadbsx_r_r_r_i); }
	def visit_I32_LOAD8_U(imm: MemArg) { emitLoad(ValueKind.I32, imm, masm.emit_loadbzx_r_r_r_i); }
	def visit_I32_LOAD16_S(imm: MemArg) { emitLoad(ValueKind.I32, imm, masm.emit_loadwsx_r_r_r_i); }
	def visit_I32_LOAD16_U(imm: MemArg) { emitLoad(ValueKind.I32, imm, masm.emit_loadwzx_r_r_r_i); }
	def visit_I64_LOAD8_S(imm: MemArg) { emitLoad(ValueKind.I64, imm, masm.emit_loadbsx_r_r_r_i); }
	def visit_I64_LOAD8_U(imm: MemArg) { emitLoad(ValueKind.I64, imm, masm.emit_loadbzx_r_r_r_i); }
	def visit_I64_LOAD16_S(imm: MemArg) { emitLoad(ValueKind.I64, imm, masm.emit_loadwsx_r_r_r_i); }
	def visit_I64_LOAD16_U(imm: MemArg) { emitLoad(ValueKind.I64, imm, masm.emit_loadwzx_r_r_r_i); }
	def visit_I64_LOAD32_S(imm: MemArg) { emitLoad(ValueKind.I64, imm, masm.emit_loaddsx_r_r_r_i); }
	def visit_I64_LOAD32_U(imm: MemArg) { emitLoad(ValueKind.I64, imm, masm.emit_loaddzx_r_r_r_i); }
	def visit_V128_LOAD(imm: MemArg) { emitLoad(ValueKind.V128, imm, masm.emit_load_r_r_r_i); }
	def visit_I32_STORE(imm: MemArg) { emitStore(ValueKind.I32, imm, masm.emit_store_r_r_r_i); }
	def visit_I64_STORE(imm: MemArg) { emitStore(ValueKind.I64, imm, masm.emit_store_r_r_r_i); }
	def visit_F32_STORE(imm: MemArg) { emitStore(ValueKind.F32, imm, masm.emit_store_r_r_r_i); }
	def visit_F64_STORE(imm: MemArg) { emitStore(ValueKind.F64, imm, masm.emit_store_r_r_r_i); }
	def visit_I32_STORE8(imm: MemArg) { emitStore(ValueKind.I32, imm, masm.emit_storeb_r_r_r_i); }
	def visit_I32_STORE16(imm: MemArg) { emitStore(ValueKind.I32, imm, masm.emit_storew_r_r_r_i); }
	def visit_I64_STORE8(imm: MemArg) { emitStore(ValueKind.I64, imm, masm.emit_storeb_r_r_r_i); }
	def visit_I64_STORE16(imm: MemArg) { emitStore(ValueKind.I64, imm, masm.emit_storew_r_r_r_i); }
	def visit_I64_STORE32(imm: MemArg) { emitStore(ValueKind.I32, imm, masm.emit_store_r_r_r_i); }
	def visit_V128_STORE(imm: MemArg) { emitStore(ValueKind.V128, imm, masm.emit_store_r_r_r_i); }

	def visit_MEMORY_SIZE(memory_index: u31) {
		var reg = allocTmp(ValueKind.REF);
		var r1 = allocRegTos(ValueKind.I64);
		var r2 = allocTmp(ValueKind.I64);
		emit_load_instance(reg);
		masm.emit_v3_Instance_memories_r_r(reg, reg);
		masm.emit_v3_Array_elem_r_ri(ValueKind.REF, reg, reg, int.!(memory_index));
		masm.emit_v3_Memory_start_r_r(r2, reg);
		masm.emit_v3_Memory_limit_r_r(r1, reg);
		def is64 = module.memories[memory_index].size.is64;
		masm.emit_subw_r_r(r1, r2);
		masm.emit_shrw_r_i(r1, 16); // TODO(custom-page-sizes): load the log2 of pagesize from decl
		state.push(if(is64, KIND_I64, KIND_I32) | IN_REG, r1, 0);
	}
	def visit_MEMORY_GROW(memory_index: u31) {
		def is64 = module.memories[memory_index].size.is64;
		emit_call_runtime_op1n(Opcode.MEMORY_GROW,memory_index, 1, [if(is64, ValueType.I64 ,ValueType.I32)], false);
	}

	def visit_I32_CONST(val: i32) {
		state.push(KIND_I32 | IS_CONST, NO_REG, val);
	}
	def visit_I64_CONST(val: i64) {
		if (i32.view(val) == val) {
			state.push(KIND_I64 | IS_CONST, NO_REG, i32.view(val));
		} else {
			var reg = allocRegTos(ValueKind.I64);
			masm.emit_mov_r_l(reg, val);
			state.push(KIND_I64 | IN_REG, reg, 0);
		}
	}
	def visit_F32_CONST(val: u32) {
		if (SpcTuning.trackFloat32Const || (val == 0 && SpcTuning.trackFloatZeroConst)) {
			state.push(KIND_F32 | IS_CONST, NO_REG, int.view(val));
		} else {
			var reg = allocRegTos(ValueKind.F32);
			masm.emit_mov_r_f32(reg, val);
			state.push(KIND_F32 | IN_REG, reg, 0);
		}
	}
	def visit_F64_CONST(val: u64) {
		if (val == 0 && SpcTuning.trackFloatZeroConst) {
			state.push(KIND_F64 | IS_CONST, NO_REG, 0);
		} else {
			var tos = state.sp;
			var reg = allocRegTos(ValueKind.F64);
			masm.emit_mov_r_d64(reg, val);
			state.push(KIND_F64 | IN_REG, reg, 0);
		}
	}
	def visit_V128_CONST(low: u64, high: u64) {
		var reg = allocRegTos(ValueKind.V128);
		masm.emit_mov_r_q(reg, low, high);
		state.push(KIND_V128 | IN_REG, reg, 0);
	}

	// In most cases this does not need to be specialized for REF_U64 due to
	// constant folding, and in those cases assigning a REF tag to a REF_U64
	// still elicit correct behavior. However when performing an in-assembly
	// value transfer to another stack without going through the runtime,
	// constant values must be tagged with the right type as the tag will be
	// copied over onto the other stack.
	def visit_REF_NULL(ht_val: int) {
		if (ht_val >= 0 && ContDecl.?(module.heaptypes[ht_val])) {
			state.push(KIND_REF_U64 | IS_CONST, NO_REG, 0); // push a REF_U64
		} else {
			state.push(KIND_REF | IS_CONST, NO_REG, 0); // push a REF
		}
	}
	def visit_REF_FUNC(func_index: u31) {
		var reg = allocRegTos(ValueKind.REF);
		emit_load_instance(reg);
		// XXX: skip loading the target function for direct intra-module calls?
		masm.emit_v3_Instance_functions_r_r(reg, reg);
		masm.emit_v3_Array_elem_r_ri(ValueKind.REF, reg, reg, int.!(func_index));
		state.push(KIND_REF | IN_REG, reg, 0);
	}
	def visit_REF_AS_NON_NULL() {
		var sv = state.peek();
		if (sv.isConst()) {
			if (sv.const == 0) {
				emitTrap(TrapReason.NULL_DEREF); // statically null
				setUnreachable();
			}
		} else if (sv.inReg()) {
			masm.emit_br_r(sv.reg, MasmBrCond.REF_NULL, newTrapLabel(TrapReason.NULL_DEREF));
		} else {
			// XXX: use masm.br_m if necessary?
			var tos = state.sp - 1;
			var reg = allocReg(ValueKind.REF, tos);
			masm.emit_mov_r_s(ValueKind.REF, reg, tos);
			masm.emit_br_r(reg, MasmBrCond.REF_NULL, newTrapLabel(TrapReason.NULL_DEREF));
			state.overwrite(sv.kindFlagsAndTag(IN_REG | (sv.flags & IS_STORED)), reg, 0);
		}
	}
	// ext: stack-switching
	def visit_CONT_NEW(cont_index: u31) {
		var decl = ContDecl.!(module.heaptypes[cont_index]);
		emit_call_runtime_op1n(Opcode.CONT_NEW, cont_index, 1, ValueTypes.ONE_CONTREF_TYPE, true);
	}
	def visit_CONT_BIND(in_cont_id: u31, out_cont_id: u31) {
		var in_cont = ContDecl.!(module.heaptypes[in_cont_id]);
		var out_cont = ContDecl.!(module.heaptypes[out_cont_id]);
		var n_binds = u32.!(in_cont.sig.params.length - out_cont.sig.params.length);
		emit_call_runtime_op2n(Opcode.CONT_BIND, in_cont_id, out_cont_id, n_binds + 1, ValueTypes.ONE_CONTREF_TYPE, true);
	}
	def visit_RESUME(cont_id: u31, handlers: Range<SuspensionHandler>) {
		var pos = it.pc;
		// record reachable label positions for handler stub
		for (i < handlers.length) {
			var handler = handlers[i];
			match (handler) {
				Suspend(tag_index, depth) => buildHandlerDest(state.getControl(depth), pos, i);
				Switch(tag_index) => buildSwitchHandler(pos, i);
			}
		}

		var stub_resume = masm.newLabel(it.pc), end = masm.newLabel(it.pc);
		var cont_decl = ContDecl.!(module.heaptypes[cont_id]);
		if (checkForConstNull(state.peek())) return;

		var vsp = allocTmpFixed(ValueKind.REF, regs.vsp);
		var top = popReg();
		var cont = allocTmp(ValueKind.REF_U64);
		var contStack = allocTmp(ValueKind.REF);
		masm.emit_mov_r_r(ValueKind.REF_U64, cont, top.reg);
		masm.emit_validate_and_consume_cont(contStack, cont);

		// transfer params to child stack
		// XXX: not necessary to store the entire value stack to memory
		state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS);
		var curStack = allocTmp(ValueKind.REF);
		var nvals = allocTmp(ValueKind.REF);
		var tmp0 = allocTmp(ValueKind.REF);
		var tmp1 = allocTmp(ValueKind.REF);
		var tmp2 = allocTmp(ValueKind.V128);

		// call to transfer
		emit_compute_vsp(regs.vsp, state.sp);
		masm.emit_mov_r_i(nvals, cont_decl.sig.params.length);
		masm.emit_cont_mv(vsp, contStack, nvals, tmp0, tmp1, tmp2);
		dropN(u32.!(cont_decl.sig.params.length));
		state.emitRestoreAll(resolver);

		// context switch: store to old stack
		masm.emit_get_curstack(curStack);
		emit_compute_vsp(regs.vsp, state.sp);
		emit_spill_vsp(xenv.vsp);
		masm.emit_v3_set_X86_64Stack_vsp_r_r(curStack, xenv.vsp);
		masm.emit_v3_set_X86_64Stack_rsp_r_r(curStack, xenv.sp);

		// reserve space for call addr
		masm.emit_push_X86_64Stack_rsp_r_r(curStack);
		masm.emit_chain_cont_to_parent(curStack, contStack);

		masm.emit_call_label(stub_resume);
		/* === CHILD STACK RUNNING === */

		// clean up after stack switch
		state.pushResults(cont_decl.sig.results);
		emit_reload_regs();
		state.emitRestoreAll(resolver);

		// skip {stub_resume}
		masm.emit_br(end);
		masm.bindLabel(stub_resume);
		masm.emit_switch_to_stack(contStack);
		masm.bindLabel(end);
	}
	def visit_SUSPEND(tag_index: u31) {
		var stub_suspend = masm.newLabel(it.pc), end = masm.newLabel(it.pc);

		state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS);

		emit_compute_vsp(regs.vsp, state.sp);
		emit_spill_vsp(xenv.vsp);
		masm.emit_store_curstack_vsp(regs.vsp);
		masm.emit_get_curstack(regs.runtime_arg0);
		masm.emit_v3_set_X86_64Stack_rsp_r_r(regs.runtime_arg0, regs.sp);
		masm.emit_push_X86_64Stack_rsp_r_r(regs.runtime_arg0);
		emit_load_instance(regs.runtime_arg1);
		masm.emit_mov_r_i(regs.runtime_arg2, tag_index);
		masm.emit_call_runtime_SUSPEND();

		var sig_index = module.tags[tag_index].sig_index;
		state.popArgsAndPushResults(SigDecl.!(module.heaptypes[sig_index]));
		masm.emit_call_label(stub_suspend);

		/* === PARENT STACK RUNNING === */

		emit_reload_regs();
		state.emitRestoreAll(resolver);

		// skip {stub_suspend}
		masm.emit_br(end);
		masm.bindLabel(stub_suspend); {
			// load new stack pointers
			masm.emit_get_curstack(regs.scratch);
			// unlike fast-int, the ptr offset for RT calls is done outside of the `call_runtime_...`
			// shorthands, so manually popping %rsp is needed
			masm.emit_pop_X86_64Stack_rsp_r_r(regs.scratch);
			// pop and go to handler stack's stub
			masm.emit_v3_X86_64Stack_rsp_r_r(regs.sp, regs.scratch);
			masm.emit_pop_X86_64Stack_rsp_r_r(regs.scratch);
			masm.emit_pop_r(ValueKind.REF, xenv.scratch);
			masm.emit_jump_r(xenv.scratch);
		}
		masm.bindLabel(end);
	}
	def visit_SWITCH(target_cont_idx: u31, tag_index: u31) {
		var stub_switch = masm.newLabel(it.pc), end = masm.newLabel(it.pc);

		state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS);

		emit_compute_vsp(regs.vsp, state.sp);
		emit_spill_vsp(xenv.vsp);
		masm.emit_store_curstack_vsp(regs.vsp);
		masm.emit_get_curstack(regs.runtime_arg0);
		masm.emit_v3_set_X86_64Stack_rsp_r_r(regs.runtime_arg0, regs.sp);
		masm.emit_push_X86_64Stack_rsp_r_r(regs.runtime_arg0);
		emit_load_instance(regs.runtime_arg1);
		masm.emit_mov_r_i(regs.runtime_arg2, target_cont_idx);
		masm.emit_mov_r_i(regs.runtime_arg3, tag_index);
		masm.emit_call_runtime_SWITCH();

		// unraveling all relevant signatures
		var target_cont_decl = ContDecl.!(module.heaptypes[target_cont_idx]);
		var target_top_val = target_cont_decl.sig.params[target_cont_decl.sig.params.length - 1];
		var this_cont_decl = ContDecl.!(HeapType.Cont.!(ValueType.Ref.!(target_top_val).heap).cont);

		state.pop(); // target_cont
		state.sp -= u32.view(target_cont_decl.sig.params.length - 1); // params to target_cont
		state.pushResults(this_cont_decl.sig.params); // params to this_cont
		masm.emit_call_label(stub_switch);

		// /* === SUSPENDED === */

		emit_reload_regs();
		state.emitRestoreAll(resolver);

		// skip {stub_switch}
		masm.emit_br(end);
		masm.bindLabel(stub_switch); {
			// load new stack pointers
			masm.emit_get_curstack(regs.scratch);
			// unlike fast-int, the ptr offset for RT calls is done outside of the `call_runtime_...`
			// shorthands, so manually popping %rsp is needed
			masm.emit_pop_X86_64Stack_rsp_r_r(regs.scratch);
			// pop and go to handler stack's stub
			masm.emit_v3_X86_64Stack_rsp_r_r(regs.sp, regs.scratch);
			masm.emit_pop_X86_64Stack_rsp_r_r(regs.scratch);
			masm.emit_pop_r(ValueKind.REF, xenv.scratch);
			masm.emit_jump_r(xenv.scratch);
		}
		masm.bindLabel(end);
	}
	def visit_RESUME_THROW(cont_id: u31, tag_id: u31, handlers: Range<SuspensionHandler>) {
		var pos = it.pc;
		// record reachable label positions for handler stub
		for (i < handlers.length) {
			var handler = handlers[i];
			match (handler) {
				Suspend(tag_index, depth) => buildHandlerDest(state.getControl(depth), pos, i);
				Switch(tag_index) => buildSwitchHandler(pos, i);
			}
		}

		var stub_resume = masm.newLabel(it.pc), end = masm.newLabel(it.pc);
		var cont_decl = ContDecl.!(module.heaptypes[cont_id]);
		var tag = module.tags[tag_id];
		if (checkForConstNull(state.peek())) return;

		var vsp = allocTmpFixed(ValueKind.REF, regs.vsp);
		var curStack = allocTmp(ValueKind.REF);

		var cont = allocTmp(ValueKind.REF_U64);
		var contStack = allocTmp(ValueKind.REF);
		masm.emit_mov_r_r(ValueKind.REF_U64, cont, popReg().reg);
		masm.emit_validate_and_consume_cont(contStack, cont);

		state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS);
		emit_compute_vsp(regs.vsp, state.sp);
		emit_spill_vsp(xenv.vsp);
		masm.emit_store_curstack_vsp(regs.vsp);
		masm.emit_get_curstack(regs.runtime_arg0);
		masm.emit_v3_set_X86_64Stack_rsp_r_r(regs.runtime_arg0, regs.sp);
		masm.emit_push_X86_64Stack_rsp_r_r(regs.runtime_arg0);
		emit_load_instance(regs.runtime_arg1);
		masm.emit_mov_r_r(ValueKind.REF, regs.runtime_arg2, contStack);
		masm.emit_mov_r_i(regs.runtime_arg3, tag_id);
		masm.emit_call_runtime_RESUME_THROW();
		dropN(u32.!(tag.fields.length));
		// {RT.runtime_handle_resume_throw} returns a {Continuation} by pushing onto the stack
		// XXX: make RT function return {Continuation} directly
		state.push(KIND_REF_U64 | TAG_STORED | IS_STORED, NO_REG, 0);

		/* === only reachable when continuation handles the error === */
		emit_reload_regs();
		state.emitRestoreAll(resolver);
		masm.emit_mov_r_r(ValueKind.REF_U64, cont, popReg().reg);
		// contStack = cont.cont
		masm.emit_ref_from_ref_u64(contStack, cont);
		// contStack = [cont == contObj].stack (since cont.cont is the Continuation object)
		if (FeatureDisable.unboxedConts) masm.emit_v3_Continuation_stack_r_r(contStack, contStack);

		// save %vsp to frame
		emit_compute_vsp(regs.vsp, state.sp);
		emit_spill_vsp(xenv.vsp);

		// chain {contStack} to {curStack}, then set {m_curStack} to {cont.top}.
		masm.emit_get_curstack(curStack);
		masm.emit_v3_set_X86_64Stack_vsp_r_r(curStack, xenv.vsp);
		masm.emit_v3_set_X86_64Stack_rsp_r_r(curStack, xenv.sp);

		// reserve space for call addr
		masm.emit_push_X86_64Stack_rsp_r_r(curStack);
		masm.emit_chain_cont_to_parent(curStack, contStack);

		masm.emit_call_label(stub_resume);
		/* === CHILD STACK RUNNING === */

		// clean up after stack switch
		state.pushResults(cont_decl.sig.results);
		emit_reload_regs();
		state.emitRestoreAll(resolver);

		// skip {stub_resume}
		masm.emit_br(end);
		masm.bindLabel(stub_resume);
		masm.emit_switch_to_stack(contStack);
		masm.bindLabel(end);
	}
	def visit_RESUME_THROW_REF(cont_id: u31, handlers: Range<SuspensionHandler>) {
		var pos = it.pc;
		// record reachable label positions for handler stub
		for (i < handlers.length) {
			var handler = handlers[i];
			match (handler) {
				Suspend(tag_index, depth) => buildHandlerDest(state.getControl(depth), pos, i);
				Switch(tag_index) => buildSwitchHandler(pos, i);
			}
		}

		var stub_resume = masm.newLabel(it.pc), end = masm.newLabel(it.pc);
		var cont_decl = ContDecl.!(module.heaptypes[cont_id]);
		if (checkForConstNull(state.peek())) return;

		var vsp = allocTmpFixed(ValueKind.REF, regs.vsp);
		var curStack = allocTmp(ValueKind.REF);

		var cont = allocTmp(ValueKind.REF_U64);
		var contStack = allocTmp(ValueKind.REF);
		masm.emit_mov_r_r(ValueKind.REF_U64, cont, popReg().reg);
		masm.emit_validate_and_consume_cont(contStack, cont);

		popFixedReg(regs.runtime_arg3);
		state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS);
		emit_compute_vsp(regs.vsp, state.sp);
		emit_spill_vsp(xenv.vsp);
		masm.emit_store_curstack_vsp(regs.vsp);
		masm.emit_get_curstack(regs.runtime_arg0);
		masm.emit_v3_set_X86_64Stack_rsp_r_r(regs.runtime_arg0, regs.sp);
		masm.emit_push_X86_64Stack_rsp_r_r(regs.runtime_arg0);
		emit_load_instance(regs.runtime_arg1);
		masm.emit_mov_r_r(ValueKind.REF, regs.runtime_arg2, contStack);
		masm.emit_call_runtime_RESUME_THROW_REF();
		// {RT.runtime_handle_resume_throw} returns a {Continuation} by pushing onto the stack
		// XXX: make RT function return {Continuation} directly
		state.push(KIND_REF | TAG_STORED | IS_STORED, NO_REG, 0);

		/* === only reachable when continuation handles the error === */
		emit_reload_regs();
		state.emitRestoreAll(resolver);
		masm.emit_mov_r_r(ValueKind.REF_U64, cont, popReg().reg);
		// contStack = cont.cont
		masm.emit_ref_from_ref_u64(contStack, cont);
		// contStack = [cont == contObj].stack (since cont.cont is the Continuation object)
		if (FeatureDisable.unboxedConts) masm.emit_v3_Continuation_stack_r_r(contStack, contStack);

		// save %vsp to frame
		emit_compute_vsp(regs.vsp, state.sp);
		emit_spill_vsp(xenv.vsp);

		// chain {contStack} to {curStack}, then set {m_curStack} to {cont.top}.
		masm.emit_get_curstack(curStack);
		masm.emit_v3_set_X86_64Stack_vsp_r_r(curStack, xenv.vsp);
		masm.emit_v3_set_X86_64Stack_rsp_r_r(curStack, xenv.sp);

		// reserve space for call addr
		masm.emit_push_X86_64Stack_rsp_r_r(curStack);
		masm.emit_chain_cont_to_parent(curStack, contStack);

		masm.emit_call_label(stub_resume);
		/* === CHILD STACK RUNNING === */

		// clean up after stack switch
		state.pushResults(cont_decl.sig.results);
		emit_reload_regs();
		state.emitRestoreAll(resolver);

		// skip {stub_resume}
		masm.emit_br(end);
		masm.bindLabel(stub_resume);
		masm.emit_switch_to_stack(contStack);
		masm.bindLabel(end);
	}
	private def checkForConstNull(sv: SpcVal) -> bool {
		if (sv.isConst() && sv.const == 0) {
			emitTrap(TrapReason.NULL_DEREF);
			setUnreachable();
			return true;
		}
		return false;
	}
	def visit_STRUCT_NEW(struct_index: u31) {
		var decl = StructDecl.!(module.heaptypes[struct_index]);
		emit_call_runtime_op1n(Opcode.STRUCT_NEW, struct_index, u32.!(decl.field_types.length), ValueTypes.ONE_STRUCTREF_TYPE, true);
	}
	def visit_STRUCT_NEW_DEFAULT(struct_index: u31) {
		var decl = StructDecl.!(module.heaptypes[struct_index]);
		emit_call_runtime_op1n(Opcode.STRUCT_NEW_DEFAULT, struct_index, 0, ValueTypes.ONE_STRUCTREF_TYPE, true);
	}
	def visit_STRUCT_GET(struct_index: u31, field_index: u31) {
		var decl = StructDecl.!(module.heaptypes[struct_index]);
		emit_call_runtime_op2n(Opcode.STRUCT_GET, struct_index, field_index, 1, [decl.field_types[field_index].valtype], true);
	}
	def visit_STRUCT_GET_S(struct_index: u31, field_index: u31) {
		var decl = StructDecl.!(module.heaptypes[struct_index]);
		emit_call_runtime_op2n(Opcode.STRUCT_GET_S, struct_index, field_index, 1, [decl.field_types[field_index].valtype], true);
	}
	def visit_STRUCT_GET_U(struct_index: u31, field_index: u31) {
		var decl = StructDecl.!(module.heaptypes[struct_index]);
		emit_call_runtime_op2n(Opcode.STRUCT_GET_U, struct_index, field_index, 1, [decl.field_types[field_index].valtype], true);
	}
	def visit_STRUCT_SET(struct_index: u31, field_index: u31) {
		var decl = StructDecl.!(module.heaptypes[struct_index]);
		emit_call_runtime_op2n(Opcode.STRUCT_SET, struct_index, field_index, 2, ValueTypes.NONE, true);
	}
	def visit_ARRAY_NEW(ht_index: u31) {
		emit_call_runtime_op1n(Opcode.ARRAY_NEW, ht_index, 2, ValueTypes.ONE_ARRAYREF_TYPE, true);
	}
	def visit_ARRAY_NEW_DEFAULT(ht_index: u31) {
		emit_call_runtime_op1n(Opcode.ARRAY_NEW_DEFAULT, ht_index, 1, ValueTypes.ONE_ARRAYREF_TYPE, true);
	}
	def visit_ARRAY_NEW_FIXED(ht_index: u31, length: u31) {
		emit_call_runtime_op2n(Opcode.ARRAY_NEW_FIXED, ht_index, length, length, ValueTypes.ONE_ARRAYREF_TYPE, true);
	}
	def visit_ARRAY_NEW_DATA(ht_index: u31, data_index: u31) {
		emit_call_runtime_op2n(Opcode.ARRAY_NEW_DATA, ht_index, data_index, 2, ValueTypes.ONE_ARRAYREF_TYPE, true);
	}
	def visit_ARRAY_NEW_ELEM(ht_index: u31, elem_index: u31) {
		emit_call_runtime_op2n(Opcode.ARRAY_NEW_ELEM, ht_index, elem_index, 2, ValueTypes.ONE_ARRAYREF_TYPE, true);
	}
	def visit_ARRAY_GET(ht_index: u31) {
		var decl = ArrayDecl.!(module.heaptypes[ht_index]);
		emit_call_runtime_op1n(Opcode.ARRAY_GET, ht_index, 2, [decl.elem_types[0].valtype], true);
	}
	def visit_ARRAY_GET_S(ht_index: u31) {
		var decl = ArrayDecl.!(module.heaptypes[ht_index]);
		emit_call_runtime_op1n(Opcode.ARRAY_GET_S, ht_index, 2, [decl.elem_types[0].valtype], true);
	}
	def visit_ARRAY_GET_U(ht_index: u31) {
		var decl = ArrayDecl.!(module.heaptypes[ht_index]);
		emit_call_runtime_op1n(Opcode.ARRAY_GET_U, ht_index, 2, [decl.elem_types[0].valtype], true);
	}
	def visit_ARRAY_SET(ht_index: u31) {
		emit_call_runtime_op1n(Opcode.ARRAY_SET, ht_index, 3, ValueTypes.NONE, true);
	}
	def visit_ARRAY_LEN() {
		var sv = popReg();
		var label = masm.newTrapLabel(TrapReason.NULL_DEREF); // XXX: constant-fold nullcheck
		masm.emit_breq_r_l(sv.reg, 0, label);
		var tmp = allocTmp(ValueKind.REF);
		var dst = allocRegTos(ValueKind.I32);
		masm.emit_v3_HeapArray_vals_r_r(tmp, sv.reg);
		masm.emit_v3_Array_length_r_r(dst, tmp);
		state.push(KIND_I32 | IN_REG, dst, 0);
	}
	def visit_ARRAY_FILL(ht_index: u31) {
		emit_call_runtime_op1n(Opcode.ARRAY_FILL, ht_index, 4, ValueTypes.NONE, true);
	}
	def visit_ARRAY_COPY(ht_index1: u31, ht_index2: u31) {
		emit_call_runtime_op2n(Opcode.ARRAY_COPY, ht_index1, ht_index2, 5, ValueTypes.NONE, true);
	}
	def visit_ARRAY_INIT_DATA(ht_index: u31, data_index: u31) {
		emit_call_runtime_op2n(Opcode.ARRAY_INIT_DATA, ht_index, data_index, 4, ValueTypes.NONE, true);
	}
	def visit_ARRAY_INIT_ELEM(ht_index: u31, elem_index: u31) {
		emit_call_runtime_op2n(Opcode.ARRAY_INIT_ELEM, ht_index, elem_index, 4, ValueTypes.NONE, true);
	}
	def emit_call_runtime_doCast(nullable: byte, ht_val: int) -> Reg {
		// XXX: specialize to abstract heap types like ANY
		// XXX: test for null and i31 inline
		// XXX: statically eliminate null and i31 if possible
		var result = allocTmpFixed(ValueKind.REF, regs.runtime_ret0);
		state.emitSaveAll(resolver, runtimeSpillMode);
		emit_compute_vsp(regs.vsp, state.sp);
		masm.emit_store_curstack_vsp(regs.vsp);
		// XXX: if a RT function cannot trap, maybe skip setting {rsp}?
		masm.emit_get_curstack(regs.runtime_arg0);
		masm.emit_v3_set_X86_64Stack_rsp_r_r(regs.runtime_arg0, regs.sp);
		masm.emit_push_X86_64Stack_rsp_r_r(regs.runtime_arg0);
		emit_load_instance(regs.runtime_arg1);
		masm.emit_mov_r_i(regs.runtime_arg2, nullable);
		masm.emit_mov_r_i(regs.runtime_arg3, ht_val);
		masm.emit_call_runtime_cast();
		masm.emit_get_curstack(regs.runtime_arg0);
		masm.emit_pop_X86_64Stack_rsp_r_r(regs.runtime_arg0);
		emit_reload_regs();
		if (!runtimeSpillMode.free_regs) state.emitRestoreAll(resolver);
		return result;
	}
	def visit_REF_TEST(ht_val: int) {
		var result = emit_call_runtime_doCast(0, ht_val);
		masm.emit_binop_r_i(Opcode.I32_AND, result, 1); // XXX: eliminate masking of Virgil bool
		allocFixed(ValueKind.I32, result, state.sp - 1); // use Runtime return reg
		state.overwrite(KIND_I32 | IN_REG, result, 0);
	}
	def visit_REF_TEST_NULL	(ht_val: int) {
		var result = emit_call_runtime_doCast(2, ht_val);
		masm.emit_binop_r_i(Opcode.I32_AND, result, 1); // XXX: eliminate masking of Virgil bool
		allocFixed(ValueKind.I32, result, state.sp - 1); // use Runtime return reg
		state.overwrite(KIND_I32 | IN_REG, result, 0);
	}
	def visit_REF_CAST(ht_val: int) {
		var result = emit_call_runtime_doCast(0, ht_val);
		var label = masm.newTrapLabel(TrapReason.FAILED_CAST);
		masm.emit_br_r(result, MasmBrCond.I1_ZERO, label);
		// fallthru and do nothing on cast success
	}
	def visit_REF_CAST_NULL(ht_val: int) {
		var result = emit_call_runtime_doCast(2, ht_val);
		var label = masm.newTrapLabel(TrapReason.FAILED_CAST);
		masm.emit_br_r(result, MasmBrCond.I1_ZERO, label);
		// fallthru and do nothing on cast success
	}
	def visit_BR_ON_CAST(imm: BrOnCastImm) {
		var target = state.getControl(imm.depth);
		var nullable: byte = if(imm.null2(), 2, 0);
		var result = emit_call_runtime_doCast(nullable, imm.ht2); // XXX: statically elimate null, i31
		var sv = SpcVal(KIND_I32 | IN_REG, result, 0);
		emitBrIf(sv, MasmBrCond.I1_ONE, target.label, target, state.isTransferEmpty(target), BrRepush.NONE);
		// fallthru and do nothing on cast fail
	}
	def visit_BR_ON_CAST_FAIL(imm: BrOnCastImm) {
		var target = state.getControl(imm.depth);
		var nullable: byte = if(imm.null2(), 2, 0);
		var result = emit_call_runtime_doCast(nullable, imm.ht2); // XXX: statically elimate null, i31
		var sv = SpcVal(KIND_I32 | IN_REG, result, 0);
		emitBrIf(sv, MasmBrCond.I1_ZERO, target.label, target, state.isTransferEmpty(target), BrRepush.NONE);
		// fallthru and do nothing on cast success
	}
	def visit_ANY_CONVERT_EXTERN() { } // nop
	def visit_EXTERN_CONVERT_ANY() { } // nop
	def visit_REF_I31() {
		var sv = state.peek();
		if (sv.isConst()) {
			var result = (sv.const << 1) | 1; // XXX: introduce into Tagging
			state.overwrite(SpcConsts.KIND_REF | IS_CONST, NO_REG, result);
		} else {
			var sv = popRegToOverwrite();
			masm.emit_shld_r_i(sv.reg, 1);
			masm.emit_binop_r_i(Opcode.I32_OR, sv.reg, 1);
			state.push(SpcConsts.KIND_REF | IN_REG, sv.reg, 0);
		}
	}
	def emitI31_GET(signed: bool) {
		var sv = state.peek();
		if (sv.isConst()) {
			if (sv.const == 0) { // statically null
				emitTrap(TrapReason.NULL_DEREF);
				setUnreachable();
			} else {
				var result = if(signed, (sv.const >> 1), (sv.const >>> 1)); // XXX: introduce into Tagging
				state.overwrite(SpcConsts.KIND_I32 | IS_CONST, NO_REG, result);
			}
		} else {
			var sv = popRegToOverwrite();
			var label = masm.newTrapLabel(TrapReason.NULL_DEREF);
			masm.emit_breq_r_i(sv.reg, 0, label);
			if (signed) masm.emit_sard_r_i(sv.reg, 1);
			else masm.emit_shrd_r_i(sv.reg, 1);
			state.push(SpcConsts.KIND_I32 | IN_REG, sv.reg, 0);
		}
	}
	def visit_I31_GET_S() { emitI31_GET(true); }
	def visit_I31_GET_U() { emitI31_GET(false); }
	def visit_MEMORY_INIT(dindex: u31, mindex: u31) {
		emit_call_runtime_op2(Opcode.MEMORY_INIT, dindex, mindex, true);
	}
	def visit_DATA_DROP(dindex: u31) {
		var tmp = allocTmp(ValueKind.REF);
		emit_load_instance(tmp);
		masm.emit_v3_Instance_dropped_data_r_r(tmp, tmp);
		masm.emit_mov_m_i(MasmAddr(tmp, masm.getOffsets().Array_contents + dindex), 1); // TODO: store to byte array
	}
	def visit_MEMORY_COPY(mindex1: u31, mindex2: u31) {
		emit_call_runtime_op2(Opcode.MEMORY_COPY, mindex1, mindex2, true);
	}
	def visit_MEMORY_FILL(mindex: u31) {
		emit_call_runtime_op1(Opcode.MEMORY_FILL, mindex, true);
	}
	def visit_TABLE_INIT(eindex: u31, tindex: u31) {
		emit_call_runtime_op2(Opcode.TABLE_INIT, eindex, tindex, true);
	}
	def visit_ELEM_DROP(dindex: u31) {
		var tmp = allocTmp(ValueKind.REF);
		emit_load_instance(tmp);
		masm.emit_v3_Instance_dropped_elems_r_r(tmp, tmp);
		masm.emit_mov_m_i(MasmAddr(tmp, masm.getOffsets().Array_contents + dindex), 1); // TODO: store to byte array
	}
	def visit_TABLE_COPY(tindex1: u31, tindex2: u31) {
		emit_call_runtime_op2(Opcode.TABLE_COPY, tindex1, tindex2, true);
	}
	def visit_TABLE_GROW(table_index: u31) {
		var t = if(module.tables[table_index].size.is64, SigCache.arr_l, SigCache.arr_i);
		emit_call_runtime_op1n(Opcode.TABLE_GROW, table_index, 2, t, false);
	}
	def visit_TABLE_SIZE(table_index: u31) {
		var tmp = allocTmp(ValueKind.REF);
		var r1 = allocRegTos(ValueKind.I32);
		emit_load_instance(tmp);
		masm.emit_v3_Instance_tables_r_r(tmp, tmp);
		masm.emit_v3_Array_elem_r_ri(ValueKind.REF, tmp, tmp, int.!(table_index));
		masm.emit_v3_Table_elems_r_r(tmp, tmp);
		masm.emit_v3_Array_length_r_r(r1, tmp);
		def is64 = module.tables[table_index].size.is64;
		state.push(if(is64, KIND_I64 | IN_REG, KIND_I32 | IN_REG), r1, 0);
	}
	def visit_TABLE_FILL(index: u31) {
		emit_call_runtime_op1n(Opcode.TABLE_FILL, index, 3, ValueTypes.NONE, true);
	}
	def visit_I8X16_RELAXED_LANESELECT() { visit_V128_BITSELECT(); }
	def visit_I16X8_RELAXED_LANESELECT() { visit_V128_BITSELECT(); }
	def visit_I32X4_RELAXED_LANESELECT() { visit_V128_BITSELECT(); }
	def visit_I64X2_RELAXED_LANESELECT() { visit_V128_BITSELECT(); }

	def emit_call_runtime_op1(op: Opcode, arg1: u31, canTrap: bool) {
		var count = u32.!(op.sig.params.length);
		emit_call_runtime_op1n(op, arg1, count, op.sig.results, canTrap);
	}
	def emit_call_runtime_op2(op: Opcode, arg1: u31, arg2: u31, canTrap: bool) {
		var count = u32.!(op.sig.params.length);
		emit_call_runtime_op2n(op, arg1, arg2, count, op.sig.results, canTrap);
	}
	def emit_call_runtime_op1n(op: Opcode, arg1: u31, args: u32, results: Array<ValueType>, canTrap: bool) {
		state.emitSaveAll(resolver, runtimeSpillMode);
		emit_compute_vsp(regs.vsp, state.sp);
		masm.emit_store_curstack_vsp(regs.vsp);

		def emit = fun {
			masm.emit_get_curstack(regs.runtime_arg0);
			masm.emit_v3_set_X86_64Stack_rsp_r_r(regs.runtime_arg0, regs.sp);
			masm.emit_push_X86_64Stack_rsp_r_r(regs.runtime_arg0);
			emit_load_instance(regs.runtime_arg1);
			masm.emit_mov_r_i(regs.runtime_arg2, arg1);
			masm.emit_call_runtime_op(op);
		};
		// Reconstruct stack frames across runtime calls that might (Wasm-level) trap.
 		if (canTrap) withReconstructedInlinedFrames(emit); else emit();
		masm.emit_get_curstack(regs.scratch);
		masm.emit_pop_X86_64Stack_rsp_r_r(regs.scratch);
		dropN(args);
		for (t in results) state.push(typeToKindFlags(t) | TAG_STORED | IS_STORED, NO_REG, 0);
		emit_reload_regs();
		if (!runtimeSpillMode.free_regs) state.emitRestoreAll(resolver);
	}
	def emit_call_runtime_op2n(op: Opcode, arg1: u31, arg2: u31, args: u32, results: Array<ValueType>, canTrap: bool) {
		state.emitSaveAll(resolver, runtimeSpillMode);
		emit_compute_vsp(regs.vsp, state.sp);
		masm.emit_store_curstack_vsp(regs.vsp);

		def emit = fun {
			masm.emit_get_curstack(regs.runtime_arg0);
			masm.emit_v3_set_X86_64Stack_rsp_r_r(regs.runtime_arg0, regs.sp);
			masm.emit_push_X86_64Stack_rsp_r_r(regs.runtime_arg0);
			emit_load_instance(regs.runtime_arg1);
			masm.emit_mov_r_i(regs.runtime_arg2, arg1);
			masm.emit_mov_r_i(regs.runtime_arg3, arg2);
			masm.emit_call_runtime_op(op);
		};
		// Reconstruct stack frames across runtime calls that might (Wasm-level) trap.
 		if (canTrap) withReconstructedInlinedFrames(emit); else emit();
		masm.emit_get_curstack(regs.scratch);
		masm.emit_pop_X86_64Stack_rsp_r_r(regs.scratch);
		dropN(args);
		for (t in results) state.push(typeToKindFlags(t) | TAG_STORED | IS_STORED, NO_REG, 0);
		emit_reload_regs();
		if (!runtimeSpillMode.free_regs) state.emitRestoreAll(resolver);
	}
	def emitBrIf(sv: SpcVal, cond: MasmBrCond, label: MasmLabel, target: SpcControl, emptyTransfer: bool, repush: BrRepush) {
		if (sv.isConst()) {
			var taken = (sv.const == 0) == cond.zero;
			if (taken) {
				if (repush.taken) state.pushV(sv);
				if (!emptyTransfer) state.emitTransfer(target, resolver);
				masm.emit_br(label);
				setUnreachable();
			}
		} else if (emptyTransfer) {
			if (sv.inReg()) emitBrIfR(sv, cond, label);
			else masm.emit_br_m(masm.slotAddr(state.sp), cond, label);
			if (repush.not_taken) state.pushV(sv);
		} else {
			var skip = masm.newLabel(it.pc);
			if (sv.inReg()) emitBrIfR(sv, masm.negate(cond), skip);
			else masm.emit_br_m(masm.slotAddr(state.sp), masm.negate(cond), skip);
			if (repush.taken) state.pushV(sv);
			state.emitTransfer(target, resolver);
			masm.emit_br(label);
			masm.bindLabel(skip);
			if (repush.taken) state.pop();
			if (repush.not_taken) state.pushV(sv);
		}
	}
	// If {sv} is a REF_U64, extract the REF part into %scratch and condition on that.
	// Asserts {sv.inReg()}.
	def emitBrIfR(sv: SpcVal, cond: MasmBrCond, label: MasmLabel) {
		if (sv.kind() == ValueKind.REF_U64) {
			masm.emit_ref_from_ref_u64(xenv.scratch, sv.reg);
			masm.emit_br_r(xenv.scratch, cond, label);
		} else {
			masm.emit_br_r(sv.reg, cond, label);
		}
	}
	def emitBrTable(sv: SpcVal, depths: Range<u31>) {
		if (sv.isConst()) {
			// constant-fold br_table into a br
			var key = sv.const;
			if (u32.view(key) >= depths.length) key = depths.length - 1;
			var target = state.getControl(depths[key]);
			state.emitTransfer(target, resolver);
			masm.emit_br(target.label);
			return;
		}
		var labels = Array<MasmLabel>.new(state.ctl_stack.top);
		var targets = Array<MasmLabel>.new(depths.length);
		for (i < targets.length) { // create labels for all targets involved in this br_table
			var depth = depths[i];
			var l = labels[depth];
			if (l == null) l = labels[depth] = masm.newLabel(it.pc);
			targets[i] = l;
		}
		masm.emit_br_table_r(ensureReg(sv, state.sp), targets);

		for (depth < labels.length) {
			var l = labels[depth];
			if (l == null) continue;
			masm.bindLabel(l);
			var target = state.getControl(u32.view(depth));
			state.emitTransfer(target, resolver);
			masm.emit_br(target.label);
		}
	}
	def emitReturn(ctl: SpcControl) {
		// All explicit RETURN instructions branch here.
		masm.bindLabel(ret_label);

		var results = sig.results;
		if (masm.valuerep.tagged) {
			// update mismatched value tags
			var params = sig.params;
			for (i < results.length) {
				var rtag = ValueTypes.kind(results[i]);
				if (i < params.length && rtag == ValueTypes.kind(params[i])) continue; // tag already correct
				masm.emit_mov_m_i(masm.tagAddr(state.sp - u32.view(results.length) + u32.view(i)), rtag.code);
			}
		}

		if (isInlined()) return;

		// Compute VSP = VFP + state.sp
		emit_compute_vsp(regs.vsp, state.sp);
		// Return to caller
		masm.emit_mov_r_i(regs.ret_throw, 0);
		// Deallocate stack frame
		masm.emit_addw_r_i(regs.sp, frame.frameSize);
		masm.emit_ret();
	}
	def emitOsrEntry(osr_entry_label: MasmLabel, state: Array<SpcVal>) {
		if (Trace.compiler) Trace.OUT.put1("  OSR (+%d)", osr_entry_label.create_pos).ln();
		masm.bindLabel(osr_entry_label);
		emit_reload_regs();
		for (i < u32.!(state.length)) {
			var sv = state[i];
			var iv = SpcVal(sv.kindFlags(IS_STORED | TAG_STORED), NO_REG, 0);
			resolver.addMove((i, sv), (i, iv));
		}
		resolver.emitMoves();
		masm.emit_br(osr_loop_label);
	}
	def emitTrap(reason: TrapReason) {
		var label = masm.newTrapLabel(reason);
		masm.emit_br(label);
	}
	def emitTrapReturn(label: MasmLabel, reason: TrapReason) {
		if (label != null) masm.bindLabel(label);
		masm.emit_mov_r_trap(regs.ret_throw, reason);
		masm.emit_addw_r_i(regs.sp, frame.frameSize);
		masm.emit_ret();
	}
	def newTrapLabel(reason: TrapReason) -> MasmLabel {
		var label = masm.newLabel(it.pc);
		var frames = snapshotFrames();
		trap_labels.put((reason, label, frames));
		return label;
	}
	def getSpcInlinedFrameIp() -> long;
	// Emit code to materialize stack frames for each inlined function.
	def emitReconstructStackFrames(frames: Array<SpcFrame>) -> int {
		Metrics.spc_static_reconst.val++;
		masm.emit_inc_metric(Metrics.spc_dynamic_reconst);
		def real_frame = frames[0];
		masm.emit_mov_m_i(xenv.pc_slot, real_frame.pc);

		// load instance
		var inst_reg = allocTmp(ValueKind.REF);
		masm.emit_mov_r_m(ValueKind.REF, inst_reg, frame.instance_slot);
		var mem_reg = allocTmp(ValueKind.REF);
		masm.emit_mov_r_m(ValueKind.REF, mem_reg, frame.mem0_base_slot);
		// Load instance.functions
		def func_reg = allocTmp(ValueKind.REF);
		masm.emit_v3_Instance_functions_r_r(func_reg, inst_reg);
		def vfp_reg = allocTmp(ValueKind.REF);
		masm.emit_mov_r_m(ValueKind.REF, vfp_reg, frame.vfp_slot);
		var prev_base_sp = int.view(frames[0].local_base_sp);
		var wasm_func_reg = allocTmp(ValueKind.REF);

		var inl_inst_reg: Reg, inl_mem0_reg: Reg;
		if (whamm_config.is_inlined) { // TODO investigate, check individual configs?
			inl_inst_reg = allocTmp(ValueKind.REF);
			inl_mem0_reg = allocTmp(ValueKind.REF);
			masm.emit_mov_r_m(ValueKind.REF, inl_inst_reg, frame.inlined_instance_slot);
			masm.emit_mov_r_m(ValueKind.REF, inl_mem0_reg, frame.inlined_mem0_base_slot);
		}

		// Pre-allocate stack space for all reconstructed frames at once.
		def total_space = (frames.length - 1) * (frame.frameSize + 8);
		masm.emit_subw_r_i(regs.sp, total_space);

		// Process the inlined frames (skip the outermost which already exists on native stack)
		for (i = 1; i < frames.length; i++) {
			def frame_info = frames[i];
			def cur_base_sp = int.view(frame_info.local_base_sp);
			def delta = (cur_base_sp - prev_base_sp) * masm.valuerep.slot_size;
			emitReconstructStackFrame(frame_info, frames.length - i - 1, delta,
					wasm_func_reg, func_reg, inst_reg, mem_reg, vfp_reg, inl_inst_reg, inl_mem0_reg);
			prev_base_sp = cur_base_sp;
		}

		return total_space;
	}
	def emitReconstructStackFrame(spcFrame: SpcFrame, offset: int, vfp_delta: int,
			wasm_func_reg: Reg, func_reg: Reg, inst_reg: Reg, mem_reg: Reg, vfp_reg: Reg, inl_inst_reg: Reg, inl_mem0_reg: Reg) {
		// Use inlined frame stub IP as return address for all reconstructed frames
		def return_addr = getSpcInlinedFrameIp();

		def frame_offset = offset * (frame.frameSize + 8);
		// Write inlined frame stub IP as return address
		def retaddr_slot = MasmAddr(regs.sp, frame_offset + frame.frameSize);
		masm.emit_mov_m_l(retaddr_slot, return_addr);

		// get functions[func_index] and save into frame
		def wasm_func_slot = frame.wasm_func_slot.plus(frame_offset);
		masm.emit_v3_Array_elem_r_ri(ValueKind.REF, wasm_func_reg, func_reg, spcFrame.func.func_index);
		masm.emit_mov_m_r(ValueKind.REF, wasm_func_slot, wasm_func_reg);

		// Save instance
		def instance_slot = frame.instance_slot.plus(frame_offset);
		masm.emit_mov_m_r(ValueKind.REF, instance_slot, inst_reg);

		// Save mem0 base
		def mem0_base_slot = frame.mem0_base_slot.plus(frame_offset);
		masm.emit_mov_m_r(ValueKind.REF, mem0_base_slot, mem_reg);

		// Step vfp_reg by change in local_base_sp from previous frame and save
		if (vfp_delta != 0) masm.emit_addw_r_i(vfp_reg, vfp_delta);
		def vfp_slot = frame.vfp_slot.plus(frame_offset);
		masm.emit_mov_m_r(ValueKind.REF, vfp_slot, vfp_reg);

		// Save PC
		def pc_slot = frame.pc_slot.plus(frame_offset);
		masm.emit_mov_m_i(pc_slot, spcFrame.pc);

		// Clear FrameAccessor
		def accessor_slot = frame.accessor_slot.plus(frame_offset);
		masm.emit_mov_m_l(accessor_slot, 0);

		// if an inlined whamm probe, also grab inlined slots
		if (whamm_config.is_inlined) {
			def inl_instance_slot = frame.inlined_instance_slot.plus(frame_offset);
			masm.emit_mov_m_r(ValueKind.REF, inl_instance_slot, inl_inst_reg);
			def inl_mem0_base_slot = frame.inlined_mem0_base_slot.plus(frame_offset);
			masm.emit_mov_m_r(ValueKind.REF, inl_mem0_base_slot, inl_mem0_reg);
		} else {
			def inl_instance_slot = frame.inlined_instance_slot.plus(frame_offset);
			masm.emit_mov_m_l(inl_instance_slot, 0);
			def inl_mem0_base_slot = frame.inlined_mem0_base_slot.plus(frame_offset);
			masm.emit_mov_m_l(inl_mem0_base_slot, 0);
		}
	}
	// Guards compiler code with frame reconstruction (if necessary).
	def withReconstructedInlinedFrames(emit: void -> void) {
		if (isInlined()) {
			if (frames_reconstructed) {
				// FIXME this should not happen (but does):
				// - in the case of deep nesting when one layer is a Whamm probe
				// - when refactoring to avoid `with` clause, GC test fails (inlining depth 2)
				if (Trace.compiler) Trace.OUT.puts("  nested frame reconstruction inhibited\n");
				emit();
				return;
			}
			unrefRegs();
			frames_reconstructed = true;
			def space = emitReconstructStackFrames(snapshotFrames());
			emit();
			frames_reconstructed = false;
			if (space > 0) {
				masm.emit_addw_r_i(regs.sp, space);
				masm.emit_mov_r_m(ValueKind.REF, regs.vfp, frame.vfp_slot);
			}
		} else {
			emit();
		}

	}
	def unsupported() {
		success = false; // XXX: add opcode
	}
	def bailout(msg: string) {
		success = false;
		if (Trace.compiler) Trace.OUT.put1("-- SPC bailout: %s", msg).ln();
		var cp = it.immptr();
		if (err != null) err.rel(cp, it.pc).FailedToCompile(func.func_index, msg);
	}

	// Fold an unary operation if a constant is on the top of the stack.
	def tryFold_i_i(f: i32 -> i32) -> bool {
		var sv = state.peek();
		if (sv.isConst()) {
			var r = f(sv.const);
			if (r != sv.const) {
				pop();
				state.push(KIND_I32 | IS_CONST, NO_REG, r);
			}
			return true;
		}
		return false;
	}
	def tryFold_x_y<X, Y>(kind: ValueKind, f: X -> Y, toX: i32 -> X, toY: i32 -> Y, fromY: Y -> i32) -> bool {
		var a = state.peek();
		if (a.isConst()) {
			var rY = f(toX(a.const));
			var r = fromY(rY);
			var cY = toY(r);
			if (cY != rY) return false;
			pop();
			state.push(SpcConsts.kindToFlags(kind) | IS_CONST, NO_REG, r);
			return true;
		}
		return false;
	}
	def tryFold_u_u(f: u32 -> u32) -> bool { return tryFold_x_y(ValueKind.I32, f, u32.view<i32>, u32.view<i32>, i32.view<u32>); }
	def tryFold_i_l(f: i32 -> i64) -> bool { return tryFold_x_y(ValueKind.I64, f, i32.view<i32>, i64.view<i32>, i32.view<i64>); }
	def tryFold_u_l(f: u32 -> i64) -> bool { return tryFold_x_y(ValueKind.I64, f, u32.view<i32>, i64.view<i32>, i32.view<i64>); }
	def tryFold_w_w(f: u64 -> u64) -> bool { return tryFold_x_y(ValueKind.I64, f, u64.view<i32>, u64.view<i32>, i32.view<u64>); }
	def tryFold_l_i(f: i64 -> i32) -> bool { return tryFold_x_y(ValueKind.I32, f, i64.view<i32>, i32.view<i32>, i32.view<i32>); }
	def tryFold_l_l(f: i64 -> i64) -> bool { return tryFold_x_y(ValueKind.I64, f, i64.view<i32>, i64.view<i32>, i32.view<i64>); }

	// Utilities to try to constant-fold using the given evaluation function {f}. Since all SPC constants
	// are represented by the V3 type {i32}, this is the fast path.
	def tryFold_ii_i(f: (i32, i32) -> i32) -> bool {
		var sv = state.peek2(), a = sv.0, b = sv.1;
		if (a.isConst() && b.isConst()) {
			var r = f(a.const, b.const);
			pop();
			pop();
			state.push(KIND_I32 | IS_CONST, NO_REG, r);
			return true;
		}
		return false;
	}
	// For evaluation functions that are not of type {i32}, adapt them polymorphically.
	def tryFold_xx_y<X, Y>(kind: ValueKind, f: (X, X) -> Y, toX: i32 -> X, toY: i32 -> Y, fromY: Y -> i32) -> bool {
		var sv = state.peek2(), a = sv.0, b = sv.1;
		if (a.isConst() && b.isConst()) {
			var rY = f(toX(a.const), toX(b.const));
			var r = fromY(rY);
			var cY = toY(r);
			if (cY != rY) return false;
			pop();
			pop();
			state.push(SpcConsts.kindToFlags(kind) | IS_CONST, NO_REG, r);
			return true;
		}
		return false;
	}
	// All the polymorphic variants of folding.
	def tryFold_uu_u(f: (u32, u32) -> u32) -> bool   { return tryFold_xx_y(ValueKind.I32, f, u32.view<i32>, u32.view<i32>, i32.view<u32>); }
	def tryFold_ii_z(f: (i32, i32) -> bool) -> bool  { return tryFold_xx_y(ValueKind.I32, f, i32.view<i32>, isNotZero, trueToOne); }
	def tryFold_uu_z(f: (u32, u32) -> bool) -> bool  { return tryFold_xx_y(ValueKind.I32, f, u32.view<i32>, isNotZero, trueToOne); }
	def tryFold_ll_l(f: (i64, i64) -> i64) -> bool   { return tryFold_xx_y(ValueKind.I64, f, i64.view<i32>, i64.view<i32>, i32.view<i64>); }
	def tryFold_ww_w(f: (u64, u64) -> u64) -> bool   { return tryFold_xx_y(ValueKind.I64, f, u64.view<i32>, u64.view<i32>, i32.view<u64>); }
	def tryFold_ll_z(f: (i64, i64) -> bool) -> bool  { return tryFold_xx_y(ValueKind.I32, f, i64.view<i32>, isNotZero, trueToOne); }
	def tryFold_qq_z(f: (u64, u64) -> bool) -> bool  { return tryFold_xx_y(ValueKind.I32, f, u64.view<i32>, isNotZero, trueToOne); }

	//====================================================================
	// codegen operations
	//====================================================================
	def emit_compute_vsp(dst: Reg, slots: u32) {
		masm.emit_mov_r_r(ValueKind.REF, dst, regs.vfp); // XXX: use 3-addr adjustment of VSP (i.e. lea)
		if (slots > 0) masm.emit_addw_r_i(dst, int.view(slots) * masm.valuerep.slot_size);
	}
	def emit_spill_vsp(reg: Reg) {
		masm.emit_mov_m_r(ValueKind.REF, frame.vsp_slot, reg);
	}
	def emit_reload_regs() {
		// XXX: recompute VFP from VSP - #slots?
		masm.emit_mov_r_m(ValueKind.REF, regs.vfp, frame.vfp_slot);
		if (module.memories.length > 0) {
			if (whamm_config.is_inlined) {
				masm.emit_mov_r_m(ValueKind.REF, regs.mem0_base, frame.inlined_mem0_base_slot);
			} else {
				masm.emit_mov_r_m(ValueKind.REF, regs.mem0_base, frame.mem0_base_slot);
			}
		}
	}
	def emit_load_instance(reg: Reg) {
		if (whamm_config.is_inlined) { // inline compilation
			masm.emit_mov_r_m(ValueKind.REF, reg, frame.inlined_instance_slot);
		} else {
			masm.emit_mov_r_m(ValueKind.REF, reg, frame.instance_slot);
		}
	}
	def emitLoad(kind: ValueKind, imm: MemArg, meth: (ValueKind, Reg, Reg, Reg, u32) -> ())  {
		var iv = pop();
		var addr = emitComputeEffectiveAddress(imm, iv);
		if (addr.oob) return; // statically OOB
		var dest = allocRegTos(kind); // XXX: can reuse index reg if frequency == 1 and ValueKind.I32
		meth(kind, dest, addr.base_reg, addr.index_reg, addr.offset);
		var indexKind = if(addr.is64, ValueKind.I64, ValueKind.I32);
		var nflags = IN_REG | SpcConsts.kindToFlags(kind);
		if (kind == indexKind) nflags |= (iv.flags & TAG_STORED); // tag may already be stored for index
		state.push(nflags, dest, 0);

		if (intrinsified_read_probe != null) { // TODO(memory64)
			// spill everything
			state.emitSaveAll(resolver, probeSpillMode);
			var index_reg = addr.index_reg;
			// load RT args (addr, val) and call RT
			if (addr.index_reg == NO_REG) { // fixed addr
				index_reg = allocTmp(ValueKind.I32);
				masm.emit_mov_r_i(index_reg, int.view(addr.offset));
			} else {
				masm.emit_addw_r_i(index_reg, int.view(addr.offset));
			}
			var arg1 = masm.getV3ParamReg(ValueKind.REF, 1);
			var arg2 = masm.getV3ParamReg(ValueKind.REF, 2);
			masm.emit_mov_r_r(ValueKind.REF, arg1, index_reg);
			masm.emit_mov_r_m(ValueKind.REF, arg2, masm.slotAddr(state.sp - 1));
			masm.emit_call_MemoryReadProbe_fire(intrinsified_read_probe);
			emit_reload_regs();
			if (!probeSpillMode.free_regs) state.emitRestoreAll(resolver);
			intrinsified_read_probe = null;
		}
	}
	def emitStore(kind: ValueKind, imm: MemArg, meth: (ValueKind, Reg, Reg, Reg, u32) -> ()) {
		var sv = popReg();
		var iv = pop();
		var addr = emitComputeEffectiveAddress(imm, iv);
		if (addr.oob) return; // statically OOB
		meth(kind, sv.reg, addr.base_reg, addr.index_reg, addr.offset);
	}
	def emitComputeEffectiveAddress(imm: MemArg, iv: SpcVal) -> EffectiveAddr {
		var mdecl = module.memories[imm.memory_index];
		var base_reg = regs.mem0_base;
		if (imm.memory_index != 0) {
			// XXX: cache the base register for memories > 0
			base_reg = allocTmp(ValueKind.REF);
			emit_load_instance(base_reg);
			masm.emit_v3_Instance_memories_r_r(base_reg, base_reg);
			masm.emit_v3_Array_elem_r_ri(ValueKind.REF, base_reg, base_reg, imm.memory_index);
			masm.emit_v3_Memory_start_r_r(base_reg, base_reg);
		}
		var index_reg: Reg;
		var offset = imm.offset;
		var limit = if(mdecl.size.is64, Target.limit_memory_size_64, Target.limit_memory_size);
		if (offset >= limit) {
			emitTrap(TrapReason.MEMORY_OOB); // statically OOB
			return OOB_ADDR;
		}
		if (iv.isConst()) {
			offset += u32.view(iv.const); // fold offset calculation
			if (offset >= limit) {
				emitTrap(TrapReason.MEMORY_OOB); // statically OOB
				return OOB_ADDR;
			}
		} else {
			index_reg = ensureReg(iv, state.sp);
			if (mdecl.size.is64 && !SpcTuning.disableMemoryBoundsChecks) {
				// emit a bounds check of (index >> N) != 0
				var shift_reg = allocTmp(ValueKind.I64);
				masm.emit_mov_r_r(ValueKind.I64, shift_reg, index_reg);
				masm.emit_shrw_r_i(shift_reg, 34); // TODO: compute shift amount from max memory size
				var oob_label = newTrapLabel(TrapReason.MEMORY_OOB);
				masm.emit_br_r(shift_reg, MasmBrCond.I64_NONZERO, oob_label); // XXX: on x86, shift sets flags
			}
		}
		if (u32.?(offset)) {
			return EffectiveAddr(false, mdecl.size.is64, base_reg, index_reg, u32.!(offset));
		} else {
			var sum_reg = allocTmp(ValueKind.I64);
			masm.emit_mov_r_l(sum_reg, long.view(offset));
			if (index_reg != NO_REG) masm.emit_addw_r_r(sum_reg, index_reg);
			return EffectiveAddr(false, mdecl.size.is64, base_reg, sum_reg, 0);
		}
	}

	//====================================================================
	// register allocation operations
	//====================================================================
	def freeReg(reg: Reg) {
		if (reg.index > 0) regAlloc.free(reg);
	}
	def allocRegTos(kind: ValueKind) -> Reg {
		return allocReg(kind, state.sp);
	}
	def allocReg(kind: ValueKind, slot: u32) -> Reg {
		var reg = regAlloc.alloc(kind, int.!(slot));
		if (reg == NO_REG) return findAndSpillReg(kind, int.!(slot));
		return reg;
	}
	def tryAllocReg(kind: ValueKind, slot: u32) -> Reg {
		return regAlloc.alloc(kind, int.!(slot));
	}
	def allocTmp(kind: ValueKind) -> Reg {
		var reg = regAlloc.alloc(kind, TMP_SLOT);
		if (reg == NO_REG) reg = findAndSpillReg(kind, TMP_SLOT);
		unrefLater(reg, TMP_SLOT);
		return reg;
	}
	def allocTmpFixed(kind: ValueKind, reg: Reg) -> Reg {
		if (!regAlloc.isFree(reg)) spillRegAndFree(reg);
		regAlloc.assign(reg, TMP_SLOT);
		unrefLater(reg, TMP_SLOT);
		return reg;
	}
	def allocFixed(kind: ValueKind, reg: Reg, slot: u32) -> Reg {
		if (!regAlloc.isFree(reg)) spillRegAndFree(reg);
		regAlloc.assign(reg, int.!(slot));
		return reg;
	}
	def ensureReg(sv: SpcVal, slot: u32) -> Reg {
		var reg = sv.reg;
		if (reg == NO_REG) {
			var kind = sv.kind();
			reg = allocTmp(kind);
			if (sv.isConst()) masm.emit_mov_r_k(kind, reg, sv.const);
			else masm.emit_mov_r_s(kind, reg, slot);
		}
		return reg;
	}
	def unrefRegs() {
		var c = num_unrefs;
		for (i < c) regAlloc.unassign(unrefs[i]);
		num_unrefs = 0;
	}
	def unrefSlot(slot: u32) {
		var sv = state.state[slot];
		if (sv.reg != NO_REG) regAlloc.unassign(sv.reg, int.!(slot));
	}
	def unrefLater(reg: Reg, assign: int) { // XXX: #inline
		unrefs[num_unrefs++] = (reg, assign);
	}
	def findAndSpillReg(kind: ValueKind, slot: int) -> Reg {
		var reg = regAlloc.findSpillCandidate(kind, addSpillCost);
		if (reg != NO_REG) {
			spillRegAndFree(reg);
			regAlloc.assign(reg, slot);
			return reg;
		}
		bailout("out of registers");
		regAlloc.clear();
		return regAlloc.alloc(kind, slot);
	}
	def addSpillCost(score: int, reg: Reg, slot: int) -> int {
		if (slot >= state.sp) return 100000000 + score;       // tmp slot?
		if (!state.state[slot].isStored()) score += 100;      // will generate a spill
		if (slot < 10 && slot < func.num_locals) score += 10; // penalize first 10 locals
		if (int.!(state.sp) - slot < 4) score += 20;          // penalize top 4 operand slots
		// XXX: boost first 8 x86 registers because of possible REX byte?
		return score;
	}

	//====================================================================
	// abstract stack operations
	//====================================================================
	def labelArgs(ctl: SpcControl) -> Array<ValueType> {
		if (ctl.opcode == Opcode.LOOP.code) return ctl.params;
		else return ctl.results;
	}
	// Pop the top of the stack.
	def pop() -> SpcVal {
		var sv = state.pop();
		if (sv.inReg()) unrefLater(sv.reg, int.!(state.sp));
		return sv;
	}
	// Pop and drop {count} items from the top of the stack.
	def dropN(count: u32) {
		for (i < count) {
			var sv = state.pop();
			if (sv.reg != NO_REG) regAlloc.unassign(sv.reg, int.!(state.sp));
		}
	}
	// Pop the top of the stack into a register of the appropriate kind.
	def popReg() -> SpcVal {
		var sv = state.pop();
		if (sv.inReg()) {
			if (Trace.compiler) Trace.OUT.put1("popReg() in %s", masm.regConfig.regSet.getName(sv.reg)).ln();
			regAlloc.reassign(sv.reg, int.!(state.sp), TMP_SLOT);
			if (Trace.compiler) { regAlloc.render(Trace.OUT, masm.regConfig.regSet); Trace.OUT.ln(); }
			unrefLater(sv.reg, TMP_SLOT);
			return sv;
		}
		var kind = sv.kind();
		var reg = allocTmp(kind);
		if (sv.isConst()) masm.emit_mov_r_k(kind, reg, sv.const);
		else masm.emit_mov_r_s(kind, reg, state.sp);
		unrefLater(sv.reg, int.!(state.sp));
		return SpcVal(sv.flags | IN_REG, reg, sv.const);
	}
	// Pop the top of the stack into {reg}, spilling any value(s) in {reg} first.
	def popFixedReg(reg: Reg) -> SpcVal {
		var sv = state.pop(), slot = int.!(state.sp);
		if (sv.reg == reg) {
			regAlloc.reassign(sv.reg, slot, TMP_SLOT);
			unrefLater(sv.reg, TMP_SLOT);
			return sv;
		}
		var kind = sv.kind();
		allocTmpFixed(kind, reg);
		if (sv.inReg()) {
			masm.emit_mov_r_r(kind, reg, sv.reg);
			regAlloc.unassign(sv.reg, slot);
		} else if (sv.isConst()) {
			masm.emit_mov_r_k(kind, reg, sv.const);
		} else {
			masm.emit_mov_r_s(kind, reg, state.sp);
		}
		return SpcVal(sv.flags | IN_REG, reg, sv.const);
	}
	// Pop the top of stack into a register and prepare for it to be overwritten.
	def popRegToOverwrite() -> SpcVal {
		var sv = state.pop();
		if (regAlloc.frequency(sv.reg) == 1) return sv; // can reuse reg immediately
		var kind = sv.kind();
		var reg = allocRegTos(kind);
		if (sv.isConst()) {
			masm.emit_mov_r_k(kind, reg, sv.const);
		} else if (sv.reg != NO_REG) {
			masm.emit_mov_r_r(kind, reg, sv.reg);
			regAlloc.unassign(sv.reg, int.!(state.sp));
		} else {
			masm.emit_mov_r_s(kind, reg, state.sp);
		}
		return SpcVal(sv.flags | IN_REG, reg, sv.const);
	}
	// Pop the top of stack into a register and prepare for it to be reused at {delta} slots.
	def popRegToReuse(delta: int) -> SpcVal {
		var sv = state.pop(), tos = int.!(state.sp);
		if (regAlloc.frequency(sv.reg) == 1) {
			regAlloc.reassign(sv.reg, tos, tos + delta);
			return sv; // can reuse reg immediately
		}
		var kind = sv.kind();
		var reg = allocReg(kind, u32.!(tos + delta));
		if (sv.isConst()) {
			masm.emit_mov_r_k(kind, reg, sv.const);
		} else if (sv.reg != NO_REG) {
			masm.emit_mov_r_r(kind, reg, sv.reg);
			regAlloc.unassign(sv.reg, tos);
		} else {
			masm.emit_mov_r_s(kind, reg, state.sp);
		}
		return SpcVal(sv.flags | IN_REG, reg, sv.const);
	}
	// Pop the top of stack into a register, but only temporarily, as it will be clobbered.
	def popTmpReg() -> SpcVal {
		var sv = state.pop(), reg = sv.reg;
		if (regAlloc.frequency(reg) == 1) { // can use register as a temporary
			regAlloc.reassign(reg, int.!(state.sp), TMP_SLOT);
			unrefLater(reg, TMP_SLOT);
			return sv;
		}
		var kind = sv.kind();
		reg = allocTmp(kind);
		if (sv.isConst()) {
			masm.emit_mov_r_k(kind, reg, sv.const);
		} else if (sv.reg != NO_REG) {
			masm.emit_mov_r_r(kind, reg, sv.reg);
		} else {
			masm.emit_mov_r_s(kind, reg, state.sp);
		}
		return SpcVal(sv.flags | IN_REG, reg, sv.const);
	}
	// Spill a register to its slot(s), if it has been allocated.
	def spillRegAndFree(reg: Reg) {
		regAlloc.forEachAssignment(reg, emitSpill); // XXX: iterate assignment list directly?
		regAlloc.free(reg);
	}
	def emitSpill(slot: int) {
		if (slot == TMP_SLOT) return;
		var sv = state.state[slot];
		if (sv.isConst()) return void(state.state[slot] = sv.withoutReg());
		if (sv.isStored()) return void(state.state[slot] = sv.withoutReg());
		masm.emit_mov_s_r(sv.kind(), u32.!(slot), sv.reg);
		state.state[slot] = SpcVal((sv.flags | IS_STORED) & ~IN_REG, NO_REG, 0);
	}
	def setUnreachable() {
		skip_to_end = true;
		state.setUnreachable();
	}
	def traceOpcodeAndStack(orig: bool) {
		traceOpcode(orig);
		state.trace();
	}
	def traceOpcodeUnreachable(orig: bool) {
		traceOpcode(orig);
		OUT.puts("(unreachable)").ln();
	}
	def traceOpcode(orig: bool) {
		OUT.flush();
		instrTracer.instr_width = Opcodes.longestName + 1;
		instrTracer.putPcAndInstr(OUT, module, func, it.pc, orig);
		if (Trace.asm) {
			OUT.puts("JIT code: ");
			masm.printCodeBytes(OUT, codegen_offset, masm.curCodeBytes());
			codegen_offset = masm.curCodeBytes();
			OUT.ln();
		}
	}
	// Gets the handler ID of the <inline_index>-th handler declared at pc.
	private def getHandlerIndex(pc: int, inline_index: int) -> int {
		var start = Ranges.binarySearchEq<(int, Array<int>)>(
			func.handlers.handler_pc_index_map, (pc, null), fun (a, b) => a.0 < b.0
		);
		return func.handlers.handler_pc_index_map[start].1[inline_index];
	}

	//====================================================================
	// frame stack (for inlining) operations
	//====================================================================
	def pushSpcFrame(frame: SpcFrame) {
		if (func != null) masm.pushInlineContext(func.func_index);

		def current = state.frame_stack.peek();
		if (current != null) {
			current.pc = it.pc;
			current.ret_label = ret_label;
		}
		state.frame_stack.push(frame);
		// Update cached copies from new top frame
		it.reset(frame.func).at(frame.pc, -1);
		module = frame.module;
		func = frame.func;
		sig = func.sig;
		num_locals = frame.num_locals;
		local_base_sp = frame.local_base_sp;
		ctl_base_sp = frame.ctl_base_sp;
		ret_label = frame.ret_label;
	}
	def popSpcFrame() -> SpcFrame {
		masm.popInlineContext();

		var frame = state.frame_stack.pop();
		// Restore cached copies from new top frame
		var current = state.frame_stack.peek();
		it.reset(current.func).at(current.pc, -1).current();
		module = current.module;
		func = current.func;
		sig = func.sig;
		num_locals = current.num_locals;
		local_base_sp = current.local_base_sp;
		ctl_base_sp = current.ctl_base_sp;
		ret_label = current.ret_label;
		return frame;
	}

	def isInlined() -> bool {
		return state.frame_stack.top > 1;
	}
	def inlineDepth() -> int {
		return state.frame_stack.top - 1;
	}
	def snapshotFrames() -> Array<SpcFrame> {
		var frames = Array<SpcFrame>.new(state.frame_stack.top);
		for (i < state.frame_stack.top) {
			var f = state.frame_stack.elems[i];
			var pc = if(i == state.frame_stack.top - 1, it.pc, f.pc);
			frames[i] = SpcFrame.new(f.func, f.module, f.local_base_sp, f.ctl_base_sp, f.num_locals, pc, null);
		}
		return frames;
	}
	// Determine if a regular function call should be inlined
	def shouldInline(func: FuncDecl) -> bool {
		if (Trace.compiler) OUT.put1("deciding on inlining call to func #%d: ", func.func_index);

		if (func.imp != null) return no("imported");
		if (inlineDepth() >= SpcTuning.maxInlineDepth) return no("max inline depth exceeded");
		if (func.orig_bytecode.length > SpcTuning.maxInlineBytecodeSize) return no("func too large");
		if (func.sig.params.length > SpcTuning.maxInlineParams) return no("too many parameters");

		// Scan bytecode for unsupported instructions
		var bi = BytecodeIterator.new().reset(func);
		while (bi.more()) {
			match (bi.current()) {
				RETURN_CALL, RETURN_CALL_INDIRECT, RETURN_CALL_REF => 
					return no("uses return instruction");
				TRY, CATCH, THROW, RETHROW, THROW_REF, DELEGATE, CATCH_ALL, TRY_TABLE =>
					return no("uses exception handling instruction");
				CONT_NEW, CONT_BIND, SUSPEND, RESUME, RESUME_THROW, RESUME_THROW_REF, SWITCH =>
					return no("uses stack switching instruction");
				_ => ;
			}
			bi.next();
		}

		if (Trace.compiler) OUT.puts("YES\n");
		return true;
	}
	private def no(reason: string) -> bool {
		if (Trace.compiler) OUT.puts("NO (").puts(reason).putc(')').ln();
		return false;
	}
}
// Different branch instructions have different repush
enum BrRepush(taken: bool, not_taken: bool) {
	NONE(false, false),
	TAKEN(true, false),
	NOT_TAKEN(false, true),
	BOTH(true, true)
}

// When spilling all registers, controls the update to the abstract state, if any.
enum SpillMode(free_regs: bool, remember_stored: bool) {
	SAVE_ONLY(false, false),		// save registers and constants, don't update state
	SAVE_AND_REMEMBER_STORED(false, true),	// save registers and constants, remember slots are stored
	SAVE_AND_FREE_REGS(true, true),		// save registers and free all registers
}

// Used for computing memory load/store addresses
type EffectiveAddr(oob: bool, is64: bool, base_reg: Reg, index_reg: Reg, offset: u32) #unboxed;
def OOB_ADDR = EffectiveAddr(true, false, NO_REG, NO_REG, 0);

// Compiler representation of slots containing abstract values on the WebAssembly value stack,
// including the {ValueKind}, whether the value is stored into value stack memory, whether the
// value tag is stored into value stack memory, which if any register holds its value, and which
// constant value, if any, it is.
type SpcVal(flags: byte, reg: Reg, const: int) #unboxed {
	def kind() -> ValueKind {
		return SpcConsts.kinds[flags >> 4];
	}
	def kindFlags(add: byte) -> byte {
		return (flags & KIND_MASK) | add;
	}
	def kindFlagsAndTag(add: byte) -> byte {
		return (flags & (KIND_MASK | TAG_STORED)) | add;
	}
	def kindFlagsMatching(kind: ValueKind, add: byte) -> byte {
		var okind = (flags & KIND_MASK);
		var nkind = SpcConsts.kindToFlags(kind);
		if (okind == nkind) nkind |= (flags & TAG_STORED);
		return nkind | add;
	}
	def isStored() -> bool {
		return (flags & IS_STORED) != 0;
	}
	def isConst() -> bool {
		if (kind() == ValueKind.REF_U64) return (flags & IS_CONST) != 0 && const == 0;
		return (flags & IS_CONST) != 0;
	}
	def inReg() -> bool {
		return (flags & IN_REG) != 0;
	}
	def tagStored() -> bool {
		return (flags & TAG_STORED) != 0;
	}
	def withoutReg() -> SpcVal {
		return SpcVal(flags & ~IN_REG, NO_REG, const);
	}
	def withFlags(add: byte) -> SpcVal {
		return SpcVal(flags | add, reg, const);
	}
	def notConst() -> SpcVal {
		return SpcVal(flags & ~IS_CONST, reg, 0);
	}
	def sameValue(that: SpcVal) -> bool {
		if (((this.flags & that.flags & IS_CONST) != 0) && this.const == that.const) return true;
		if (((this.flags & that.flags & IN_REG) != 0) && this.reg == that.reg) return true;
		return false;
	}
	def render(buf: StringBuilder, regSet: RegSet) -> StringBuilder {
		match (kind()) {
			I32 => buf.puts("i");
			I64 => buf.puts("l");
			F32 => buf.puts("f");
			F64 => buf.puts("d");
			V128 => buf.puts("v");
			REF => buf.puts("r");
			REF_U64 => buf.puts("c");
		}
		buf.put2("%s%s",
			if(tagStored(), "T", ""),
			if(isStored(), "S", ""));
		if (inReg()) buf.put1("@%s", regSet.getName(reg));
		if (isConst()) buf.put1("=%d", const);
		buf.puts("|");
		return buf;
	}
	def renderTrace(buf: TraceBuilder, regSet: RegSet) -> TraceBuilder {
		if (buf.hasColor(Color.UNDERLINE) && isStored()) buf.beginColor(Color.UNDERLINE);
		if (isConst()) buf.beginColor(Color.CONST);
		else if (inReg()) buf.beginColor(Color.REGISTER);
		if (buf.hasColor(Color.BOLD) && tagStored()) buf.beginColor(Color.BOLD);
		match (kind()) {
			I32 => buf.puts("i");
			I64 => buf.puts("l");
			F32 => buf.puts("f");
			F64 => buf.puts("d");
			V128 => buf.puts("v");
			REF => buf.puts("r");
			REF_U64 => buf.puts("c");
		}
		if (!buf.hasColor(Color.BOLD) && tagStored()) buf.puts("T");
		if (!buf.hasColor(Color.UNDERLINE) && isStored()) buf.puts("S");
		if (inReg()) buf.put1("@%s", regSet.getName(reg));
		if (isConst()) buf.put1("=%d", const);
		buf.endColor(Color.DEFAULT);
		buf.puts("|");
		return buf;
	}
}

// An entry in the abstract control stack.
class SpcControl {
	var opcode: u16;
	var params: Array<ValueType>;
	var results: Array<ValueType>;
	var reachable = true;
	var val_stack_top: u32;
	var label: MasmLabel;
	var else_label: MasmLabel;
	// the state at the merge (label)
	var merge_count: byte;
	var merge_state: Array<SpcVal>;
	// the state used to reset back to before the true branch of an if
	var reset_state: Array<SpcVal>;
	// the state (up to val_stack_top, excluding tag fields) to reset back to before catch
	// XXX: merge with {reset_state}?
	var catch_reset_state: Array<SpcVal>;
	// true if a SPC handler stub has already been recorded for this block
	var recorded_as_handler = false;

	def clearMerge() {
		merge_count = 0;
		merge_state = null;
	}
	def clearReset() {
		reset_state = null;
	}
	def copyMerge() -> Array<SpcVal> {
		return Arrays.dup(merge_state);
	}
}

def isNotZero = int.!=(0, _);
def trueToOne(z: bool) -> int { return if(z, 1, 0); }

// Contains both the abstract control and abstract value stack.
// Represents the compiler state for one function frame in the inlining stack.
class SpcFrame {
	var func: FuncDecl;
	var module: Module;
	var local_base_sp: u31;  // Base index into SpcState.state
	var ctl_base_sp: u31;    // Base index into SpcState.ctl_stack
	var num_locals: int;
	var pc: int;
	var ret_label: MasmLabel;

	new(func, module, local_base_sp, ctl_base_sp, num_locals, pc, ret_label) {}
}

class SpcState(regAlloc: RegAlloc) {
	// Abstract state of the value stack (shared across frames)
	var state = Array<SpcVal>.new(INITIAL_VALUE_STACK_SIZE);
	var sp: u32;
	var ctl_stack = ArrayStack<SpcControl>.new();
	// Frame management (for inlining)
	var frame_stack = ArrayStack<SpcFrame>.new();

	// Reset the state for starting a new function.
	def reset(sig: SigDecl, ret_label: MasmLabel) {
		sp = 0;
		ctl_stack.clear();
		// manually set up first control entry and return merge state
		var results = sig.results;
		var ctl = pushFuncBody(ValueTypes.NONE, results, ret_label);
		var merge_state = Array<SpcVal>.new(results.length);
		for (i < results.length) {
			// request the merged values be stored to the stack, but don't require tags
			merge_state[i] = SpcVal(typeToKindFlags(results[i]) | IS_STORED, NO_REG, 0);
		}
		ctl.merge_state = merge_state;
		ctl.merge_count = 1;
		// initialize parameters
		var params = sig.params;
		grow(params.length);
		for (i < params.length) {
			// params start on the stack and already have tags
			state[i] = SpcVal(typeToKindFlags(params[i]) | TAG_STORED | IS_STORED, NO_REG, 0);
		}
		sp = u32.view(params.length);
	}
	// Add the specified number of locals of the specified type.
	def addLocals(count: u32, ltype: ValueType) {
		var nlength = sp + count;
		if (nlength > state.length) grow(int.view(nlength + sp * 2));
		var flags = typeToKindFlags(ltype) | IS_CONST;
		if (SpcTuning.eagerTagLocals) flags |= TAG_STORED;
		for (j < count) {
			var k = j + sp;
			state[k] = SpcVal(flags, NO_REG, 0);
		}
		sp = nlength;
	}
	def pushBlock(params: Array<ValueType>, results: Array<ValueType>, end_label: MasmLabel) -> SpcControl {
		return pushControl(Opcode.BLOCK.code, params, results, end_label);
	}
	def pushFuncBody(params: Array<ValueType>, results: Array<ValueType>, end_label: MasmLabel) -> SpcControl {
		return pushControl(Opcode.RETURN.code, params, results, end_label);
	}
	def pushLoop(params: Array<ValueType>, results: Array<ValueType>, start_label: MasmLabel) -> SpcControl {
		var ctl = pushControl(Opcode.LOOP.code, params, results, start_label);
		return ctl;
	}
	def prepareLoop(resolver: SpcMoveResolver) {
		var target = ctl_stack.peek();
		target.merge_count = 1;
		var valcount = u32.view(target.params.length);
		var popcount = 0u;
		var num_locals = frame_stack.peek().num_locals;
		target.merge_state = doFirstTransferAndGetMerge(valcount, popcount, resolver);
		resolver.emitMoves();
		if (Trace.compiler) trace0("    loop merge = ", target.merge_state, target.merge_state.length, num_locals);
		resetTo(sp, target.merge_state);
	}
	def doFirstTransferAndGetMerge(valcount: u32, popcount: u32, resolver: SpcMoveResolver) -> Array<SpcVal> {
		var num_locals = frame_stack.peek().num_locals;
		return SpcMerger.new(state, num_locals, sp, regAlloc, resolver).createMerge(valcount, popcount);
	}
	// Get a merge that contains the current stack up to {val_stack_top} plus {args}, all stored in memory.
	def getInMemoryMergeWithArgs(val_stack_top: int, args: Array<ValueType>) -> Array<SpcVal> {
		var merge = Array<SpcVal>.new(val_stack_top + args.length);
		for (slot < val_stack_top) {
			var flags = state[slot].kindFlags(IS_STORED | TAG_STORED);
			merge[slot] = SpcVal(flags, NO_REG, 0);
		}
		for (i < args.length) {
			var slot = val_stack_top + i;
			merge[slot] = SpcVal(typeToKindFlags(args[i]) | TAG_STORED | IS_STORED, NO_REG, 0);
		}
		return merge;
	}
	def pushIf(params: Array<ValueType>, results: Array<ValueType>, else_label: MasmLabel, end_label: MasmLabel) -> SpcControl {
		var ctl = pushControl(Opcode.IF.code, params, results, end_label);
		ctl.else_label = else_label;
		ctl.reset_state = Arrays.dup(state);
		return ctl;
	}
	def doElse() {
		var ctl = ctl_stack.peek();
		ctl.else_label = null;
		// reset state to start of if
		var max = ctl.val_stack_top + u32.view(ctl.params.length);
		resetTo(max, ctl.reset_state);
		ctl.clearReset();
		if (ctl_stack.top > 1) ctl.reachable = ctl_stack.elems[ctl_stack.top - 2].reachable;
		else ctl.reachable = true;
	}
	def resetToMerge(ctl: SpcControl) {
		if (ctl.merge_count > 0) {
			var max = ctl.val_stack_top + u32.view(ctl.results.length);
			resetTo(max, ctl.merge_state);
		} else {
			// merge not reached; push "bottom" value for all results
			regAlloc.clear();
			sp = ctl.val_stack_top;
			for (i < sp) { // XXX: we clear register allocation state mostly for debug.
				var sv = state[i];
				if ((sv.flags & IN_REG) != 0) state[i] = SpcVal(sv.flags & ~IN_REG, NO_REG, 0);
			}
			for (r in ctl.results) {
				push(typeToKindFlags(r) | IS_STORED | TAG_STORED | IS_CONST, NO_REG, 0);
			}
		}
	}
	def isTransferEmpty(target: SpcControl) -> bool {
		return false; // XXX: approximate
	}
	def emitFallthru(resolver: SpcMoveResolver) {
		emitTransfer(ctl_stack.peek(), resolver);
	}
	def emitTransfer(target: SpcControl, resolver: SpcMoveResolver) {
		if (!ctl_stack.peek().reachable) {
			if (Trace.compiler) OUT.puts("    xfer not reachable").ln();
			return; // do nothing
		}
		var valcount = u32.view(if(target.opcode == Opcode.LOOP.code, target.params, target.results).length);
		var top = target.val_stack_top, max = top + valcount;
		if (Trace.compiler) OUT.put3("    xfer -> sp=%d vals=%d merges=%d", top, valcount, target.merge_count).ln();
		if (target.merge_count == 0) {
			target.merge_count = 1;
			var popcount = sp - valcount - target.val_stack_top;
			var num_locals = frame_stack.peek().num_locals;
			target.merge_state = doFirstTransferAndGetMerge(valcount, popcount, resolver);
			if (Trace.compiler) trace0("    merge = ", target.merge_state, target.merge_state.length, num_locals);
		} else {
			target.merge_count = 2;
			// XXX: allow matching constants in merges
			for (i < top) {
				var from = state[i], to = target.merge_state[i];
				if (from != to) resolver.addMove((i, to), (i, from));
			}
			for (i < valcount) {
				var f = (sp - valcount + i), t = top + i;
				var from = state[f], to = target.merge_state[t];
				resolver.addMove((t, to), (f, from));
			}
		}
		resolver.emitMoves();
	}
	def emitSaveAll(resolver: SpcMoveResolver, spillMode: SpillMode) {
		if (Trace.compiler) OUT.puts("    save all").ln();
		def STORE_MASK = IS_STORED | TAG_STORED;
		for (slot < sp) {
			var sv = state[slot], nv = sv;
			if (sv.inReg() && spillMode.free_regs) {
				nv = nv.withoutReg();
				regAlloc.free(sv.reg);
			}
			var stored = nv.withFlags(STORE_MASK);
			if ((sv.flags & STORE_MASK) != STORE_MASK) resolver.addMove((slot, stored), (slot, sv));
			if (spillMode.remember_stored) nv = stored;
			state[slot] = nv;
		}
		resolver.emitMoves();
	}
	def emitRestoreAll(resolver: SpcMoveResolver) {
		if (Trace.compiler) OUT.puts("    restore all").ln();
		for (slot < sp) {
			var sv = state[slot];
			if (!sv.inReg()) continue;
			var fv = SpcVal(sv.kindFlags(IS_STORED), NO_REG, sv.const);
			var tv = SpcVal(sv.kindFlags(IN_REG), sv.reg, sv.const);
			resolver.addMove((slot, tv), (slot, fv));
		}
		resolver.emitMoves();
	}
	private def toMergeVal(from: SpcVal) -> SpcVal {
		// XXX: allow constants in merges
		var force_store = if(!from.inReg(), IS_STORED);
		return SpcVal((from.flags & ~(IS_CONST)) | force_store, from.reg, 0);
	}
	private def resetTo(max: u32, nstate: Array<SpcVal>) {
		regAlloc.clear();
		for (i < max) {
			var sv = nstate[i];
			if (sv.inReg()) regAlloc.assign(sv.reg, int.!(i));
			state[i] = sv;
		}
		sp = max;
	}
	private def pushControl(opcode: u16, params: Array<ValueType>, results: Array<ValueType>, label: MasmLabel) -> SpcControl {
		// XXX: reimplement {SpcControl} caching without breaking handler links
		var reachable = if(ctl_stack.top > 0, ctl_stack.peek().reachable, true);
		var ctl = SpcControl.new();
		ctl_stack.push(ctl);
		ctl.opcode = opcode;
		ctl.label = label;
		ctl.params = params;
		ctl.results = results;
		ctl.val_stack_top = sp - u32.view(params.length);
		ctl.reachable = reachable;
		ctl.merge_count = 0;
		return ctl;
	}
	def getControl(depth: u32) -> SpcControl {
		var result = ctl_stack.elems[ctl_stack.top - int.!(depth) - 1];
		return result;
	}
	def push(flags: byte, reg: Reg, const: int) {
		var sp = this.sp;
		if (sp >= state.length) grow(8 + state.length * 2);
		state[sp] = SpcVal(flags, reg, const);
		this.sp++;
	}
	def pushV(v: SpcVal) {
		var sp = this.sp;
		if (sp >= state.length) grow(8 + state.length * 2);
		state[sp] = v;
		this.sp++;
		// TODO: check that if .reg is non-zero, IN_REG is true
		// TODO: check that if .reg is non-zero, regalloc assignment includes top of stack
	}
	def get(slot: u32) -> SpcVal {
		return state[slot];
	}
	def pop() -> SpcVal {
		if (sp == 0) {
			if (Trace.compiler) Trace.OUT.puts("WARNING: stack underflow in SPC").ln(); // TODO: get to errorgen
			var d: SpcVal;
			return d;
		}
		var result = state[--this.sp];
		return result;
	}
	def popArgsAndPushResults(sig: SigDecl) {
		sp -= u32.view(sig.params.length); // note: assume registers have been freed
		for (t in sig.results) {
			push(typeToKindFlags(t) | TAG_STORED | IS_STORED, NO_REG, 0);
		}
	}
	def pushResults(results: Range<ValueType>) {
		for (t in results) {
			push(typeToKindFlags(t) | TAG_STORED | IS_STORED, NO_REG, 0);
		}
	}
	def peek() -> SpcVal {
		return state[sp - 1];
	}
	def peek2() -> (SpcVal, SpcVal) {
		return (state[sp - 2], state[sp - 1]);
	}
	def overwrite(flags: byte, reg: Reg, const: int) {
		var old = state[sp - 1];
		var tag_stored = if((old.flags & KIND_MASK) == (flags & KIND_MASK), old.flags & TAG_STORED);
		state[sp - 1] = SpcVal(tag_stored | flags, reg, const);
	}
	def set(slot: u32, flags: byte, reg: Reg, const: int) {
		state[slot] = SpcVal(flags, reg, const);
	}
	def setStored(slot: u32) {
		var b = state[slot];
		state[slot] = SpcVal(b.flags | IS_STORED, b.reg, b.const);
	}
	def setUnreachable() {
		ctl_stack.peek().reachable = false;
	}
	def grow(nlength: int) {
		state = Arrays.grow(state, nlength);
	}
	def trace() {
		var num_locals = frame_stack.peek().num_locals;
		trace0("", state, int.!(sp), num_locals);
	}
	def trace0(str: string, vals: Array<SpcVal>, sp: int, num_locals: int) {
		OUT.puts(str);
		OUT.puts("|");
		// Don't assume vals.length >= num_locals (e.g., for implicit function blocks in inlined functions)
		var trace_limit = if(num_locals < vals.length, num_locals, vals.length);
		for (i < trace_limit) vals[i].renderTrace(OUT, regAlloc.poolMap.regSet);
		OUT.puts("  |");
		for (i = num_locals; i < sp && i < vals.length; i++) vals[i].renderTrace(OUT, regAlloc.poolMap.regSet);
		OUT.ln();
	}
}
def typeToKindFlags(vt: ValueType) -> byte {
	match (vt) {
		I32 => return KIND_I32;
		I64 => return KIND_I64;
		F32 => return KIND_F32;
		F64 => return KIND_F64;
		V128 => return KIND_V128;
		Ref(_, ht) => return if (HeapType.Cont.?(ht), KIND_REF_U64, KIND_REF);
		_ => return KIND_REF;
	}
}
def tagToKindFlags(t: byte) -> byte {
	match (t) {
		BpTypeCode.I8.code,
		BpTypeCode.I16.code,
		BpTypeCode.I32.code => return KIND_I32;
		BpTypeCode.I64.code => return KIND_I64;
		BpTypeCode.F32.code => return KIND_F32;
		BpTypeCode.F64.code => return KIND_F64;
		BpTypeCode.V128.code => return KIND_V128;
		_ => return KIND_REF;
	}
}
class SpcMerger(state: Array<SpcVal>, num_locals: int, sp: u32, regAlloc: RegAlloc, resolver: SpcMoveResolver) {
	var valcount: u32;
	var popcount: u32;
	var merge: Array<SpcVal>;

	// Prepare for the first transfer to a label by revoking registers/constants and emitting
	// any necessary moves. Here, {L} refers to locals, {Z} to the frozen portion of the operand
	// stack, {V} to the {valcount} number of arguments to the label, and {P} to the {popcount} number
	// of values that will be popped. Neither the current state nor the register allocation state is
	// updated; the new target merge state is returned.
	//   | L | Z | P | V |
	//     |           |
	//     |       +---+
	//     |       |
	//     v       v
	//   |!L!| Z |!V!|
	def createMerge(valcount: u32, popcount: u32) -> Array<SpcVal> {
		this.valcount = valcount;
		this.popcount = popcount;
		merge = Arrays.range(state, 0, int.view(sp - popcount));
		// process transfer slots
		var end = (sp - valcount - popcount);
		for (i < valcount) {
			var fi = sp - i - 1, ti = fi - popcount;
			var from = state[fi], to = merge[ti];
			if (from.reg != NO_REG && to.reg == from.reg) {
				// A register used in the range can only be used once. Spill uses in L and V.
				if (regAlloc.frequency(from.reg) > 1) {
					revokeRegFromRange(from.reg, fi, 0, int.!(end));
					revokeRegFromRange(from.reg, fi, int.!(sp - valcount), int.!(fi));
				}
				to = SpcVal(from.kindFlagsAndTag(IN_REG), from.reg, 0);
			} else {
				to = SpcVal(from.kindFlagsAndTag(IS_STORED), NO_REG, 0);
				resolver.addMove((ti, to), (fi, from));
			}
			merge[ti] = to;
		}
		// process L and Z
		for (fi: u32 < end) {
			var ti = fi;
			var from = state[fi], to = merge[ti];
			if (from.reg != NO_REG && to.reg == from.reg) {
				// A register used in the range can only be used once. Spill uses in L and V.
				if (regAlloc.frequency(from.reg) > 1) {
					revokeRegFromRange(from.reg, fi, int.!(fi + 1), int.!(end));
					revokeRegFromRange(from.reg, fi, int.!(sp - valcount), int.!(sp));
				}
				to = SpcVal(from.kindFlagsAndTag(IN_REG), from.reg, 0);
			} else {
				to = SpcVal(from.kindFlagsAndTag(IS_STORED), NO_REG, 0);
				resolver.addMove((ti, to), (fi, from));
			}
			merge[ti] = to;
		}
		return merge;
	}
	def revokeRegFromRange(reg: Reg, slot: u32, start: int, end: int) {
		if (Trace.compiler) {
			OUT.put2("      revoke fi=%d r=%s", slot, regAlloc.poolMap.regSet.getName(reg))
				.put2(" [%d...%d)", start, end)
				.ln();
		}
		regAlloc.forEachAssignmentInRange(reg, start, end, revoke);
	}
	def revoke(index: int) {
		if (Trace.compiler) OUT.put3("      revoke %d (valcount=%d, popcount=%d)", index, valcount, popcount).ln();
		if (index >= (sp - valcount)) index -= int.!(popcount);
		merge[index] = SpcVal(merge[index].flags & ~IN_REG, NO_REG, 0);
	}
}

// Collects the parallel moves and orders them properly.
// Reused repeatedly by the compiler, so nodes are recycled internally.
class SpcMoveResolver(masm: MacroAssembler) {
	private var moves = Array<MoveNode>.new(16);
	private var numMoves = 0;
	private def constRegMoves = Vector<(ValueKind, int, Reg)>.new();
	private def constSlotMoves = Vector<(ValueKind, int, u32)>.new();

	// Add a move or moves to the collection of parallel moves to transfer the
	// abstract value in one slot to another. The abstract value may be a constant,
	// in a register, or in a slot.
	// Generates a value tag store if necessary.
	def addMove(to: (u32, SpcVal), from: (u32, SpcVal)) {
		if (Trace.compiler) {
			OUT.put1("    addMove slot=%d ", to.0);
			to.1.render(OUT, masm.regConfig.regSet);
			OUT.put1(" <- slot=%d ", from.0);
			from.1.render(OUT, masm.regConfig.regSet);
			OUT.ln();
		}
		var tv = to.1, fv = from.1;
		if (masm.valuerep.tagged && tv.tagStored()) { // XXX: move tag store operation into state?
			if (from.0 != to.0 || !fv.tagStored()) {
				// store the tag into to-slot
				masm.emit_mov_m_i(masm.tagAddr(to.0), fv.kind().code);
			}
		}
		if (fv.isConst()) {
			if (tv.isStored()) constSlotMoves.put(fv.kind(), fv.const, to.0);
			if (tv.inReg()) constRegMoves.put(fv.kind(), fv.const, tv.reg);
			return;
		}
		var kind = fv.kind();
		if (tv.isStored()) {
			// store the value into slot
			if (from.0 != to.0 || !fv.isStored()) {
				if (fv.inReg()) mov(kind, s(to.0), r(fv.reg));
				else mov(kind, s(to.0), s(from.0));
			}
		}
		if (tv.inReg()) {
			// load or move the value into appropriate register
			if (fv.inReg()) {
				if (tv.reg != fv.reg) mov(kind, r(tv.reg), r(fv.reg));
			} else {
				mov(kind, r(tv.reg), s(from.0));
			}
		}
	}
	// Order and emit the moves to the macro assembler.
	def emitMoves() {
		for (i < numMoves) {
			var m = moves[i];
			if (m.color == 0) orderMove(null, m);
		}
		numMoves = 0; // resize back to 0, but reuse node objects

		for (i < constRegMoves.length) {
			var m = constRegMoves[i];
			masm.emit_mov_r_k(m.0, m.2, m.1);
		}
		constRegMoves.resize(0);
		for (i < constSlotMoves.length) {
			var m = constSlotMoves[i];
			masm.emit_mov_s_k(m.0, m.2, m.1);
		}
		constSlotMoves.resize(0);
		constRegMoves.resize(0);
	}
	private def mov(kind: ValueKind, dst: MoveNode, src: MoveNode) {
		dst.kind = kind;
		dst.dstNext = src.dstList;
		src.dstList = dst;
		dst.src = src;
	}
	private def s(slot: u32) -> MoveNode {
		for (i < numMoves) { // XXX: avoid linear search for matching MoveNode
			var m = moves[i];
			if (m.slot == slot && m.reg == NO_REG) return m;
		}
		var m = allocMove();
		m.slot = slot;
		m.reg = NO_REG;
		return m;
	}
	private def r(reg: Reg) -> MoveNode {
		for (i < numMoves) { // XXX: avoid linear search for matching MoveNode
			var m = moves[i];
			if (m.reg == reg) return m;
		}
		var m = allocMove();
		m.slot = 0;
		m.reg = reg;
		return m;
	}
	private def allocMove() -> MoveNode {
		if (numMoves >= moves.length) moves = Arrays.grow(moves, moves.length * 2);
		var m = moves[numMoves++];
		if (m == null) {
			m = moves[numMoves - 1] = MoveNode.new();
			m.kind = ValueKind.REF; // TODO
			return m;
		}
		m.kind = ValueKind.REF;
		m.src = null;
		m.dstList = null;
		m.dstNext = null;
		m.color = 0;
		return m;
	}
	// depth-first recursive traversal of the move graph, inserting moves in post-order
	private def orderMove(alloc: int -> int, node: MoveNode) {
		if (node.color == BLACK) return;
		node.color = GRAY;
		for (l = node.dstList; l != null; l = l.dstNext) {
			if (l.color == GRAY) {
				// cycle detected; break it with temporary register
				var tmp = allocTmp(l.kind);
				emitMove(tmp, l);		// save
				emitMove(l, node);		// overwrite
				l.reg = tmp.reg;		// on-stack uses will use tmp
			} else {
				orderMove(alloc, l);
			}
		}
		node.color = BLACK;
		if (node.src != null) emitMove(node, node.src);
	}
	private def allocTmp(kind: ValueKind) -> MoveNode {
		var m = MoveNode.new(); // XXX: could reuse tmp by kind
		m.kind = kind;
		m.reg = masm.getScratchReg(kind);
		return m;
	}
	private def emitMove(dst: MoveNode, src: MoveNode) {
		if (dst.reg == NO_REG) {
			if (src.reg == NO_REG) {
				if (Trace.compiler) OUT.put3("    emitMove[%s] slot=%d <- slot=%d", dst.kind.name, dst.slot, src.slot).ln();
				masm.emit_mov_s_s(dst.kind, dst.slot, src.slot);
			} else {
				if (Trace.compiler) OUT.put3("    emitMove[%s] slot=%d <- %s", dst.kind.name, dst.slot, name(src.reg)).ln();
				masm.emit_mov_s_r(dst.kind, dst.slot, src.reg);
			}
		} else {
			if (src.reg == NO_REG) {
				if (Trace.compiler) OUT.put3("    emitMove[%s] %s <- slot=%d", dst.kind.name, name(dst.reg), src.slot).ln();
				masm.emit_mov_r_s(dst.kind, dst.reg, src.slot);
			} else {
				if (Trace.compiler) OUT.put3("    emitMove[%s] %s <- %s", dst.kind.name, name(dst.reg), name(src.reg)).ln();
				masm.emit_mov_r_r(dst.kind, dst.reg, src.reg);
			}
		}
	}
	def name(r: Reg) -> string {
		return masm.regConfig.regSet.getName(r);
	}
}

// Nodes used in the internal move graph, which can be either a register or a value slot.
// Due to the nature of parallel moves, each node can be the destination of at most one
// move. Thus the {kind} of the node actually represents the kind of destination.
def GRAY    = '\x01';
def BLACK   = '\x02';
class MoveNode {
	var slot: u32;		// slot, if reg == REG_NONE
	var reg: Reg;		// register, REG_NONE if slot
	var kind: ValueKind;	// value kind of destination
	var color: byte;	// used in traversing the graph
	var src: MoveNode;	// source of the value for this node
	var dstList: MoveNode;	// head of destination list
	var dstNext: MoveNode;	// next in a list of successors
}

type WhammInlineConfig(swap_membase: bool, swap_instance: bool, is_inlined: bool);

// Used to record the entry point of exception/suspension handlers. Jumping to {stub_label} allows
// control transfer to its corresponding handler without falling back to fast-int.
//
// The SPC emits a stub at {stub_label} for each handler in the function. The stub restores the
// expected state of the environment, then jumps to {dest_label} to continue execution at handler.
type SpcHandlerInfo(is_dummy: bool, func_end: bool, dest_label: MasmLabel, stub_label: MasmLabel, merge_state: Array<SpcVal>);