[wide-arithmetic] Implement Phase 4 proposal

README.md

@@ -28,6 +28,7 @@ Newer features are under development and vary in their support in the different
 | memory64 | ✅| ✅ | ✅ | ✅ |
 | stack-switching  | ✅| ✅ | ✅ |  ✅ |
 | custom-page-sizes  | ✅| ✅ |  |  |
+| wide-arithmetic  | ✅| ✅ | ✅ | ✅ |
 | legacy EH  | ✅| ✅ |  ☑ | ☑ |
 | threads  | ✅| ☑ | ☑ |  |
 

src/engine/BytecodeIterator.v3

@@ -450,6 +450,10 @@ class BytecodeIterator {
 			TABLE_GROW =>			v.visit_TABLE_GROW(read_TABLE());
 			TABLE_SIZE =>			v.visit_TABLE_SIZE(read_TABLE());
 			TABLE_FILL =>			v.visit_TABLE_FILL(read_TABLE());
+			I64_ADD128 =>			v.visit_I64_ADD128();
+			I64_SUB128 =>			v.visit_I64_SUB128();
+			I64_MUL_WIDE_S =>		v.visit_I64_MUL_WIDE_S();
+			I64_MUL_WIDE_U =>		v.visit_I64_MUL_WIDE_U();
 			V128_LOAD =>			v.visit_V128_LOAD(read_MEMARG());
 			V128_LOAD_8X8_S =>		v.visit_V128_LOAD_8X8_S(read_MEMARG());
 			V128_LOAD_8X8_U =>		v.visit_V128_LOAD_8X8_U(read_MEMARG());

src/engine/CodeValidator.v3

@@ -1109,6 +1109,10 @@ class CodeValidator(extensions: Extension.set, limits: Limits, module: Module, e
 					popE(table.elemtype);
 					popE(it);
 				}
+				I64_ADD128, I64_SUB128, I64_MUL_WIDE_S, I64_MUL_WIDE_U => {
+					if (!checkExtension(Extension.WIDE_ARITHMETIC, opcode)) return;
+					checkSignature(opcode.sig);
+				}
 				V128_LOAD_8_SPLAT => checkLoad(opcode, 0, ValueType.V128);
 				V128_LOAD_16_SPLAT => checkLoad(opcode, 1, ValueType.V128);
 				V128_LOAD_32_SPLAT => checkLoad(opcode, 2, ValueType.V128);

src/engine/Extension.v3

@@ -16,6 +16,7 @@ enum Extension(short_name: string, help: string) {
 	CUSTOM_PAGE_SIZES("custom-page-sizes", "Custom page sizes"),
 	EXTENDED_CONST("extended-const", "Extended constant expressions"),
 	RELAXED_SIMD("relaxed-simd", "Relaxed SIMD"),
+	WIDE_ARITHMETIC("wide-arithmetic", "128-bit wide arithmetic"),
 	WASM_3_0("wasm-3.0", "All Wasm 3.0 features"),
 	JIT_INTERFACE("jit-interface", "Runtime code generation with func.new"),
 	WIZENG("wizeng", "Wizard-specific engine capabilities"),

src/engine/Opcodes.v3

@@ -276,6 +276,11 @@ enum Opcode(prefix: u8, code: u16, mnemonic: string, imms: Array<ImmKind>, sig:
 	TABLE_GROW		(0xFC, 0x0F, "table.grow", imm.TABLE, null),
 	TABLE_SIZE		(0xFC, 0x10, "table.size", imm.TABLE, sig.v_i),
 	TABLE_FILL		(0xFC, 0x11, "table.fill", imm.TABLE, null),
+	// 0xFC prefix: wide arithmetic.
+	I64_ADD128		(0xFC, 0x13, "i64.add128", imm.NONE, sig.llll_ll),
+	I64_SUB128		(0xFC, 0x14, "i64.sub128", imm.NONE, sig.llll_ll),
+	I64_MUL_WIDE_S		(0xFC, 0x15, "i64.mul_wide_s", imm.NONE, sig.ll_ll),
+	I64_MUL_WIDE_U		(0xFC, 0x16, "i64.mul_wide_u", imm.NONE, sig.ll_ll),
 	// 0xFD prefix: vector instructions.
 	V128_LOAD			(0xFD, 0, "v128.load", imm.MEMARG, sig.i_s),
 	V128_LOAD_8X8_S			(0xFD, 1, "v128.load8x8_s", imm.MEMARG, sig.i_s),

src/engine/SigCache.v3

@@ -44,6 +44,7 @@ component SigCache {
 	def arr_iig: Array<ValueType> = [ValueType.I32, ValueType.I32, ValueTypes.FUNCREF];
 
 	def arr_ll: Array<ValueType> = [ValueType.I64, ValueType.I64];
+	def arr_llll: Array<ValueType> = [ValueType.I64, ValueType.I64, ValueType.I64, ValueType.I64];
 	def arr_l: Array<ValueType> = [ValueType.I64];
 
 	def arr_ff: Array<ValueType> = [ValueType.F32, ValueType.F32];
@@ -133,6 +134,8 @@ component SigCache {
 	def l_d = S(arr_l, arr_d);
 	def ll_l = S(arr_ll, arr_l);
 	def ll_i = S(arr_ll, arr_i);
+	def ll_ll = S(arr_ll, arr_ll);
+	def llll_ll = S(arr_llll, arr_ll);
 
 	def f_i = S(arr_f, arr_i);
 	def f_l = S(arr_f, arr_l);

src/engine/V3Eval.v3

@@ -531,6 +531,37 @@ component V3Eval {
 	def F32_REINTERPRET_I32	= float.view<u32>;
 	def F64_REINTERPRET_I64	= double.view<u64>;
 
+	// ---- wide (128-bit) arithmetic --------------------------------------
+	// All of these take and return 128-bit quantities as (low, high) pairs of 64-bit halves.
+	def I64_ADD128(alo: u64, ahi: u64, blo: u64, bhi: u64) -> (u64, u64) {
+		var lo = alo + blo;
+		var carry: u64 = if(lo < alo, 1, 0);
+		return (lo, ahi + bhi + carry);
+	}
+	def I64_SUB128(alo: u64, ahi: u64, blo: u64, bhi: u64) -> (u64, u64) {
+		var lo = alo - blo;
+		var borrow: u64 = if(alo < blo, 1, 0);
+		return (lo, ahi - bhi - borrow);
+	}
+	def I64_MUL_WIDE_U(x: u64, y: u64) -> (u64, u64) {
+		var x0 = u64.!(u32.view(x)), x1 = x >> 32;
+		var y0 = u64.!(u32.view(y)), y1 = y >> 32;
+		var p00 = x0 * y0, p01 = x0 * y1, p10 = x1 * y0, p11 = x1 * y1;
+		var mid = (p00 >> 32) + u64.!(u32.view(p01)) + u64.!(u32.view(p10));
+		var lo = u64.!(u32.view(p00)) | (mid << 32);
+		var hi = p11 + (p01 >> 32) + (p10 >> 32) + (mid >> 32);
+		return (lo, hi);
+	}
+	def I64_MUL_WIDE_S(x: i64, y: i64) -> (u64, u64) {
+		var t = I64_MUL_WIDE_U(u64.view(x), u64.view(y));
+		var hi = t.1;
+		// Fix up the unsigned result: subtract each operand that was interpreted
+		// as 2^64 too large from the upper half of the product.
+		if (x < 0) hi = hi - u64.view(y);
+		if (y < 0) hi = hi - u64.view(x);
+		return (t.0, hi);
+	}
+
 	// ---- sign-extension and zero-extension helpers ----------------------
 	def I32_EXTEND8_S(a: i32) -> i32 {
 		return i8.view(a);

src/engine/v3/V3Interpreter.v3

@@ -981,6 +981,10 @@ class V3Interpreter extends WasmStack {
 				var table_index = codeptr.read_uleb31();
 				Runtime.TABLE_FILL(this, frame.func.instance, table_index);
 			}
+			I64_ADD128 =>		do_wwww_ww(V3Eval.I64_ADD128);
+			I64_SUB128 =>		do_wwww_ww(V3Eval.I64_SUB128);
+			I64_MUL_WIDE_S =>	do_ll_ww(V3Eval.I64_MUL_WIDE_S);
+			I64_MUL_WIDE_U =>	do_ww_ww(V3Eval.I64_MUL_WIDE_U);
 			V128_LOAD => 		doLoad(ACCESS_V128, pushs);
 			V128_LOAD_64_LANE => 	doLoadLane(ACCESS_U64);
 			V128_LOAD_32_LANE => 	doLoadLane(ACCESS_U32);
@@ -1962,6 +1966,30 @@ class V3Interpreter extends WasmStack {
 		values.elems[values.top-2] = Value.I64(r);
 		values.top--;
 	}
+	def do_wwww_ww(f: (u64, u64, u64, u64) -> (u64, u64)) {
+		var bhi = Values.unbox_w(values.elems[values.top-1]);
+		var blo = Values.unbox_w(values.elems[values.top-2]);
+		var ahi = Values.unbox_w(values.elems[values.top-3]);
+		var alo = Values.unbox_w(values.elems[values.top-4]);
+		var r = f(alo, ahi, blo, bhi);
+		values.elems[values.top-4] = Value.I64(r.0);
+		values.elems[values.top-3] = Value.I64(r.1);
+		values.top -= 2;
+	}
+	def do_ww_ww(f: (u64, u64) -> (u64, u64)) {
+		var y = Values.unbox_w(values.elems[values.top-1]);
+		var x = Values.unbox_w(values.elems[values.top-2]);
+		var r = f(x, y);
+		values.elems[values.top-2] = Value.I64(r.0);
+		values.elems[values.top-1] = Value.I64(r.1);
+	}
+	def do_ll_ww(f: (i64, i64) -> (u64, u64)) {
+		var y = Values.unbox_l(values.elems[values.top-1]);
+		var x = Values.unbox_l(values.elems[values.top-2]);
+		var r = f(x, y);
+		values.elems[values.top-2] = Value.I64(r.0);
+		values.elems[values.top-1] = Value.I64(r.1);
+	}
 	def do_ww_wt(f: (u64, u64) -> (u64, TrapReason)) {
 		var y = Values.unbox_w(values.elems[values.top-1]);
 		var x = Values.unbox_w(values.elems[values.top-2]);

src/engine/x86-64/X86_64Interpreter.v3

@@ -327,7 +327,7 @@ class X86_64InterpreterGen(ic: X86_64InterpreterCode, w: DataWriter) {
 
 	def k_frame_size	= X86_64InterpreterFrame.size;
 
-	def vsph = VspHelper.new(r_vsp, valuerep, 3);
+	def vsph = VspHelper.new(r_vsp, valuerep, 4);
 
 	def dispatchTables = Array<(byte, IcCodeRef, IcCodeRef, byte, IcCodeRef)>.new(
 				Opcodes.code_pages.length + Opcodes.num_subpages + 1);
@@ -926,6 +926,7 @@ class X86_64InterpreterGen(ic: X86_64InterpreterCode, w: DataWriter) {
 		genCompares();
 		genI32Arith();
 		genI64Arith();
+		genWideArith();
 		genExtensions();
 		genF32Arith();
 		genF64Arith();
@@ -1998,6 +1999,40 @@ class X86_64InterpreterGen(ic: X86_64InterpreterCode, w: DataWriter) {
 	def genAtomicCompareAndExchange<T>(cmpxchg: (X86_64Addr, X86_64Gpr) -> T) {
 		genAtomicOp(cmpxchg, null, null, true);
 	}
+	// ext:wide-arithmetic. Operands and results are (low, high) pairs of i64 slots,
+	// with the low half deeper in the stack.
+	def genWideArith() {
+		for (t in [
+			(Opcode.I64_ADD128, asm.q.add_r_m, asm.q.adc_r_m),
+			(Opcode.I64_SUB128, asm.q.sub_r_m, asm.q.sbb_r_m)
+		]) {
+			bindHandler(t.0);
+			asm.movq_r_m(r_tmp0, vsph[-4].value);	// a.low
+			asm.movq_r_m(r_tmp1, vsph[-3].value);	// a.high
+			t.1(r_tmp0, vsph[-2].value);		// low halves, setting carry/borrow
+			t.2(r_tmp1, vsph[-1].value);		// high halves, consuming carry/borrow
+			asm.movq_m_r(vsph[-4].value, r_tmp0);
+			asm.movq_m_r(vsph[-3].value, r_tmp1);
+			adjustVsp(-2);
+			endHandler();
+		}
+		for (t in [
+			(Opcode.I64_MUL_WIDE_S, asm.q.imul_r),
+			(Opcode.I64_MUL_WIDE_U, asm.q.mul_r)
+		]) {
+			bindHandler(t.0);
+			asm.movq_r_m(r_tmp0, vsph[-1].value);	// {r_tmp1} is RDX, so use RCX
+			masm.emit_spill_ivar(X86_64MasmRegs.RAX);
+			masm.emit_spill_ivar(X86_64MasmRegs.RDX);
+			asm.movq_r_m(R.RAX, vsph[-2].value);
+			t.1(r_tmp0);				// RDX:RAX = RAX * RCX
+			asm.movq_m_r(vsph[-2].value, R.RAX);	// low
+			asm.movq_m_r(vsph[-1].value, R.RDX);	// high
+			masm.emit_restore_ivar(X86_64MasmRegs.RAX);
+			masm.emit_restore_ivar(X86_64MasmRegs.RDX);
+			endHandler();
+		}
+	}
 	def genExtensions() {
 		bindHandler(Opcode.I32_WRAP_I64); {
 			genTagUpdate(BpTypeCode.I32.code);

src/engine/x86-64/X86_64SinglePassCompiler.v3

@@ -230,6 +230,33 @@ class X86_64SinglePassCompiler extends SinglePassCompiler {
 	def visit_I64_OR()  { visitSimpleOp2_ww_w(V3Eval.I64_OR, asm.q.or_r_i, asm.q.or_r_m, asm.q.or_r_r); }
 	def visit_I64_XOR() { visitSimpleOp2_ww_w(V3Eval.I64_XOR, asm.q.xor_r_i, asm.q.xor_r_m, asm.q.xor_r_r); }
 
+	// ext:wide-arithmetic. Both operands and results are (low, high) pairs of i64
+	// values, with the low half deeper on the stack.
+	private def visitWideAddSub(emit_lo: (X86_64Gpr, X86_64Gpr) -> X86_64Assembler,
+		emit_hi: (X86_64Gpr, X86_64Gpr) -> X86_64Assembler) {
+		var bhi = popReg(), blo = popReg();
+		var ahi = popRegToReuse(0), alo = popRegToReuse(0); // reused in place as the results
+		emit_lo(G(alo.reg), G(blo.reg)); // sets carry/borrow
+		emit_hi(G(ahi.reg), G(bhi.reg)); // consumes carry/borrow
+		state.push(alo.kindFlagsAndTag(IN_REG), alo.reg, 0);
+		state.push(ahi.kindFlagsAndTag(IN_REG), ahi.reg, 0);
+	}
+	def visit_I64_ADD128() { visitWideAddSub(asm.q.add_r_r, asm.q.adc_r_r); }
+	def visit_I64_SUB128() { visitWideAddSub(asm.q.sub_r_r, asm.q.sbb_r_r); }
+	private def visitMulWide(emit: X86_64Gpr -> X86_64Assembler) {
+		var b = popFixedReg(X86_64MasmRegs.RCX); // XXX: use any reg except RAX or RDX
+		var a = popFixedReg(X86_64MasmRegs.RAX);
+		spillRegAndFree(X86_64MasmRegs.RAX);
+		spillRegAndFree(X86_64MasmRegs.RDX);
+		emit(G(b.reg)); // RDX:RAX = RAX * b
+		regAlloc.assign(X86_64MasmRegs.RAX, int.!(state.sp));
+		state.push(a.kindFlagsAndTag(IN_REG), X86_64MasmRegs.RAX, 0);
+		regAlloc.assign(X86_64MasmRegs.RDX, int.!(state.sp));
+		state.push(b.kindFlagsAndTag(IN_REG), X86_64MasmRegs.RDX, 0); // {b}'s slot, so {b}'s tag state
+	}
+	def visit_I64_MUL_WIDE_S() { visitMulWide(asm.q.imul_r); }
+	def visit_I64_MUL_WIDE_U() { visitMulWide(asm.q.mul_r); }
+
 	// XXX: try s_m addressing mode for floating point ops
 	def visit_F32_ABS() {
 		var sv = popReg(), r = X(sv.reg), scratch = mmasm.scratch;

src/util/BytecodeVisitor.v3

@@ -319,6 +319,11 @@ class BytecodeVisitor {
 	def visit_TABLE_GROW		(table_index: u31)		{ visitTable(Opcode.TABLE_GROW, table_index); }
 	def visit_TABLE_SIZE		(table_index: u31)		{ visitTable(Opcode.TABLE_SIZE, table_index); }
 	def visit_TABLE_FILL		(table_index: u31)		{ visitTable(Opcode.TABLE_FILL, table_index); }
+	// FC prefix: wide arithmetic.
+	def visit_I64_ADD128		() { visitBinop(Opcode.I64_ADD128); }
+	def visit_I64_SUB128		() { visitBinop(Opcode.I64_SUB128); }
+	def visit_I64_MUL_WIDE_S	() { visitBinop(Opcode.I64_MUL_WIDE_S); }
+	def visit_I64_MUL_WIDE_U	() { visitBinop(Opcode.I64_MUL_WIDE_U); }
 	// FD prefix: vector instructions.
 	def visit_V128_LOAD			(imm: MemArg) { visitLoad(Opcode.V128_LOAD, imm, 16); }
 	def visit_V128_LOAD_8X8_S		(imm: MemArg) { visitLoad(Opcode.V128_LOAD_8X8_S, imm, 8); }