x64: lower isub by a constant to lea

cranelift/codegen/src/isa/x64/inst.isle

@@ -621,6 +621,12 @@
 (decl zero_offset () Offset32)
 (extern constructor zero_offset zero_offset)
 
+;; Negate a constant `i32` into an `Offset32`, used to fold `x - C` into an
+;; `lea -C(x)`. Fails (partial) when the negation overflows, i.e. `C` is
+;; `i32::MIN`.
+(decl pure partial neg_i32_to_offset32 (i32) Offset32)
+(extern constructor neg_i32_to_offset32 neg_i32_to_offset32)
+
 ;; Shift kinds.
 
 (type ShiftKind extern

cranelift/codegen/src/isa/x64/lower.isle

@@ -266,6 +266,15 @@
 (rule -3 (lower (isub (fits_in_64 ty) x y))
       (x64_sub ty x y))
 
+;; `x - C` for a constant `C` lowers to `lea -C(x)`, mirroring the `iadd` lea
+;; lowering. `lea` reads `x` and writes the result directly, avoiding the
+;; register copy the two-operand `sub` needs when the destination differs from
+;; `x`. Fires only for 32/64-bit types (which `lea` supports) and when `-C`
+;; fits in an `Offset32`.
+(rule isub_const_lea -2 (lower (isub (ty_32_or_64 ty) x (i32_from_iconst c)))
+      (if-let neg_off (neg_i32_to_offset32 c))
+      (x64_lea ty (to_amode (mem_flags_trusted_data) x neg_off)))
+
 ;; SSE.
 
 (rule (lower (isub (multi_lane 8 16) x y))

cranelift/codegen/src/isa/x64/lower/isle.rs

@@ -1402,6 +1402,11 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
         Offset32::new(0)
     }
 
+    #[inline]
+    fn neg_i32_to_offset32(&mut self, c: i32) -> Option<Offset32> {
+        c.checked_neg().map(Offset32::new)
+    }
+
     #[inline]
     fn preg_rbp(&mut self) -> PReg {
         regs::rbp().to_real_reg().unwrap().into()

cranelift/filetests/filetests/isa/x64/isub-const-lea.clif

@@ -0,0 +1,85 @@
+test compile precise-output
+target x86_64
+
+;; `x - C` should lower to a single `lea -C(x)`, not `mov`+`sub`.
+function %isub_c2(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 2
+    v2 = isub v0, v1
+    return v2
+}
+
+; VCode:
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block0:
+;   leaq -2(%rdi), %rax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   leaq -2(%rdi), %rax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+;; 32-bit variant.
+function %isub_c5_i32(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 5
+    v2 = isub v0, v1
+    return v2
+}
+
+; VCode:
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block0:
+;   leal -5(%rdi), %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   leal -5(%rdi), %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+;; reg-reg isub must still use `sub` (rule must not fire).
+function %isub_regreg(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = isub v0, v1
+    return v2
+}
+
+; VCode:
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block0:
+;   movq %rdi, %rax
+;   subq %rsi, %rax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movq %rdi, %rax
+;   subq %rsi, %rax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+

cranelift/filetests/filetests/runtests/isub-const-lea.clif

@@ -0,0 +1,36 @@
+; Execution coverage for lowering `x - C` (constant) to `lea` on x64.
+; The lowering only changes x64 codegen, but running on every target plus
+; the interpreter confirms the result is unchanged.
+test interpret
+test run
+target x86_64
+target aarch64
+target s390x
+target riscv64
+
+function %isub_c7_i64(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 7
+    v2 = isub v0, v1
+    return v2
+}
+; run: %isub_c7_i64(100) == 93
+; run: %isub_c7_i64(0) == -7
+; run: %isub_c7_i64(7) == 0
+
+function %isub_c5_i32(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 5
+    v2 = isub v0, v1
+    return v2
+}
+; run: %isub_c5_i32(10) == 5
+; run: %isub_c5_i32(0) == -5
+
+function %isub_cneg_i64(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 -3
+    v2 = isub v0, v1
+    return v2
+}
+; run: %isub_cneg_i64(10) == 13

tests/disas/load-store/x64/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0_offset.wat

@@ -21,29 +21,29 @@
 ;; wasm[0]::function[0]:
 ;;       pushq   %rbp
 ;;       movq    %rsp, %rbp
-;;       movq    0x40(%rdi), %rsi
+;;       movq    0x40(%rdi), %r9
 ;;       movl    %edx, %r8d
-;;       subq    $4, %rsi
-;;       cmpq    %rsi, %r8
-;;       ja      0x25
-;;   18: movq    0x38(%rdi), %r9
+;;       addq    $-4, %r9
+;;       cmpq    %r9, %r8
+;;       ja      0x28
+;;   1b: movq    0x38(%rdi), %r9
 ;;       movl    %ecx, (%r9, %r8)
 ;;       movq    %rbp, %rsp
 ;;       popq    %rbp
 ;;       retq
-;;   25: ud2
+;;   28: ud2
 ;;
 ;; wasm[0]::function[1]:
 ;;       pushq   %rbp
 ;;       movq    %rsp, %rbp
-;;       movq    0x40(%rdi), %rsi
+;;       movq    0x40(%rdi), %r9
 ;;       movl    %edx, %r8d
-;;       subq    $4, %rsi
-;;       cmpq    %rsi, %r8
-;;       ja      0x65
-;;   58: movq    0x38(%rdi), %r9
+;;       addq    $-4, %r9
+;;       cmpq    %r9, %r8
+;;       ja      0x68
+;;   5b: movq    0x38(%rdi), %r9
 ;;       movl    (%r9, %r8), %eax
 ;;       movq    %rbp, %rsp
 ;;       popq    %rbp
 ;;       retq
-;;   65: ud2
+;;   68: ud2

tests/disas/load-store/x64/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat

@@ -21,10 +21,10 @@
 ;; wasm[0]::function[0]:
 ;;       pushq   %rbp
 ;;       movq    %rsp, %rbp
-;;       movq    0x40(%rdi), %rsi
+;;       movq    0x40(%rdi), %r9
 ;;       movl    %edx, %r8d
-;;       subq    $0x1004, %rsi
-;;       cmpq    %rsi, %r8
+;;       addq    $-0x1004, %r9
+;;       cmpq    %r9, %r8
 ;;       ja      0x2c
 ;;   1b: movq    0x38(%rdi), %r9
 ;;       movl    %ecx, 0x1000(%r9, %r8)
@@ -36,10 +36,10 @@
 ;; wasm[0]::function[1]:
 ;;       pushq   %rbp
 ;;       movq    %rsp, %rbp
-;;       movq    0x40(%rdi), %rsi
+;;       movq    0x40(%rdi), %r9
 ;;       movl    %edx, %r8d
-;;       subq    $0x1004, %rsi
-;;       cmpq    %rsi, %r8
+;;       addq    $-0x1004, %r9
+;;       cmpq    %r9, %r8
 ;;       ja      0x6c
 ;;   5b: movq    0x38(%rdi), %r9
 ;;       movl    0x1000(%r9, %r8), %eax

tests/disas/load-store/x64/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat

@@ -21,10 +21,10 @@
 ;; wasm[0]::function[0]:
 ;;       pushq   %rbp
 ;;       movq    %rsp, %rbp
-;;       movq    0x40(%rdi), %rsi
+;;       movq    0x40(%rdi), %r9
 ;;       movl    %edx, %r8d
-;;       subq    $0x1001, %rsi
-;;       cmpq    %rsi, %r8
+;;       addq    $-0x1001, %r9
+;;       cmpq    %r9, %r8
 ;;       ja      0x2c
 ;;   1b: movq    0x38(%rdi), %r9
 ;;       movb    %cl, 0x1000(%r9, %r8)
@@ -36,10 +36,10 @@
 ;; wasm[0]::function[1]:
 ;;       pushq   %rbp
 ;;       movq    %rsp, %rbp
-;;       movq    0x40(%rdi), %rsi
+;;       movq    0x40(%rdi), %r9
 ;;       movl    %edx, %r8d
-;;       subq    $0x1001, %rsi
-;;       cmpq    %rsi, %r8
+;;       addq    $-0x1001, %r9
+;;       cmpq    %r9, %r8
 ;;       ja      0x6d
 ;;   5b: movq    0x38(%rdi), %r9
 ;;       movzbq  0x1000(%r9, %r8), %rax

tests/disas/load-store/x64/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat

@@ -21,13 +21,13 @@
 ;; wasm[0]::function[0]:
 ;;       pushq   %rbp
 ;;       movq    %rsp, %rbp
-;;       movq    0x40(%rdi), %r8
+;;       movq    0x40(%rdi), %r11
 ;;       movl    %edx, %r10d
-;;       subq    $4, %r8
+;;       leaq    -4(%r11), %rax
 ;;       xorq    %r11, %r11
 ;;       movq    %r10, %r9
 ;;       addq    0x38(%rdi), %r9
-;;       cmpq    %r8, %r10
+;;       cmpq    %rax, %r10
 ;;       cmovaq  %r11, %r9
 ;;       movl    %ecx, (%r9)
 ;;       movq    %rbp, %rsp
@@ -37,13 +37,13 @@
 ;; wasm[0]::function[1]:
 ;;       pushq   %rbp
 ;;       movq    %rsp, %rbp
-;;       movq    0x40(%rdi), %r8
+;;       movq    0x40(%rdi), %r11
 ;;       movl    %edx, %r10d
-;;       subq    $4, %r8
+;;       leaq    -4(%r11), %rax
 ;;       xorq    %r11, %r11
 ;;       movq    %r10, %r9
 ;;       addq    0x38(%rdi), %r9
-;;       cmpq    %r8, %r10
+;;       cmpq    %rax, %r10
 ;;       cmovaq  %r11, %r9
 ;;       movl    (%r9), %eax
 ;;       movq    %rbp, %rsp

tests/disas/load-store/x64/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat

@@ -21,13 +21,14 @@
 ;; wasm[0]::function[0]:
 ;;       pushq   %rbp
 ;;       movq    %rsp, %rbp
-;;       movq    0x40(%rdi), %r8
+;;       movq    %rdx, %rax
+;;       movq    0x40(%rdi), %rdx
 ;;       movq    0x38(%rdi), %r11
-;;       movl    %edx, %eax
-;;       subq    $0x1004, %r8
+;;       movl    %eax, %eax
+;;       leaq    -0x1004(%rdx), %rsi
 ;;       xorq    %rdx, %rdx
 ;;       leaq    0x1000(%r11, %rax), %r10
-;;       cmpq    %r8, %rax
+;;       cmpq    %rsi, %rax
 ;;       cmovaq  %rdx, %r10
 ;;       movl    %ecx, (%r10)
 ;;       movq    %rbp, %rsp
@@ -37,13 +38,13 @@
 ;; wasm[0]::function[1]:
 ;;       pushq   %rbp
 ;;       movq    %rsp, %rbp
-;;       movq    0x40(%rdi), %r8
+;;       movq    0x40(%rdi), %rcx
 ;;       movq    0x38(%rdi), %r11
 ;;       movl    %edx, %eax
-;;       subq    $0x1004, %r8
+;;       leaq    -0x1004(%rcx), %rdx
 ;;       xorq    %rcx, %rcx
 ;;       leaq    0x1000(%r11, %rax), %r10
-;;       cmpq    %r8, %rax
+;;       cmpq    %rdx, %rax
 ;;       cmovaq  %rcx, %r10
 ;;       movl    (%r10), %eax
 ;;       movq    %rbp, %rsp

tests/disas/load-store/x64/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat

@@ -21,13 +21,14 @@
 ;; wasm[0]::function[0]:
 ;;       pushq   %rbp
 ;;       movq    %rsp, %rbp
-;;       movq    0x40(%rdi), %r8
+;;       movq    %rdx, %rax
+;;       movq    0x40(%rdi), %rdx
 ;;       movq    0x38(%rdi), %r11
-;;       movl    %edx, %eax
-;;       subq    $0x1001, %r8
+;;       movl    %eax, %eax
+;;       leaq    -0x1001(%rdx), %rsi
 ;;       xorq    %rdx, %rdx
 ;;       leaq    0x1000(%r11, %rax), %r10
-;;       cmpq    %r8, %rax
+;;       cmpq    %rsi, %rax
 ;;       cmovaq  %rdx, %r10
 ;;       movb    %cl, (%r10)
 ;;       movq    %rbp, %rsp
@@ -37,13 +38,13 @@
 ;; wasm[0]::function[1]:
 ;;       pushq   %rbp
 ;;       movq    %rsp, %rbp
-;;       movq    0x40(%rdi), %r8
+;;       movq    0x40(%rdi), %rcx
 ;;       movq    0x38(%rdi), %r11
 ;;       movl    %edx, %eax
-;;       subq    $0x1001, %r8
+;;       leaq    -0x1001(%rcx), %rdx
 ;;       xorq    %rcx, %rcx
 ;;       leaq    0x1000(%r11, %rax), %r10
-;;       cmpq    %r8, %rax
+;;       cmpq    %rdx, %rax
 ;;       cmovaq  %rcx, %r10
 ;;       movzbq  (%r10), %rax
 ;;       movq    %rbp, %rsp