From 924ab0837999b5dc7329c13347bef692f3239da1 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Thu, 2 Jul 2026 21:58:19 +0300 Subject: [PATCH 01/18] cranelift: introduce an arm32 (AArch32) backend skeleton First step towards a 32-bit ARM backend. This wires arm32 into the meta build, the isa::lookup path, and the codegen crate, and adds just enough of the MachInst/ABIMachineSpec plumbing to compile and emit a trivial function. An iconst.i32 followed by a return now lowers to real A32 (movw + bx lr) and is covered by filetests/isa/arm32/iconst.clif. Everything off that path is still a stub. Signed-off-by: Obei Sideg --- Cargo.toml | 2 +- cranelift/codegen/Cargo.toml | 3 +- cranelift/codegen/meta/src/isa/arm32.rs | 16 + cranelift/codegen/meta/src/isa/mod.rs | 6 + cranelift/codegen/meta/src/isle.rs | 16 + cranelift/codegen/src/isa/arm32/abi.rs | 442 ++++++++++++++++++ cranelift/codegen/src/isa/arm32/inst.isle | 75 +++ cranelift/codegen/src/isa/arm32/inst/emit.rs | 209 +++++++++ .../codegen/src/isa/arm32/inst/emit_tests.rs | 62 +++ cranelift/codegen/src/isa/arm32/inst/mod.rs | 320 +++++++++++++ cranelift/codegen/src/isa/arm32/inst/regs.rs | 140 ++++++ cranelift/codegen/src/isa/arm32/lower.isle | 21 + cranelift/codegen/src/isa/arm32/lower.rs | 30 ++ cranelift/codegen/src/isa/arm32/lower/isle.rs | 75 +++ .../isa/arm32/lower/isle/generated_code.rs | 15 + cranelift/codegen/src/isa/arm32/mod.rs | 214 +++++++++ cranelift/codegen/src/isa/arm32/settings.rs | 9 + cranelift/codegen/src/isa/mod.rs | 6 +- .../filetests/filetests/isa/arm32/iconst.clif | 18 + 19 files changed, 1676 insertions(+), 3 deletions(-) create mode 100644 cranelift/codegen/meta/src/isa/arm32.rs create mode 100644 cranelift/codegen/src/isa/arm32/abi.rs create mode 100644 cranelift/codegen/src/isa/arm32/inst.isle create mode 100644 cranelift/codegen/src/isa/arm32/inst/emit.rs create mode 100644 cranelift/codegen/src/isa/arm32/inst/emit_tests.rs create mode 100644 cranelift/codegen/src/isa/arm32/inst/mod.rs create mode 100644 cranelift/codegen/src/isa/arm32/inst/regs.rs create mode 100644 cranelift/codegen/src/isa/arm32/lower.isle create mode 100644 cranelift/codegen/src/isa/arm32/lower.rs create mode 100644 cranelift/codegen/src/isa/arm32/lower/isle.rs create mode 100644 cranelift/codegen/src/isa/arm32/lower/isle/generated_code.rs create mode 100644 cranelift/codegen/src/isa/arm32/mod.rs create mode 100644 cranelift/codegen/src/isa/arm32/settings.rs create mode 100644 cranelift/filetests/filetests/isa/arm32/iconst.clif diff --git a/Cargo.toml b/Cargo.toml index ff504efa3d12..f2d71b28a9e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -392,7 +392,7 @@ log = { version = "0.4.28", default-features = false } clap = { version = "4.5.48", default-features = false, features = ["std", "derive"] } clap_complete = "4.5.58" hashbrown = { version = "0.17", default-features = false } -capstone = { version = "0.14.0", default-features = false, features = ['full', 'arch_x86', 'arch_riscv', 'arch_arm64', 'arch_sysz'] } +capstone = { version = "0.14.0", default-features = false, features = ['full', 'arch_x86', 'arch_riscv', 'arch_arm', 'arch_arm64', 'arch_sysz'] } smallvec = { version = "1.15.1", features = ["union"] } tracing = { version = "0.1.41", default-features = false } bitflags = "2.9.4" diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 9235281ec986..4aa87d890e60 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -92,6 +92,7 @@ unwind = ["gimli"] # ISA targets for which we should build. # If no ISA targets are explicitly enabled, the ISA target for the host machine is enabled. x86 = [] +arm32 = [] arm64 = [] s390x = [] riscv64 = [] @@ -109,7 +110,7 @@ all-arch = ["all-native-arch", "pulley"] # Option to enable all architectures that correspond to an actual native target # (that is, exclude Pulley). -all-native-arch = ["x86", "arm64", "s390x", "riscv64"] +all-native-arch = ["x86", "arm32", "arm64", "s390x", "riscv64"] # For dependent crates that want to serialize some parts of cranelift enable-serde = [ diff --git a/cranelift/codegen/meta/src/isa/arm32.rs b/cranelift/codegen/meta/src/isa/arm32.rs new file mode 100644 index 000000000000..a1f0314c0bff --- /dev/null +++ b/cranelift/codegen/meta/src/isa/arm32.rs @@ -0,0 +1,16 @@ +use crate::cdsl::isa::TargetIsa; +use crate::cdsl::settings::SettingGroupBuilder; + +pub(crate) fn define() -> TargetIsa { + let mut settings = SettingGroupBuilder::new("arm32"); + + settings.add_bool( + "has_neon", + "Has Advanced SIMD (NEON) support; does not have an effect on code \ + generation by itself yet, reserved for future use.", + "", + false, + ); + + TargetIsa::new("arm32", settings.build()) +} diff --git a/cranelift/codegen/meta/src/isa/mod.rs b/cranelift/codegen/meta/src/isa/mod.rs index 655b14a9c5a1..350f3a082160 100644 --- a/cranelift/codegen/meta/src/isa/mod.rs +++ b/cranelift/codegen/meta/src/isa/mod.rs @@ -2,6 +2,7 @@ use crate::cdsl::isa::TargetIsa; use std::fmt; +mod arm32; mod arm64; mod pulley; mod riscv64; @@ -12,6 +13,7 @@ pub(crate) mod x86; #[derive(PartialEq, Copy, Clone)] pub enum Isa { X86, + Arm32, Arm64, S390x, Riscv64, @@ -32,6 +34,7 @@ impl Isa { pub fn from_arch(arch: &str) -> Option { match arch { "aarch64" => Some(Isa::Arm64), + x if x.starts_with("arm") || x.starts_with("thumb") => Some(Isa::Arm32), "s390x" => Some(Isa::S390x), x if ["x86_64", "i386", "i586", "i686"].contains(&x) => Some(Isa::X86), "riscv64" | "riscv64gc" | "riscv64imac" => Some(Isa::Riscv64), @@ -45,6 +48,7 @@ impl Isa { pub fn all() -> &'static [Isa] { &[ Isa::X86, + Isa::Arm32, Isa::Arm64, Isa::S390x, Isa::Riscv64, @@ -59,6 +63,7 @@ impl fmt::Display for Isa { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { Isa::X86 => write!(f, "x86"), + Isa::Arm32 => write!(f, "arm32"), Isa::Arm64 => write!(f, "arm64"), Isa::S390x => write!(f, "s390x"), Isa::Riscv64 => write!(f, "riscv64"), @@ -72,6 +77,7 @@ pub(crate) fn define(isas: &[Isa]) -> Vec { isas.iter() .map(|isa| match isa { Isa::X86 => x86::define(), + Isa::Arm32 => arm32::define(), Isa::Arm64 => arm64::define(), Isa::S390x => s390x::define(), Isa::Riscv64 => riscv64::define(), diff --git a/cranelift/codegen/meta/src/isle.rs b/cranelift/codegen/meta/src/isle.rs index 540cabbe0b6f..09b68c388212 100644 --- a/cranelift/codegen/meta/src/isle.rs +++ b/cranelift/codegen/meta/src/isle.rs @@ -136,6 +136,7 @@ pub fn get_isle_compilations( // Directories for lowering backends. let src_isa_x64 = codegen_crate_dir.join("src").join("isa").join("x64"); + let src_isa_arm32 = codegen_crate_dir.join("src").join("isa").join("arm32"); let src_isa_aarch64 = codegen_crate_dir.join("src").join("isa").join("aarch64"); let src_isa_s390x = codegen_crate_dir.join("src").join("isa").join("s390x"); let src_isa_risc_v = codegen_crate_dir.join("src").join("isa").join("riscv64"); @@ -244,6 +245,21 @@ pub fn get_isle_compilations( .concat(), untracked_inputs: vec![numerics_isle.clone(), clif_lower_isle.clone()], }, + // The arm32 (AArch32 / A32) instruction selector. + IsleCompilation { + name: "arm32".to_string(), + output: gen_dir.join("isle_arm32.rs"), + tracked_inputs: [ + vec![prelude_isle.clone(), prelude_lower_isle.clone()], + spec_inputs(&[]), + vec![ + src_isa_arm32.join("inst.isle"), + src_isa_arm32.join("lower.isle"), + ], + ] + .concat(), + untracked_inputs: vec![numerics_isle.clone(), clif_lower_isle.clone()], + }, // The risc-v instruction selector. IsleCompilation { name: "riscv64".to_string(), diff --git a/cranelift/codegen/src/isa/arm32/abi.rs b/cranelift/codegen/src/isa/arm32/abi.rs new file mode 100644 index 000000000000..9b76c1da0123 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/abi.rs @@ -0,0 +1,442 @@ +//! Implementation of a standard Arm32 ABI. + +use crate::CodegenResult; +use crate::ir; +use crate::ir::types::*; +use crate::ir::{Signature, Type}; +use crate::isa; +use crate::isa::arm32::inst::*; +use crate::isa::arm32::settings::Flags as Arm32Flags; +use crate::machinst::*; +use crate::settings; +use alloc::vec::Vec; +use regalloc2::{MachineEnv, PRegSet}; +use smallvec::{SmallVec, smallvec}; + +/// Support for the arm32 ABI from the callee side (within a function body). +pub(crate) type Arm32Callee = Callee; + +/// arm32-specific ABI behavior. This struct just serves as an implementation +/// point for the trait; it is never actually instantiated. +pub struct Arm32MachineDeps; + +impl IsaFlags for Arm32Flags {} + +impl ABIMachineSpec for Arm32MachineDeps { + type I = Inst; + type F = Arm32Flags; + + /// 128 MB, as with the other backends, to avoid overflow with 32-bit + /// arithmetic on stack offsets. + const STACK_ARG_RET_SIZE_LIMIT: u32 = 128 * 1024 * 1024; + + fn word_bits() -> u32 { + 32 + } + + fn stack_align(_call_conv: isa::CallConv) -> u32 { + // AAPCS requires 8-byte stack alignment at public interfaces. + 8 + } + + fn compute_arg_locs( + call_conv: isa::CallConv, + flags: &settings::Flags, + params: &[ir::AbiParam], + args_or_rets: ArgsOrRets, + add_ret_area_ptr: bool, + mut args: ArgsAccumulator, + ) -> CodegenResult<(u32, Option)> { + // Integer arguments/returns go in r0-r3 (r0-r1 for returns), the rest on + // the stack. This is a simplified AAPCS. + let (x_start, x_end) = match args_or_rets { + ArgsOrRets::Args => (0u8, 3u8), + ArgsOrRets::Rets => (0u8, 1u8), + }; + let mut next_x_reg = x_start; + let mut next_stack: u32 = 0; + + let ret_area_ptr = if add_ret_area_ptr { + assert!(ArgsOrRets::Args == args_or_rets); + let reg = xreg(next_x_reg); + next_x_reg += 1; + Some(ABIArg::reg( + reg.to_real_reg().unwrap(), + I32, + ir::ArgumentExtension::None, + ir::ArgumentPurpose::Normal, + )) + } else { + None + }; + + for param in params { + if let ir::ArgumentPurpose::StructArgument(_) = param.purpose { + panic!("StructArgument parameters are not supported on arm32."); + } + + let (rcs, reg_tys) = Inst::rc_for_type(param.value_type)?; + let mut slots = ABIArgSlotVec::new(); + for (rc, reg_ty) in rcs.iter().zip(reg_tys.iter()) { + assert_eq!(*rc, RegClass::Int, "arm32 only supports integer values"); + if next_x_reg <= x_end { + slots.push(ABIArgSlot::Reg { + reg: xreg(next_x_reg).to_real_reg().unwrap(), + ty: *reg_ty, + extension: param.extension, + }); + next_x_reg += 1; + } else { + if args_or_rets == ArgsOrRets::Rets && !flags.enable_multi_ret_implicit_sret() { + return Err(crate::CodegenError::Unsupported( + "Too many return values to fit in registers.".into(), + )); + } + let size = core::cmp::max(reg_ty.bytes(), 4); + next_stack = align_to(next_stack, size); + slots.push(ABIArgSlot::Stack { + offset: next_stack as i64, + ty: *reg_ty, + extension: param.extension, + }); + next_stack += size; + } + } + args.push(ABIArg::Slots { + slots, + purpose: param.purpose, + }); + } + + let pos = if let Some(ret_area_ptr) = ret_area_ptr { + args.push_non_formal(ret_area_ptr); + Some(args.args().len() - 1) + } else { + None + }; + + next_stack = align_to(next_stack, Self::stack_align(call_conv)); + Ok((next_stack, pos)) + } + + fn gen_load_stack(_mem: StackAMode, _into_reg: Writable, _ty: Type) -> Inst { + unimplemented!("arm32: stack loads (spills/reloads) not yet implemented") + } + + fn gen_store_stack(_mem: StackAMode, _from_reg: Reg, _ty: Type) -> Inst { + unimplemented!("arm32: stack stores (spills/reloads) not yet implemented") + } + + fn gen_move(to_reg: Writable, from_reg: Reg, ty: Type) -> Inst { + Inst::gen_move(to_reg, from_reg, ty) + } + + fn gen_extend( + _to_reg: Writable, + _from_reg: Reg, + _signed: bool, + _from_bits: u8, + _to_bits: u8, + ) -> Inst { + unimplemented!("arm32: integer extension not yet implemented") + } + + fn get_ext_mode( + _call_conv: isa::CallConv, + specified: ir::ArgumentExtension, + ) -> ir::ArgumentExtension { + specified + } + + fn gen_args(args: Vec) -> Inst { + Inst::Args { args } + } + + fn gen_rets(rets: Vec) -> Inst { + Inst::Rets { rets } + } + + fn get_stacklimit_reg(_call_conv: isa::CallConv) -> Reg { + spilltmp_reg() + } + + fn gen_add_imm( + _call_conv: isa::CallConv, + _into_reg: Writable, + _from_reg: Reg, + _imm: u32, + ) -> SmallInstVec { + unimplemented!("arm32: gen_add_imm not yet implemented") + } + + fn gen_stack_lower_bound_trap(_limit_reg: Reg) -> SmallInstVec { + unimplemented!("arm32: stack-limit checks not yet implemented") + } + + fn gen_get_stack_addr(_mem: StackAMode, _into_reg: Writable) -> Inst { + unimplemented!("arm32: stack-address computation not yet implemented") + } + + fn gen_load_base_offset(into_reg: Writable, base: Reg, offset: i32, _ty: Type) -> Inst { + Inst::Load { + rt: into_reg, + base, + offset, + } + } + + fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, _ty: Type) -> Inst { + Inst::Store { + rt: from_reg, + base, + offset, + } + } + + fn gen_sp_reg_adjust(amount: i32) -> SmallInstVec { + if amount == 0 { + smallvec![] + } else { + smallvec![Inst::AdjustSp { amount }] + } + } + + fn gen_prologue_frame_setup( + _call_conv: isa::CallConv, + _flags: &settings::Flags, + _isa_flags: &Arm32Flags, + frame_layout: &FrameLayout, + ) -> SmallInstVec { + let mut insts = smallvec![]; + if frame_layout.setup_area_size > 0 { + // push {fp, lr}; mov fp, sp + insts.push(Inst::AdjustSp { amount: -8 }); + insts.push(Inst::Store { + rt: link_reg(), + base: stack_reg(), + offset: 4, + }); + insts.push(Inst::Store { + rt: fp_reg(), + base: stack_reg(), + offset: 0, + }); + insts.push(Inst::Mov { + rd: writable_fp_reg(), + rm: stack_reg(), + }); + } + insts + } + + fn gen_epilogue_frame_restore( + _call_conv: isa::CallConv, + _flags: &settings::Flags, + _isa_flags: &Arm32Flags, + frame_layout: &FrameLayout, + ) -> SmallInstVec { + let mut insts = smallvec![]; + if frame_layout.setup_area_size > 0 { + insts.push(Inst::Load { + rt: writable_fp_reg(), + base: stack_reg(), + offset: 0, + }); + insts.push(Inst::Load { + rt: writable_link_reg(), + base: stack_reg(), + offset: 4, + }); + insts.push(Inst::AdjustSp { amount: 8 }); + } + insts + } + + fn gen_return( + _call_conv: isa::CallConv, + _isa_flags: &Arm32Flags, + _frame_layout: &FrameLayout, + ) -> SmallInstVec { + smallvec![Inst::Ret] + } + + fn gen_probestack(_insts: &mut SmallInstVec, _frame_size: u32) { + unimplemented!("arm32: probestack not yet implemented") + } + + fn gen_inline_probestack( + _insts: &mut SmallInstVec, + _call_conv: isa::CallConv, + _frame_size: u32, + _guard_size: u32, + ) { + unimplemented!("arm32: inline probestack not yet implemented") + } + + fn gen_clobber_save( + _call_conv: isa::CallConv, + _flags: &settings::Flags, + frame_layout: &FrameLayout, + ) -> SmallVec<[Inst; 16]> { + let mut insts = smallvec![]; + let stack_size = frame_layout.clobber_size + + frame_layout.fixed_frame_storage_size + + frame_layout.outgoing_args_size; + if stack_size > 0 { + insts.extend(Self::gen_sp_reg_adjust(-(stack_size as i32))); + let mut cur_offset = 0i32; + for reg in &frame_layout.clobbered_callee_saves { + insts.push(Inst::Store { + rt: Reg::from(reg.to_reg()), + base: stack_reg(), + offset: cur_offset, + }); + cur_offset += 4; + } + } + insts + } + + fn gen_clobber_restore( + _call_conv: isa::CallConv, + _flags: &settings::Flags, + frame_layout: &FrameLayout, + ) -> SmallVec<[Inst; 16]> { + let mut insts = smallvec![]; + let stack_size = frame_layout.clobber_size + + frame_layout.fixed_frame_storage_size + + frame_layout.outgoing_args_size; + let mut cur_offset = 0i32; + for reg in &frame_layout.clobbered_callee_saves { + insts.push(Inst::Load { + rt: reg.map(Reg::from), + base: stack_reg(), + offset: cur_offset, + }); + cur_offset += 4; + } + if stack_size > 0 { + insts.extend(Self::gen_sp_reg_adjust(stack_size as i32)); + } + insts + } + + fn gen_memcpy Writable>( + _call_conv: isa::CallConv, + _dst: Reg, + _src: Reg, + _size: usize, + _alloc_tmp: F, + ) -> SmallVec<[Self::I; 8]> { + unimplemented!("arm32: memcpy not yet implemented") + } + + fn get_number_of_spillslots_for_value( + rc: RegClass, + _target_vector_bytes: u32, + _isa_flags: &Arm32Flags, + ) -> u32 { + match rc { + RegClass::Int => 1, + RegClass::Float | RegClass::Vector => { + unimplemented!("arm32: no float/vector registers yet") + } + } + } + + fn get_machine_env(_flags: &settings::Flags, _call_conv: isa::CallConv) -> &MachineEnv { + static MACHINE_ENV: MachineEnv = create_reg_environment(); + &MACHINE_ENV + } + + fn get_regs_clobbered_by_call( + _call_conv_of_callee: isa::CallConv, + _is_exception: bool, + ) -> PRegSet { + DEFAULT_CLOBBERS + } + + fn compute_frame_layout( + call_conv: isa::CallConv, + flags: &settings::Flags, + _sig: &Signature, + regs: &[Writable], + function_calls: FunctionCalls, + incoming_args_size: u32, + tail_args_size: u32, + stackslots_size: u32, + fixed_frame_storage_size: u32, + outgoing_args_size: u32, + ) -> FrameLayout { + let is_callee_saved = |reg: &Writable| match call_conv { + isa::CallConv::PreserveAll => true, + _ => DEFAULT_CALLEE_SAVES.contains(reg.to_reg().into()), + }; + let mut regs: Vec> = + regs.iter().cloned().filter(is_callee_saved).collect(); + regs.sort_unstable(); + + let clobber_size = compute_clobber_size(®s); + + let setup_area_size = if flags.preserve_frame_pointers() + || function_calls != FunctionCalls::None + || incoming_args_size > 0 + || clobber_size > 0 + || fixed_frame_storage_size > 0 + { + 8 // FP, LR + } else { + 0 + }; + + FrameLayout { + word_bytes: 4, + incoming_args_size, + tail_args_size, + setup_area_size, + clobber_size, + fixed_frame_storage_size, + stackslots_size, + outgoing_args_size, + clobbered_callee_saves: regs, + function_calls, + } + } + + fn retval_temp_reg(_call_conv_of_callee: isa::CallConv) -> Writable { + // r12 (ip) is caller-saved and never used to return a value. + writable_xreg(12) + } + + fn exception_payload_regs(_call_conv: isa::CallConv) -> &'static [Reg] { + &[] + } +} + +/// Callee-saved GPRs under AAPCS: r4-r11. +const DEFAULT_CALLEE_SAVES: PRegSet = PRegSet::empty() + .with(preg(4)) + .with(preg(5)) + .with(preg(6)) + .with(preg(7)) + .with(preg(8)) + .with(preg(9)) + .with(preg(10)) + .with(preg(11)); + +/// Caller-saved (clobbered) GPRs: r0-r3 and r12. +const DEFAULT_CLOBBERS: PRegSet = PRegSet::empty() + .with(preg(0)) + .with(preg(1)) + .with(preg(2)) + .with(preg(3)) + .with(preg(12)); + +fn compute_clobber_size(clobbers: &[Writable]) -> u32 { + let mut size = 0; + for reg in clobbers { + debug_assert_eq!(reg.to_reg().class(), RegClass::Int); + size += 4; + } + align_to(size, 8) +} diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle new file mode 100644 index 000000000000..1c5339d92f42 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -0,0 +1,75 @@ +;; Arm32 instruction definitions. +;; +;; The `MInst` type below is the machine-instruction enum for this backend; it +;; is generated into Rust and re-exported as `Inst` by `inst/mod.rs`. + +(type MInst + (enum + ;; A zero-length no-op. + (Nop0) + ;; A four-byte no-op. + (Nop4) + + ;; Load an arbitrary 32-bit immediate into `rd` (expands to `movw`, plus + ;; `movt` when the high half is non-zero). + (MovImm + (rd WritableReg) + (imm u64)) + + ;; Register-to-register move: `mov rd, rm`. + (Mov + (rd WritableReg) + (rm Reg)) + + ;; Adjust the stack pointer by a (signed) constant amount. + (AdjustSp + (amount i32)) + + ;; Store `rt` to `[base, #offset]`. + (Store + (rt Reg) + (base Reg) + (offset i32)) + + ;; Load `[base, #offset]` into `rt`. + (Load + (rt WritableReg) + (base Reg) + (offset i32)) + + ;; An unconditional branch to a label. + (Jump + (dest MachLabel)) + + ;; Pseudo-instruction capturing incoming register arguments in vregs. + (Args + (args VecArgPair)) + + ;; Pseudo-instruction moving vregs into the return registers. + (Rets + (rets VecRetPair)) + + ;; Return from the current function (`bx lr`). + (Ret) + ) +) + +;;;; Instruction-constructor helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Emit a `MovImm` and return its destination register. +(decl movimm (u64) Reg) +(rule (movimm n) + (let ((dst WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.MovImm dst n)))) + dst)) + +;; Materialize an integer constant of the given type into a register. +(decl imm (Type u64) Reg) +(rule (imm _ty n) (movimm n)) + +;;;; Branch lowering ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; An unconditional jump as a side effect. +(decl jump_impl (MachLabel) SideEffectNoResult) +(rule (jump_impl label) + (SideEffectNoResult.Inst (MInst.Jump label))) diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs new file mode 100644 index 000000000000..b5e7bc27910f --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -0,0 +1,209 @@ +//! arm32 (A32) ISA: binary code emission. + +use crate::ir; +use crate::isa::arm32::inst::*; +use crate::settings; +use cranelift_control::ControlPlane; + +/// Information carried along during emission, derived from the shared and ISA +/// flags. +pub struct EmitInfo { + #[expect(dead_code, reason = "may be used once more instructions are added")] + shared_flags: settings::Flags, + #[expect(dead_code, reason = "may be used once more instructions are added")] + isa_flags: super::super::settings::Flags, +} + +impl EmitInfo { + pub(crate) fn new( + shared_flags: settings::Flags, + isa_flags: super::super::settings::Flags, + ) -> Self { + Self { + shared_flags, + isa_flags, + } + } +} + +/// State carried between emissions of a sequence of instructions. +#[derive(Default, Clone, Debug)] +pub struct EmitState { + /// The user stack map for the upcoming instruction. + user_stack_map: Option, + /// Only used during fuzz-testing. + ctrl_plane: ControlPlane, + frame_layout: FrameLayout, +} + +impl EmitState { + #[expect(dead_code, reason = "will be used once safepoints are supported")] + fn take_stack_map(&mut self) -> Option { + self.user_stack_map.take() + } +} + +impl MachInstEmitState for EmitState { + fn new( + abi: &Callee, + ctrl_plane: ControlPlane, + ) -> Self { + EmitState { + user_stack_map: None, + ctrl_plane, + frame_layout: abi.frame_layout().clone(), + } + } + + fn pre_safepoint(&mut self, user_stack_map: Option) { + self.user_stack_map = user_stack_map; + } + + fn ctrl_plane_mut(&mut self) -> &mut ControlPlane { + &mut self.ctrl_plane + } + + fn take_ctrl_plane(self) -> ControlPlane { + self.ctrl_plane + } + + fn frame_layout(&self) -> &FrameLayout { + &self.frame_layout + } +} + +/// Encoding of the "always" (AL) condition code, in the top nibble of an A32 +/// instruction word. +const COND_AL: u32 = 0xe000_0000; + +/// The hardware encoding number (0-15) of a real register. +fn machreg_to_gpr(reg: Reg) -> u32 { + u32::from(reg.to_real_reg().unwrap().hw_enc() & 0xf) +} + +/// `movw rd, #imm16` — load a 16-bit immediate into the low half of `rd`, +/// zeroing the high half. +fn enc_movw(rd: u32, imm16: u32) -> u32 { + let imm4 = (imm16 >> 12) & 0xf; + let imm12 = imm16 & 0xfff; + COND_AL | 0x0300_0000 | (imm4 << 16) | (rd << 12) | imm12 +} + +/// `movt rd, #imm16` — load a 16-bit immediate into the high half of `rd`, +/// preserving the low half. +fn enc_movt(rd: u32, imm16: u32) -> u32 { + let imm4 = (imm16 >> 12) & 0xf; + let imm12 = imm16 & 0xfff; + COND_AL | 0x0340_0000 | (imm4 << 16) | (rd << 12) | imm12 +} + +/// `mov rd, rm` (register). +fn enc_mov(rd: u32, rm: u32) -> u32 { + COND_AL | 0x01a0_0000 | (rd << 12) | rm +} + +/// `bx lr` — branch and exchange to the link register. +fn enc_bx_lr() -> u32 { + COND_AL | 0x012f_ff10 | 14 +} + +/// `add`/`sub sp, sp, #imm` for a small `imm` (< 256, encoded with rotation 0). +fn enc_sp_adjust(amount: i32) -> u32 { + let sp = 13u32; + let (base, mag) = if amount < 0 { + (0x0240_0000u32, (-amount) as u32) // sub + } else { + (0x0280_0000u32, amount as u32) // add + }; + assert!( + mag < 256, + "arm32 sp adjust out of simple-immediate range: {amount}" + ); + COND_AL | base | (sp << 16) | (sp << 12) | mag +} + +/// `str`/`ldr rt, [base, #offset]` with a 12-bit immediate offset. +fn enc_ldr_str(load: bool, rt: u32, base: u32, offset: i32) -> u32 { + let (u_bit, mag) = if offset < 0 { + (0u32, (-offset) as u32) + } else { + (1u32, offset as u32) + }; + assert!(mag < 4096, "arm32 ldr/str offset out of range: {offset}"); + // cond 01 I(0) P(1) U W(0) B(0) L rn rt imm12 + let l_bit = if load { 1u32 } else { 0 }; + COND_AL + | 0x0400_0000 + | (1 << 24) // P = 1 (pre-indexed) + | (u_bit << 23) + | (l_bit << 20) + | (base << 16) + | (rt << 12) + | mag +} + +/// `b label` — unconditional branch (offset patched in later). +fn enc_b_placeholder() -> u32 { + COND_AL | 0x0a00_0000 +} + +fn put_u32(sink: &mut MachBuffer, word: u32) { + for b in word.to_le_bytes() { + sink.put1(b); + } +} + +impl MachInstEmit for Inst { + type State = EmitState; + type Info = EmitInfo; + + fn emit(&self, sink: &mut MachBuffer, _emit_info: &Self::Info, _state: &mut EmitState) { + let start = sink.cur_offset(); + match self { + Inst::Nop0 => {} + Inst::Nop4 => put_u32(sink, COND_AL | 0x0320_f000), + Inst::Ret => put_u32(sink, enc_bx_lr()), + Inst::MovImm { rd, imm } => { + let rd = machreg_to_gpr(rd.to_reg()); + put_u32(sink, enc_movw(rd, (imm & 0xffff) as u32)); + if imm >> 16 != 0 { + put_u32(sink, enc_movt(rd, (imm >> 16) as u32)); + } + } + Inst::Mov { rd, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_mov(rd, rm)); + } + Inst::AdjustSp { amount } => put_u32(sink, enc_sp_adjust(*amount)), + Inst::Store { rt, base, offset } => { + let rt = machreg_to_gpr(*rt); + let base = machreg_to_gpr(*base); + put_u32(sink, enc_ldr_str(false, rt, base, *offset)); + } + Inst::Load { rt, base, offset } => { + let rt = machreg_to_gpr(rt.to_reg()); + let base = machreg_to_gpr(*base); + put_u32(sink, enc_ldr_str(true, rt, base, *offset)); + } + Inst::Jump { dest } => { + sink.use_label_at_offset(sink.cur_offset(), *dest, LabelUse::Branch26); + sink.add_uncond_branch(sink.cur_offset(), sink.cur_offset() + 4, *dest); + put_u32(sink, enc_b_placeholder()); + } + Inst::Args { .. } | Inst::Rets { .. } => { + // Pseudo-instructions: no machine code. + } + } + + let end = sink.cur_offset(); + debug_assert!( + (end - start) <= Inst::worst_case_size(), + "instruction {self:?} exceeded worst-case size" + ); + } + + fn pretty_print_inst(&self, state: &mut Self::State) -> String { + self.print_with_state(state) + } +} diff --git a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs new file mode 100644 index 000000000000..3db1d409fbfb --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs @@ -0,0 +1,62 @@ +//! Encoding tests for arm32 instructions. + +use super::*; + +/// Emit a single instruction and return the encoded bytes. +fn encode(inst: Inst) -> Vec { + use crate::isa::arm32::settings as arm32_settings; + use crate::settings::{self, Configurable}; + + let mut b = settings::builder(); + b.set("enable_verifier", "false").unwrap(); + let flags = settings::Flags::new(b); + let isa_flags = arm32_settings::Flags::new(&flags, &arm32_settings::builder()); + let emit_info = EmitInfo::new(flags, isa_flags); + + let mut buffer = MachBuffer::new(); + let mut state = EmitState::default(); + inst.emit(&mut buffer, &emit_info, &mut state); + let mut ctrl_plane = Default::default(); + let buffer = buffer.finish(&Default::default(), &mut ctrl_plane); + buffer.data().to_vec() +} + +fn u32_le(inst: Inst) -> u32 { + let bytes = encode(inst); + assert_eq!(bytes.len(), 4, "expected a single 4-byte instruction"); + u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) +} + +#[test] +fn bx_lr() { + assert_eq!(u32_le(Inst::Ret), 0xe12f_ff1e); +} + +#[test] +fn nop4() { + assert_eq!(u32_le(Inst::Nop4), 0xe320_f000); +} + +#[test] +fn mov_reg() { + // mov r0, r1 + assert_eq!( + u32_le(Inst::Mov { + rd: writable_xreg(0), + rm: xreg(1), + }), + 0xe1a0_0001 + ); +} + +#[test] +fn movw_small_imm() { + // movw r0, #42 + assert_eq!( + u32_le(Inst::MovImm { + rd: writable_xreg(0), + imm: 42, + }), + 0xe300_002a + ); +} diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs new file mode 100644 index 000000000000..872a16d06642 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -0,0 +1,320 @@ +//! This module defines arm32 (AArch32 / A32) machine instruction types. + +use crate::binemit::{Addend, CodeOffset, Reloc}; +use crate::ir::types::{I8, I16, I32}; +use crate::ir::{Type, types}; +use crate::isa::FunctionAlignment; +use crate::isa::arm32::abi::Arm32MachineDeps; +use crate::machinst::*; +use crate::{CodegenError, CodegenResult, settings}; + +use alloc::string::{String, ToString}; +use alloc::vec::Vec; +use core::fmt::Write; +use regalloc2::RegClass; + +pub mod regs; +pub use self::regs::*; +pub mod emit; +pub use self::emit::*; + +#[cfg(test)] +mod emit_tests; + +// The `Inst` type itself is generated from `inst.isle` and re-exported here so +// that the rest of the backend can refer to `Inst` variants directly. +pub use crate::isa::arm32::lower::isle::generated_code::MInst as Inst; + +//============================================================================= +// Operand collection. + +fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { + // Only collect virtual (allocatable) registers as operands. Physical + // registers such as sp/fp/lr are fixed in the encoding and are not tracked + // by the register allocator. + fn use_if_virtual(collector: &mut impl OperandVisitor, reg: &mut Reg) { + if reg.to_real_reg().is_none() { + collector.reg_use(reg); + } + } + fn def_if_virtual(collector: &mut impl OperandVisitor, reg: &mut Writable) { + if reg.to_reg().to_real_reg().is_none() { + collector.reg_def(reg); + } + } + + match inst { + Inst::Nop0 | Inst::Nop4 | Inst::Ret | Inst::AdjustSp { .. } | Inst::Jump { .. } => {} + Inst::MovImm { rd, .. } => def_if_virtual(collector, rd), + Inst::Mov { rd, rm } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::Store { rt, base, .. } => { + use_if_virtual(collector, rt); + use_if_virtual(collector, base); + } + Inst::Load { rt, base, .. } => { + use_if_virtual(collector, base); + def_if_virtual(collector, rt); + } + Inst::Args { args } => { + for ArgPair { vreg, preg } in args { + collector.reg_fixed_def(vreg, *preg); + } + } + Inst::Rets { rets } => { + for RetPair { vreg, preg } in rets { + collector.reg_fixed_use(vreg, *preg); + } + } + } +} + +impl MachInst for Inst { + type LabelUse = LabelUse; + type ABIMachineSpec = Arm32MachineDeps; + + // The undefined instruction `udf`. Used to fill trap-reachable padding. + const TRAP_OPCODE: &'static [u8] = &0xe7f000f0u32.to_le_bytes(); + + fn gen_dummy_use(_reg: Reg) -> Self { + // Represented as a zero-length nop that "uses" nothing; a dummy use is + // only needed by targets that must keep a value live artificially. + Inst::Nop0 + } + + fn canonical_type_for_rc(rc: RegClass) -> Type { + match rc { + RegClass::Int => I32, + RegClass::Float => types::F64, + RegClass::Vector => types::I8X16, + } + } + + fn is_safepoint(&self) -> bool { + false + } + + fn get_operands(&mut self, collector: &mut impl OperandVisitor) { + arm32_get_operands(self, collector); + } + + fn is_move(&self) -> Option<(Writable, Reg)> { + match self { + Inst::Mov { rd, rm } => Some((*rd, *rm)), + _ => None, + } + } + + fn is_included_in_clobbers(&self) -> bool { + !matches!(self, Inst::Args { .. }) + } + + fn is_trap(&self) -> bool { + false + } + + fn is_args(&self) -> bool { + matches!(self, Inst::Args { .. }) + } + + fn call_type(&self) -> CallType { + CallType::None + } + + fn is_term(&self) -> MachTerminator { + match self { + Inst::Rets { .. } => MachTerminator::Ret, + Inst::Jump { .. } => MachTerminator::Branch, + _ => MachTerminator::None, + } + } + + fn is_mem_access(&self) -> bool { + matches!(self, Inst::Load { .. } | Inst::Store { .. }) + } + + fn gen_move(to_reg: Writable, from_reg: Reg, _ty: Type) -> Inst { + Inst::Mov { + rd: to_reg, + rm: from_reg, + } + } + + fn gen_nop(preferred_size: usize) -> Inst { + if preferred_size == 0 { + return Inst::Nop0; + } + assert!(preferred_size >= 4); + Inst::Nop4 + } + + fn gen_nop_units() -> Vec> { + vec![0xe320f000u32.to_le_bytes().to_vec()] + } + + fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> { + match ty { + I8 => Ok((&[RegClass::Int], &[I8])), + I16 => Ok((&[RegClass::Int], &[I16])), + I32 => Ok((&[RegClass::Int], &[I32])), + _ => Err(CodegenError::Unsupported(alloc::format!( + "Unsupported type on arm32 (only i8/i16/i32 are implemented so far): {ty}" + ))), + } + } + + fn gen_jump(target: MachLabel) -> Inst { + Inst::Jump { dest: target } + } + + fn worst_case_size() -> CodeOffset { + // The largest instruction is `MovImm`, which expands to a `movw` plus a + // `movt`: 8 bytes. + 8 + } + + fn worst_case_island_growth() -> CodeOffset { + 8 + } + + fn ref_type_regclass(_settings: &settings::Flags) -> RegClass { + RegClass::Int + } + + fn function_alignment() -> FunctionAlignment { + FunctionAlignment { + minimum: 4, + preferred: 4, + } + } +} + +//============================================================================= +// Pretty-printing. + +impl Inst { + pub(crate) fn print_with_state(&self, _state: &mut EmitState) -> String { + let r = |reg: Reg| reg_name(reg); + match self { + Inst::Nop0 => "nop-zero-len".to_string(), + Inst::Nop4 => "nop".to_string(), + Inst::Ret => "bx lr".to_string(), + Inst::MovImm { rd, imm } => { + let rd = r(rd.to_reg()); + if *imm >> 16 == 0 { + alloc::format!("movw {rd}, #{imm}") + } else { + alloc::format!("movw {rd}, #{}; movt {rd}, #{}", imm & 0xffff, imm >> 16) + } + } + Inst::Mov { rd, rm } => { + alloc::format!("mov {}, {}", r(rd.to_reg()), r(*rm)) + } + Inst::AdjustSp { amount } => { + if *amount < 0 { + alloc::format!("sub sp, sp, #{}", -*amount) + } else { + alloc::format!("add sp, sp, #{amount}") + } + } + Inst::Store { rt, base, offset } => { + alloc::format!("str {}, [{}, #{}]", r(*rt), r(*base), offset) + } + Inst::Load { rt, base, offset } => { + alloc::format!("ldr {}, [{}, #{}]", r(rt.to_reg()), r(*base), offset) + } + Inst::Jump { dest } => alloc::format!("b {}", dest.to_string()), + Inst::Args { args } => { + let mut s = "args".to_string(); + for arg in args { + write!(&mut s, " {}={}", r(arg.vreg.to_reg()), r(arg.preg)).unwrap(); + } + s + } + Inst::Rets { rets } => { + let mut s = "rets".to_string(); + for ret in rets { + write!(&mut s, " {}={}", r(ret.vreg), r(ret.preg)).unwrap(); + } + s + } + } + } +} + +//============================================================================= +// Label uses (branch fixups). + +/// A use of a label by an instruction, for branch fixups. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LabelUse { + /// A 24-bit signed PC-relative branch offset, as used by the A32 `B`/`BL` + /// instructions. The stored immediate is the offset in units of 4 bytes, + /// relative to the branch instruction's address plus 8. + Branch26, +} + +impl MachInstLabelUse for LabelUse { + const ALIGN: CodeOffset = 4; + + fn max_pos_range(self) -> CodeOffset { + match self { + // 24-bit signed immediate, scaled by 4. + LabelUse::Branch26 => ((1 << 23) - 1) * 4, + } + } + + fn max_neg_range(self) -> CodeOffset { + match self { + LabelUse::Branch26 => (1 << 23) * 4, + } + } + + fn patch_size(self) -> CodeOffset { + 4 + } + + fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) { + match self { + LabelUse::Branch26 => { + // The ARM PC reads as the instruction address plus 8. + let pc_base = use_offset as i64 + 8; + let offset = label_offset as i64 - pc_base; + debug_assert!(offset & 0b11 == 0); + let imm24 = ((offset >> 2) as u32) & 0x00ff_ffff; + let insn = u32::from_le_bytes(buffer[0..4].try_into().unwrap()); + let insn = (insn & 0xff00_0000) | imm24; + buffer[0..4].copy_from_slice(&insn.to_le_bytes()); + } + } + } + + fn supports_veneer(self) -> bool { + false + } + + fn veneer_size(self) -> CodeOffset { + 0 + } + + fn worst_case_veneer_size() -> CodeOffset { + 0 + } + + fn generate_veneer( + self, + _buffer: &mut [u8], + _veneer_offset: CodeOffset, + ) -> (CodeOffset, LabelUse) { + panic!("arm32 does not support branch veneers yet"); + } + + fn from_reloc(reloc: Reloc, _addend: Addend) -> Option { + match reloc { + Reloc::Arm32Call => Some(LabelUse::Branch26), + _ => None, + } + } +} diff --git a/cranelift/codegen/src/isa/arm32/inst/regs.rs b/cranelift/codegen/src/isa/arm32/inst/regs.rs new file mode 100644 index 000000000000..c7f252f51fcf --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/inst/regs.rs @@ -0,0 +1,140 @@ +//! Arm32 (AArch32 / A32) ISA definitions: registers. +//! +//! We model the 16 general-purpose ARM registers r0-r15 as `RegClass::Int`. +//! Following the AAPCS calling convention: +//! +//! * r0-r3 : argument / caller-saved (scratch) +//! * r4-r10 : callee-saved +//! * r11 (fp): frame pointer +//! * r12 (ip): intra-procedure scratch / spill temporary (reserved) +//! * r13 (sp): stack pointer +//! * r14 (lr): link register +//! * r15 (pc): program counter +//! +//! Floating-point (VFP/NEON) registers are not modelled yet. + +use crate::machinst::{Reg, Writable}; +use regalloc2::{MachineEnv, PReg, RegClass, VReg}; + +/// Construct a virtual-ish `Reg` referencing GPR `enc` (0-15). +#[inline] +pub const fn xreg(enc: u8) -> Reg { + let p_reg = PReg::new(enc as usize, RegClass::Int); + let v_reg = VReg::new(p_reg.index(), p_reg.class()); + Reg::from_virtual_reg(v_reg) +} + +/// Construct a `PReg` referencing GPR `enc` (0-15). +pub const fn preg(enc: u8) -> PReg { + PReg::new(enc as usize, RegClass::Int) +} + +/// Get a writable reference to GPR `enc`. +#[inline] +pub fn writable_xreg(enc: u8) -> Writable { + Writable::from_reg(xreg(enc)) +} + +/// Frame pointer (r11). +#[inline] +pub fn fp_reg() -> Reg { + xreg(11) +} + +/// Writable frame pointer. +#[inline] +pub fn writable_fp_reg() -> Writable { + Writable::from_reg(fp_reg()) +} + +/// Intra-procedure-call scratch register (r12), used as the spill temporary. +#[inline] +pub fn spilltmp_reg() -> Reg { + xreg(12) +} + +/// Stack pointer (r13). +#[inline] +pub fn stack_reg() -> Reg { + xreg(13) +} + +/// Writable stack pointer. +#[inline] +#[allow( + dead_code, + reason = "part of the register API, used as the backend grows" +)] +pub fn writable_stack_reg() -> Writable { + Writable::from_reg(stack_reg()) +} + +/// Link register (r14). +#[inline] +pub fn link_reg() -> Reg { + xreg(14) +} + +/// Writable link register. +#[inline] +pub fn writable_link_reg() -> Writable { + Writable::from_reg(link_reg()) +} + +/// Program counter (r15). +#[inline] +#[allow( + dead_code, + reason = "part of the register API, used as the backend grows" +)] +pub fn pc_reg() -> Reg { + xreg(15) +} + +/// Pretty-print a register with its AAPCS name. +pub fn reg_name(reg: Reg) -> alloc::string::String { + use alloc::string::ToString; + match reg.to_real_reg() { + Some(real) => match real.hw_enc() { + 11 => "fp".to_string(), + 12 => "ip".to_string(), + 13 => "sp".to_string(), + 14 => "lr".to_string(), + 15 => "pc".to_string(), + enc => alloc::format!("r{enc}"), + }, + None => alloc::format!("{reg:?}"), + } +} + +/// Build the register environment describing which registers the allocator may +/// use, in preference order. +pub const fn create_reg_environment() -> MachineEnv { + // Preferred registers are the caller-saved (scratch) GPRs; the allocator + // reaches for these first since they don't need to be saved/restored. + let preferred_int = PRegSet_int(&[0, 1, 2, 3]); + // Non-preferred registers are the callee-saved GPRs. + let non_preferred_int = PRegSet_int(&[4, 5, 6, 7, 8, 9, 10]); + + MachineEnv { + preferred_regs_by_class: [preferred_int, empty(), empty()], + non_preferred_regs_by_class: [non_preferred_int, empty(), empty()], + fixed_stack_slots: vec![], + scratch_by_class: [None, None, None], + } +} + +#[allow(non_snake_case)] +const fn PRegSet_int(encs: &[u8]) -> regalloc2::PRegSet { + let mut set = regalloc2::PRegSet::empty(); + let mut i = 0; + while i < encs.len() { + set = set.with(preg(encs[i])); + i += 1; + } + set +} + +const fn empty() -> regalloc2::PRegSet { + regalloc2::PRegSet::empty() +} diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle new file mode 100644 index 000000000000..c5e1ac9fe3b5 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -0,0 +1,21 @@ +;; arm32 (AArch32 / A32) lowering rules. + +;; The main lowering entry point. +(decl partial lower (Inst) InstOutput) + +;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (iconst (fits_in_32 ty) (u64_from_imm64 n))) + (imm ty n)) + +;;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (return args)) + (lower_return args)) + +;;;; Branch lowering ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl partial lower_branch (Inst MachLabelSlice) Unit) + +(rule (lower_branch (jump _) (single_target label)) + (emit_side_effect (jump_impl label))) diff --git a/cranelift/codegen/src/isa/arm32/lower.rs b/cranelift/codegen/src/isa/arm32/lower.rs new file mode 100644 index 000000000000..d4b67ff5d323 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/lower.rs @@ -0,0 +1,30 @@ +//! Lowering rules for Arm32. + +use crate::ir::Inst as IRInst; +use crate::isa::arm32::Arm32Backend; +use crate::isa::arm32::inst::*; +use crate::machinst::lower::*; +use crate::machinst::*; + +pub mod isle; + +impl LowerBackend for Arm32Backend { + type MInst = Inst; + + fn lower(&self, ctx: &mut Lower, ir_inst: IRInst) -> Option { + isle::lower(ctx, self, ir_inst) + } + + fn lower_branch( + &self, + ctx: &mut Lower, + ir_inst: IRInst, + targets: &[MachLabel], + ) -> Option<()> { + isle::lower_branch(ctx, self, ir_inst, targets) + } + + fn maybe_pinned_reg(&self) -> Option { + None + } +} diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs new file mode 100644 index 000000000000..d53421b44a03 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -0,0 +1,75 @@ +//! ISLE integration glue code for arm32 lowering. + +// Pull in the ISLE generated code. +pub mod generated_code; +use generated_code::MInst; + +// Types that the generated ISLE code refers to via `use super::*`. +use crate::ir::condcodes::{FloatCC, IntCC}; +use crate::ir::immediates::*; +use crate::ir::types::*; +use crate::ir::{ + BlockCall, ExternalName, Inst, InstructionData, MemFlags, Opcode, TrapCode, Value, ValueList, +}; +use crate::isa::arm32::Arm32Backend; +use crate::machinst::isle::*; +use crate::machinst::{ + ArgPair, CallArgList, CallRetList, InstOutput, Lower, MachInst, MachLabel, RetPair, + VCodeConstant, VCodeConstantData, VCodeInst, +}; +use alloc::boxed::Box; +use alloc::vec::Vec; +use regalloc2::PReg; + +type VecArgPair = Vec; +type VecRetPair = Vec; + +/// The ISLE lowering context for arm32. +pub(crate) struct Arm32IsleContext<'a, 'b, I, B> +where + I: VCodeInst, + B: LowerBackend, +{ + pub lower_ctx: &'a mut Lower<'b, I>, + #[allow(dead_code, reason = "kept for symmetry with other backends")] + pub backend: &'a B, +} + +impl<'a, 'b> Arm32IsleContext<'a, 'b, MInst, Arm32Backend> { + fn new(lower_ctx: &'a mut Lower<'b, MInst>, backend: &'a Arm32Backend) -> Self { + Self { lower_ctx, backend } + } + + pub(crate) fn dfg(&self) -> &crate::ir::DataFlowGraph { + &self.lower_ctx.f.dfg + } +} + +impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { + isle_lower_prelude_methods!(); + + fn emit(&mut self, inst: &MInst) -> Unit { + self.lower_ctx.emit(inst.clone()); + } +} + +/// The main entry point for lowering with ISLE. +pub(crate) fn lower( + lower_ctx: &mut Lower, + backend: &Arm32Backend, + inst: Inst, +) -> Option { + let mut isle_ctx = Arm32IsleContext::new(lower_ctx, backend); + generated_code::constructor_lower(&mut isle_ctx, inst) +} + +/// The main entry point for branch lowering with ISLE. +pub(crate) fn lower_branch( + lower_ctx: &mut Lower, + backend: &Arm32Backend, + branch: Inst, + targets: &[MachLabel], +) -> Option<()> { + let mut isle_ctx = Arm32IsleContext::new(lower_ctx, backend); + generated_code::constructor_lower_branch(&mut isle_ctx, branch, targets) +} diff --git a/cranelift/codegen/src/isa/arm32/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/arm32/lower/isle/generated_code.rs new file mode 100644 index 000000000000..e3ac3badcadf --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/lower/isle/generated_code.rs @@ -0,0 +1,15 @@ +// See https://github.com/rust-lang/rust/issues/47995: we cannot use `#![...]` attributes inside of +// the generated ISLE source below because we include!() it. We must include!() it because its path +// depends on an environment variable; and also because of this, we can't do the `#[path = "..."] +// mod generated_code;` trick either. +#![allow( + dead_code, + unreachable_patterns, + unused_imports, + unused_variables, + irrefutable_let_patterns, + clippy::clone_on_copy, + reason = "generated code" +)] + +include!(concat!(env!("ISLE_DIR"), "/isle_arm32.rs")); diff --git a/cranelift/codegen/src/isa/arm32/mod.rs b/cranelift/codegen/src/isa/arm32/mod.rs new file mode 100644 index 000000000000..ccf3bbfc72a2 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/mod.rs @@ -0,0 +1,214 @@ +//! Arm32 Instruction Set Architecture. + +use crate::dominator_tree::DominatorTree; +use crate::ir::{self, Function, Type}; +use crate::isa::arm32::settings as arm32_settings; +use crate::isa::{ + Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey, OwnedTargetIsa, TargetIsa, +}; +use crate::machinst::{ + CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet, TextSectionBuilder, VCode, + compile, +}; +use crate::result::CodegenResult; +use crate::settings::{self as shared_settings, Flags}; +use alloc::string::String; +use alloc::{boxed::Box, vec::Vec}; +use core::fmt; +use cranelift_control::ControlPlane; +use target_lexicon::{Architecture, Triple}; + +mod abi; +pub(crate) mod inst; +mod lower; +mod settings; + +use self::inst::EmitInfo; + +/// An arm32 backend. +pub struct Arm32Backend { + triple: Triple, + flags: shared_settings::Flags, + isa_flags: arm32_settings::Flags, +} + +impl Arm32Backend { + /// Create a new arm32 backend with the given (shared) flags. + pub fn new_with_flags( + triple: Triple, + flags: shared_settings::Flags, + isa_flags: arm32_settings::Flags, + ) -> Arm32Backend { + Arm32Backend { + triple, + flags, + isa_flags, + } + } + + /// Lower to VCode, register-allocate, compute block layout and finalize + /// branches. The result is ready for binary emission. + fn compile_vcode( + &self, + func: &Function, + domtree: &DominatorTree, + ctrl_plane: &mut ControlPlane, + ) -> CodegenResult<(VCode, regalloc2::Output)> { + let emit_info = EmitInfo::new(self.flags.clone(), self.isa_flags.clone()); + let sigs = SigSet::new::(func, &self.flags)?; + let abi = abi::Arm32Callee::new(func, self, &self.isa_flags, &sigs)?; + compile::compile::(func, domtree, self, abi, emit_info, sigs, ctrl_plane) + } +} + +impl TargetIsa for Arm32Backend { + fn compile_function( + &self, + func: &Function, + domtree: &DominatorTree, + want_disasm: bool, + ctrl_plane: &mut ControlPlane, + ) -> CodegenResult { + let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?; + + let want_disasm = want_disasm || log::log_enabled!(log::Level::Debug); + let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane); + let value_labels_ranges = emit_result.value_labels_ranges; + let buffer = emit_result.buffer; + + if let Some(disasm) = emit_result.disasm.as_ref() { + log::debug!("disassembly:\n{disasm}"); + } + + Ok(CompiledCodeStencil { + buffer, + vcode: emit_result.disasm, + value_labels_ranges, + bb_starts: emit_result.bb_offsets, + bb_edges: emit_result.bb_edges, + }) + } + + fn name(&self) -> &'static str { + "arm32" + } + + fn dynamic_vector_bytes(&self, _dynamic_ty: ir::Type) -> u32 { + 0 + } + + fn triple(&self) -> &Triple { + &self.triple + } + + fn flags(&self) -> &shared_settings::Flags { + &self.flags + } + + fn isa_flags(&self) -> Vec { + self.isa_flags.iter().collect() + } + + fn isa_flags_hash_key(&self) -> IsaFlagsHashKey<'_> { + IsaFlagsHashKey(self.isa_flags.hash_key()) + } + + #[cfg(feature = "unwind")] + fn emit_unwind_info( + &self, + _result: &crate::machinst::CompiledCode, + _kind: crate::isa::unwind::UnwindInfoKind, + ) -> CodegenResult> { + // Unwind info is not yet emitted for arm32. + Ok(None) + } + + fn text_section_builder(&self, num_funcs: usize) -> Box { + Box::new(MachTextSectionBuilder::::new(num_funcs)) + } + + fn function_alignment(&self) -> FunctionAlignment { + inst::Inst::function_alignment() + } + + fn page_size_align_log2(&self) -> u8 { + debug_assert_eq!(1 << 12, 0x1000); + 12 + } + + #[cfg(feature = "disas")] + fn to_capstone(&self) -> Result { + use capstone::prelude::*; + let mut cs = Capstone::new() + .arm() + .mode(arch::arm::ArchMode::Arm) + .build()?; + cs.set_skipdata(true)?; + Ok(cs) + } + + fn pretty_print_reg(&self, reg: Reg, _size: u8) -> String { + inst::reg_name(reg) + } + + fn has_native_fma(&self) -> bool { + false + } + + fn has_round(&self) -> bool { + false + } + + fn has_blendv_lowering(&self, _: Type) -> bool { + false + } + + fn has_x86_pshufb_lowering(&self) -> bool { + false + } + + fn has_x86_pmulhrsw_lowering(&self) -> bool { + false + } + + fn has_x86_pmaddubsw_lowering(&self) -> bool { + false + } + + fn default_argument_extension(&self) -> ir::ArgumentExtension { + ir::ArgumentExtension::None + } +} + +impl fmt::Display for Arm32Backend { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("MachBackend") + .field("name", &self.name()) + .field("triple", &self.triple()) + .field("flags", &format!("{}", self.flags())) + .finish() + } +} + +/// Create a new `isa::Builder`. +pub fn isa_builder(triple: Triple) -> IsaBuilder { + match triple.architecture { + Architecture::Arm(..) => {} + _ => unreachable!(), + } + IsaBuilder { + triple, + setup: arm32_settings::builder(), + constructor: isa_constructor, + } +} + +fn isa_constructor( + triple: Triple, + shared_flags: Flags, + builder: &shared_settings::Builder, +) -> CodegenResult { + let isa_flags = arm32_settings::Flags::new(&shared_flags, builder); + let backend = Arm32Backend::new_with_flags(triple, shared_flags, isa_flags); + Ok(backend.wrapped()) +} diff --git a/cranelift/codegen/src/isa/arm32/settings.rs b/cranelift/codegen/src/isa/arm32/settings.rs new file mode 100644 index 000000000000..a3a500c65025 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/settings.rs @@ -0,0 +1,9 @@ +//! arm32 Settings. + +use crate::settings::{self, Builder, Value, detail}; +use core::fmt; + +// Include code generated by `cranelift/codegen/meta/src/gen_settings.rs`. This +// file contains a public `Flags` struct with an impl for all of the settings +// defined in `cranelift/codegen/meta/src/isa/arm32.rs`. +include!(concat!(env!("OUT_DIR"), "/settings-arm32.rs")); diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index f9e30082724e..6fbf8101cb0c 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -70,6 +70,9 @@ pub mod x64; #[cfg(feature = "arm64")] pub mod aarch64; +#[cfg(feature = "arm32")] +pub mod arm32; + #[cfg(feature = "riscv64")] pub mod riscv64; @@ -111,6 +114,7 @@ pub fn lookup(triple: Triple) -> Result { isa_builder!(x64, (feature = "x86"), triple) } Architecture::Aarch64 { .. } => isa_builder!(aarch64, (feature = "arm64"), triple), + Architecture::Arm(..) => isa_builder!(arm32, (feature = "arm32"), triple), Architecture::S390x { .. } => isa_builder!(s390x, (feature = "s390x"), triple), Architecture::Riscv64 { .. } => isa_builder!(riscv64, (feature = "riscv64"), triple), Architecture::Pulley32 | Architecture::Pulley32be => { @@ -126,7 +130,7 @@ pub fn lookup(triple: Triple) -> Result { /// The string names of all the supported, but possibly not enabled, architectures. The elements of /// this slice are suitable to be passed to the [lookup_by_name] function to obtain the default /// configuration for that architecture. -pub const ALL_ARCHITECTURES: &[&str] = &["x86_64", "aarch64", "s390x", "riscv64"]; +pub const ALL_ARCHITECTURES: &[&str] = &["x86_64", "aarch64", "arm", "s390x", "riscv64"]; /// Look for a supported ISA with the given `name`. /// Return a builder that can create a corresponding `TargetIsa`. diff --git a/cranelift/filetests/filetests/isa/arm32/iconst.clif b/cranelift/filetests/filetests/isa/arm32/iconst.clif new file mode 100644 index 000000000000..b7feaf52d425 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/iconst.clif @@ -0,0 +1,18 @@ +test compile precise-output +target arm + +function %f() -> i32 { +block0: + v0 = iconst.i32 42 + return v0 +} + +; VCode: +; block0: +; movw r0, #42 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; movw r0, #0x2a +; bx lr From 361bfb8f3e9c20978b2a60a156431c0f6371d46a Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Fri, 3 Jul 2026 02:14:23 +0300 Subject: [PATCH 02/18] arm32: add the core data, memory, and control-flow instructions Enough of the common instructions to lower straight-line integer code: mov/movw/movt/mvn, ldr/str and the sub-word and push/pop forms, add/sub/rsb in register and rotated-immediate forms, cmp/cmn, and the branch family (b, bl, bx, and conditional b). Each comes with its A32 encoding, operand handling, and pretty-printing. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/abi.rs | 78 +++--- cranelift/codegen/src/isa/arm32/inst.isle | 243 +++++++++++++---- cranelift/codegen/src/isa/arm32/inst/args.rs | 241 +++++++++++++++++ cranelift/codegen/src/isa/arm32/inst/emit.rs | 252 +++++++++++++++--- .../codegen/src/isa/arm32/inst/emit_tests.rs | 194 +++++++++++++- cranelift/codegen/src/isa/arm32/inst/mod.rs | 133 +++++++-- cranelift/codegen/src/isa/arm32/inst/regs.rs | 9 +- cranelift/codegen/src/isa/arm32/lower.isle | 62 ++++- cranelift/codegen/src/isa/arm32/lower/isle.rs | 47 +++- .../filetests/isa/arm32/arithmetic.clif | 68 +++++ .../filetests/filetests/isa/arm32/branch.clif | 39 +++ .../filetests/filetests/isa/arm32/iconst.clif | 5 +- .../filetests/filetests/isa/arm32/memory.clif | 83 ++++++ 13 files changed, 1293 insertions(+), 161 deletions(-) create mode 100644 cranelift/codegen/src/isa/arm32/inst/args.rs create mode 100644 cranelift/filetests/filetests/isa/arm32/arithmetic.clif create mode 100644 cranelift/filetests/filetests/isa/arm32/branch.clif create mode 100644 cranelift/filetests/filetests/isa/arm32/memory.clif diff --git a/cranelift/codegen/src/isa/arm32/abi.rs b/cranelift/codegen/src/isa/arm32/abi.rs index 9b76c1da0123..70c60c274849 100644 --- a/cranelift/codegen/src/isa/arm32/abi.rs +++ b/cranelift/codegen/src/isa/arm32/abi.rs @@ -119,12 +119,20 @@ impl ABIMachineSpec for Arm32MachineDeps { Ok((next_stack, pos)) } - fn gen_load_stack(_mem: StackAMode, _into_reg: Writable, _ty: Type) -> Inst { - unimplemented!("arm32: stack loads (spills/reloads) not yet implemented") + fn gen_load_stack(mem: StackAMode, into_reg: Writable, _ty: Type) -> Inst { + Inst::Load { + rt: into_reg, + mem: amode_from_stack(mem), + kind: LoadKind::Word, + } } - fn gen_store_stack(_mem: StackAMode, _from_reg: Reg, _ty: Type) -> Inst { - unimplemented!("arm32: stack stores (spills/reloads) not yet implemented") + fn gen_store_stack(mem: StackAMode, from_reg: Reg, _ty: Type) -> Inst { + Inst::Store { + rt: from_reg, + mem: amode_from_stack(mem), + kind: StoreKind::Word, + } } fn gen_move(to_reg: Writable, from_reg: Reg, ty: Type) -> Inst { @@ -180,16 +188,16 @@ impl ABIMachineSpec for Arm32MachineDeps { fn gen_load_base_offset(into_reg: Writable, base: Reg, offset: i32, _ty: Type) -> Inst { Inst::Load { rt: into_reg, - base, - offset, + mem: AMode::RegOffset { rn: base, offset }, + kind: LoadKind::Word, } } fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, _ty: Type) -> Inst { Inst::Store { rt: from_reg, - base, - offset, + mem: AMode::RegOffset { rn: base, offset }, + kind: StoreKind::Word, } } @@ -210,18 +218,10 @@ impl ABIMachineSpec for Arm32MachineDeps { let mut insts = smallvec![]; if frame_layout.setup_area_size > 0 { // push {fp, lr}; mov fp, sp - insts.push(Inst::AdjustSp { amount: -8 }); - insts.push(Inst::Store { - rt: link_reg(), - base: stack_reg(), - offset: 4, - }); - insts.push(Inst::Store { - rt: fp_reg(), - base: stack_reg(), - offset: 0, + insts.push(Inst::Push { + reg_list: FP_LR_REG_LIST, }); - insts.push(Inst::Mov { + insts.push(Inst::MovReg { rd: writable_fp_reg(), rm: stack_reg(), }); @@ -237,17 +237,10 @@ impl ABIMachineSpec for Arm32MachineDeps { ) -> SmallInstVec { let mut insts = smallvec![]; if frame_layout.setup_area_size > 0 { - insts.push(Inst::Load { - rt: writable_fp_reg(), - base: stack_reg(), - offset: 0, - }); - insts.push(Inst::Load { - rt: writable_link_reg(), - base: stack_reg(), - offset: 4, + // pop {fp, lr} + insts.push(Inst::Pop { + reg_list: FP_LR_REG_LIST, }); - insts.push(Inst::AdjustSp { amount: 8 }); } insts } @@ -288,8 +281,10 @@ impl ABIMachineSpec for Arm32MachineDeps { for reg in &frame_layout.clobbered_callee_saves { insts.push(Inst::Store { rt: Reg::from(reg.to_reg()), - base: stack_reg(), - offset: cur_offset, + mem: AMode::SPOffset { + offset: i64::from(cur_offset), + }, + kind: StoreKind::Word, }); cur_offset += 4; } @@ -310,8 +305,10 @@ impl ABIMachineSpec for Arm32MachineDeps { for reg in &frame_layout.clobbered_callee_saves { insts.push(Inst::Load { rt: reg.map(Reg::from), - base: stack_reg(), - offset: cur_offset, + mem: AMode::SPOffset { + offset: i64::from(cur_offset), + }, + kind: LoadKind::Word, }); cur_offset += 4; } @@ -413,6 +410,21 @@ impl ABIMachineSpec for Arm32MachineDeps { } } +/// Register-list mask for `{fp, lr}` (r11 and r14), as used by push/pop. +const FP_LR_REG_LIST: u32 = (1 << 11) | (1 << 14); + +/// Convert a `StackAMode` (a nominal stack reference) into an `AMode` that is +/// resolved against the frame layout at emit time. +fn amode_from_stack(mem: StackAMode) -> AMode { + match mem { + StackAMode::IncomingArg(off, stack_args_size) => AMode::IncomingArg { + offset: i64::from(stack_args_size) - off, + }, + StackAMode::Slot(off) => AMode::SlotOffset { offset: off }, + StackAMode::OutgoingArg(off) => AMode::SPOffset { offset: off }, + } +} + /// Callee-saved GPRs under AAPCS: r4-r11. const DEFAULT_CALLEE_SAVES: PRegSet = PRegSet::empty() .with(preg(4)) diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index 1c5339d92f42..03f99935ef23 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -1,8 +1,67 @@ -;; Arm32 instruction definitions. +;; arm32 (AArch32 / A32) instruction definitions. ;; ;; The `MInst` type below is the machine-instruction enum for this backend; it ;; is generated into Rust and re-exported as `Inst` by `inst/mod.rs`. +;;;; Supporting types ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; A data-processing ALU operation. +(type ALUOp + (enum + (Add) + (Sub) + (Rsb) + (And) + (Orr) + (Eor) + (Bic))) + +;; A flag-setting compare operation. +(type CmpOp + (enum + (Cmp) + (Cmn))) + +;; An ARM condition code (the "always" and "never" codes are omitted). +(type Cond + (enum + (Eq) (Ne) (Hs) (Lo) (Mi) (Pl) (Vs) (Vc) + (Hi) (Ls) (Ge) (Lt) (Gt) (Le))) + +;; The width/signedness of a load. +(type LoadKind + (enum + (Word) + (UByte) + (SByte) + (UHalf) + (SHalf))) + +;; The width of a store. +(type StoreKind + (enum + (Word) + (Byte) + (Half))) + +;; A memory addressing mode. +(type AMode + (enum + ;; `[rn, #offset]` + (RegOffset (rn Reg) (offset i32)) + ;; `[rn, rm]` + (RegReg (rn Reg) (rm Reg)) + ;; `[sp, #offset]` + (SPOffset (offset i64)) + ;; A reference to a stack slot, resolved to an SP offset at emit time. + (SlotOffset (offset i64)) + ;; A reference into the incoming-argument area, resolved at emit time. + (IncomingArg (offset i64)))) + +(type BoxCallInfo (primitive BoxCallInfo)) + +;;;; Instruction formats ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (type MInst (enum ;; A zero-length no-op. @@ -10,66 +69,156 @@ ;; A four-byte no-op. (Nop4) - ;; Load an arbitrary 32-bit immediate into `rd` (expands to `movw`, plus - ;; `movt` when the high half is non-zero). - (MovImm - (rd WritableReg) - (imm u64)) - - ;; Register-to-register move: `mov rd, rm`. - (Mov - (rd WritableReg) - (rm Reg)) - - ;; Adjust the stack pointer by a (signed) constant amount. - (AdjustSp - (amount i32)) - - ;; Store `rt` to `[base, #offset]`. - (Store - (rt Reg) - (base Reg) - (offset i32)) - - ;; Load `[base, #offset]` into `rt`. - (Load - (rt WritableReg) - (base Reg) - (offset i32)) - - ;; An unconditional branch to a label. - (Jump - (dest MachLabel)) + ;; Load an arbitrary 32-bit constant into `rd` via `movw` + `movt`. + (MovImm (rd WritableReg) (imm u64)) + ;; `mov rd, #imm` with a rotated 8-bit immediate (pre-encoded imm12). + (MovRotImm (rd WritableReg) (imm12 u32)) + ;; `mvn rd, #imm` with a rotated 8-bit immediate (pre-encoded imm12). + (MvnRotImm (rd WritableReg) (imm12 u32)) + ;; `movw rd, #imm16`. + (Movw (rd WritableReg) (imm16 u32)) + ;; `movt rd, #imm16` (sets the top half, preserving the low half). + (Movt (rd WritableReg) (imm16 u32)) + ;; `mov rd, rm`. + (MovReg (rd WritableReg) (rm Reg)) - ;; Pseudo-instruction capturing incoming register arguments in vregs. - (Args - (args VecArgPair)) + ;; `op rd, rn, rm`. + (AluRRR (op ALUOp) (rd WritableReg) (rn Reg) (rm Reg)) + ;; `op rd, rn, #imm` (pre-encoded rotated imm12). + (AluRRImm (op ALUOp) (rd WritableReg) (rn Reg) (imm12 u32)) - ;; Pseudo-instruction moving vregs into the return registers. - (Rets - (rets VecRetPair)) + ;; `cmp/cmn rn, rm`. + (CmpRR (op CmpOp) (rn Reg) (rm Reg)) + ;; `cmp/cmn rn, #imm` (pre-encoded rotated imm12). + (CmpRImm (op CmpOp) (rn Reg) (imm12 u32)) + + ;; A load of the given kind. + (Load (rt WritableReg) (mem AMode) (kind LoadKind)) + ;; A store of the given kind. + (Store (rt Reg) (mem AMode) (kind StoreKind)) + + ;; `push {reglist}` (STMDB sp!). The register list is a 16-bit mask. + (Push (reg_list u32)) + ;; `pop {reglist}` (LDMIA sp!). + (Pop (reg_list u32)) + + ;; Adjust the stack pointer by a signed constant (add/sub sp). + (AdjustSp (amount i32)) + + ;; A direct call (`bl`). + (Call (info BoxCallInfo)) + ;; An unconditional branch to a label (`b`). + (Jump (dest MachLabel)) + + ;; A conditional branch: `b taken; b not_taken`. + (CondBr (cond Cond) (taken MachLabel) (not_taken MachLabel)) + + ;; Pseudo-instruction capturing incoming register arguments in vregs. + (Args (args VecArgPair)) + ;; Pseudo-instruction moving vregs into the return registers. + (Rets (rets VecRetPair)) ;; Return from the current function (`bx lr`). (Ret) ) ) -;;;; Instruction-constructor helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; Constant materialization ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; Emit a `MovImm` and return its destination register. -(decl movimm (u64) Reg) -(rule (movimm n) - (let ((dst WritableReg (temp_writable_reg $I32)) - (_ Unit (emit (MInst.MovImm dst n)))) - dst)) +;; Materialize an integer constant into a register, choosing the shortest of +;; `mov`/`mvn`/`movw`/`movw+movt`. +(decl gen_constant (u64) Reg) +(extern constructor gen_constant gen_constant) ;; Materialize an integer constant of the given type into a register. (decl imm (Type u64) Reg) -(rule (imm _ty n) (movimm n)) +(rule (imm _ty n) (gen_constant n)) + +;;;; ALU helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `op rd, rn, rm`. +(decl alu_rrr (ALUOp Reg Reg) Reg) +(rule (alu_rrr op rn rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AluRRR op rd rn rm)))) + rd)) + +;; `op rd, rn, #imm` (immediate must be a rotated imm12). +(decl alu_rr_imm12 (ALUOp Reg u32) Reg) +(rule (alu_rr_imm12 op rn imm12) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AluRRImm op rd rn imm12)))) + rd)) + +;; A constant is encodable as an A32 data-processing immediate; the result is +;; its pre-encoded 12-bit form. +(decl pure partial u64_from_rotated_imm12 (u64) u32) +(extern constructor u64_from_rotated_imm12 u64_from_rotated_imm12) + +;; Add: prefer the immediate form when the RHS is an encodable constant. +(decl add_reg (Reg Reg) Reg) +(rule (add_reg a b) (alu_rrr (ALUOp.Add) a b)) + +(decl add_imm (Reg u64) Reg) +(rule 1 (add_imm a b) + (if-let imm12 (u64_from_rotated_imm12 b)) + (alu_rr_imm12 (ALUOp.Add) a imm12)) +(rule (add_imm a b) (alu_rrr (ALUOp.Add) a (gen_constant b))) + +(decl sub_reg (Reg Reg) Reg) +(rule (sub_reg a b) (alu_rrr (ALUOp.Sub) a b)) -;;;; Branch lowering ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(decl sub_imm (Reg u64) Reg) +(rule 1 (sub_imm a b) + (if-let imm12 (u64_from_rotated_imm12 b)) + (alu_rr_imm12 (ALUOp.Sub) a imm12)) +(rule (sub_imm a b) (alu_rrr (ALUOp.Sub) a (gen_constant b))) -;; An unconditional jump as a side effect. +;; Reverse subtract, used for negation: `rsb rd, rn, #0`. +(decl rsb_imm (Reg u32) Reg) +(rule (rsb_imm a imm12) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AluRRImm (ALUOp.Rsb) rd a imm12)))) + rd)) + +;;;; Loads and stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl arm_load (AMode LoadKind) Reg) +(rule (arm_load mem kind) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.Load rd mem kind)))) + rd)) + +(decl arm_store (Reg AMode StoreKind) InstOutput) +(rule (arm_store val mem kind) + (side_effect (SideEffectNoResult.Inst (MInst.Store val mem kind)))) + +;; Build an addressing mode from an address value plus a constant offset. +(decl amode (Value Offset32) AMode) +(rule (amode base offset) (AMode.RegOffset base (offset32_to_i32 offset))) + +;;;; Comparisons and branches ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Emit a compare (`cmp rn, rm`) as a side effect. +(decl cmp_rr (Reg Reg) SideEffectNoResult) +(rule (cmp_rr rn rm) + (SideEffectNoResult.Inst (MInst.CmpRR (CmpOp.Cmp) rn rm))) + +;; Emit a compare against a rotated imm12 (`cmp rn, #imm`) as a side effect. +(decl cmp_imm (Reg u32) SideEffectNoResult) +(rule (cmp_imm rn imm12) + (SideEffectNoResult.Inst (MInst.CmpRImm (CmpOp.Cmp) rn imm12))) + +;; A conditional branch. +(decl cond_br (Cond MachLabel MachLabel) SideEffectNoResult) +(rule (cond_br cond taken not_taken) + (SideEffectNoResult.Inst (MInst.CondBr cond taken not_taken))) + +;; An unconditional branch. (decl jump_impl (MachLabel) SideEffectNoResult) (rule (jump_impl label) (SideEffectNoResult.Inst (MInst.Jump label))) + +;; Map an `IntCC` to the corresponding ARM condition code. +(decl cond_from_intcc (IntCC) Cond) +(extern constructor cond_from_intcc cond_from_intcc) diff --git a/cranelift/codegen/src/isa/arm32/inst/args.rs b/cranelift/codegen/src/isa/arm32/inst/args.rs new file mode 100644 index 000000000000..beb0ee208040 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/inst/args.rs @@ -0,0 +1,241 @@ +//! arm32 addressing modes and immediate/operand helpers. + +use crate::isa::arm32::inst::*; +use crate::machinst::{OperandVisitor, Reg}; + +pub use crate::isa::arm32::lower::isle::generated_code::{ + ALUOp, AMode, CmpOp, Cond, LoadKind, StoreKind, +}; + +/// A memory address resolved to a concrete base register and either an +/// immediate or register offset, ready for encoding. +pub(crate) enum ResolvedAMode { + /// `[base, #offset]` + Imm { base: Reg, offset: i32 }, + /// `[base, index]` + Reg { base: Reg, index: Reg }, +} + +impl AMode { + /// Collect the register operands referenced by this addressing mode. + pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) { + // Only virtual registers are tracked; sp is implicit for the + // stack-relative modes. + fn use_if_virtual(collector: &mut impl OperandVisitor, reg: &mut Reg) { + if reg.to_real_reg().is_none() { + collector.reg_use(reg); + } + } + match self { + AMode::RegOffset { rn, .. } => use_if_virtual(collector, rn), + AMode::RegReg { rn, rm } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + } + AMode::SPOffset { .. } | AMode::SlotOffset { .. } | AMode::IncomingArg { .. } => {} + } + } + + /// Resolve this addressing mode against the current frame layout. + pub(crate) fn resolve(&self, state: &EmitState) -> ResolvedAMode { + match self { + &AMode::RegOffset { rn, offset } => ResolvedAMode::Imm { base: rn, offset }, + &AMode::RegReg { rn, rm } => ResolvedAMode::Reg { + base: rn, + index: rm, + }, + &AMode::SPOffset { offset } => ResolvedAMode::Imm { + base: stack_reg(), + offset: offset as i32, + }, + &AMode::SlotOffset { offset } => { + let fl = state.frame_layout(); + ResolvedAMode::Imm { + base: stack_reg(), + offset: (offset + i64::from(fl.outgoing_args_size)) as i32, + } + } + &AMode::IncomingArg { offset } => { + let fl = state.frame_layout(); + let sp_off = i64::from(fl.tail_args_size) + + i64::from(fl.setup_area_size) + + i64::from(fl.clobber_size) + + i64::from(fl.fixed_frame_storage_size) + + i64::from(fl.outgoing_args_size) + - offset; + ResolvedAMode::Imm { + base: stack_reg(), + offset: sp_off as i32, + } + } + } + } + + /// Pretty-print this addressing mode. + pub(crate) fn pretty_print(&self) -> String { + match self { + AMode::RegOffset { rn, offset } => { + alloc::format!("[{}, #{}]", reg_name(*rn), offset) + } + AMode::RegReg { rn, rm } => { + alloc::format!("[{}, {}]", reg_name(*rn), reg_name(*rm)) + } + AMode::SPOffset { offset } => alloc::format!("[sp, #{offset}]"), + AMode::SlotOffset { offset } => alloc::format!("[slot, #{offset}]"), + AMode::IncomingArg { offset } => alloc::format!("[incoming_arg, #{offset}]"), + } + } +} + +impl ALUOp { + /// The 4-bit data-processing opcode for this operation. + pub(crate) fn opcode(self) -> u32 { + match self { + ALUOp::And => 0b0000, + ALUOp::Eor => 0b0001, + ALUOp::Sub => 0b0010, + ALUOp::Rsb => 0b0011, + ALUOp::Add => 0b0100, + ALUOp::Orr => 0b1100, + ALUOp::Bic => 0b1110, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + ALUOp::And => "and", + ALUOp::Eor => "eor", + ALUOp::Sub => "sub", + ALUOp::Rsb => "rsb", + ALUOp::Add => "add", + ALUOp::Orr => "orr", + ALUOp::Bic => "bic", + } + } +} + +impl CmpOp { + /// The 4-bit data-processing opcode (with the S bit set separately). + pub(crate) fn opcode(self) -> u32 { + match self { + CmpOp::Cmp => 0b1010, + CmpOp::Cmn => 0b1011, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + CmpOp::Cmp => "cmp", + CmpOp::Cmn => "cmn", + } + } +} + +impl Cond { + /// The 4-bit condition-code field. + pub(crate) fn bits(self) -> u32 { + match self { + Cond::Eq => 0b0000, + Cond::Ne => 0b0001, + Cond::Hs => 0b0010, + Cond::Lo => 0b0011, + Cond::Mi => 0b0100, + Cond::Pl => 0b0101, + Cond::Vs => 0b0110, + Cond::Vc => 0b0111, + Cond::Hi => 0b1000, + Cond::Ls => 0b1001, + Cond::Ge => 0b1010, + Cond::Lt => 0b1011, + Cond::Gt => 0b1100, + Cond::Le => 0b1101, + } + } + + /// The condition that is true exactly when `self` is false. + pub(crate) fn invert(self) -> Cond { + match self { + Cond::Eq => Cond::Ne, + Cond::Ne => Cond::Eq, + Cond::Hs => Cond::Lo, + Cond::Lo => Cond::Hs, + Cond::Mi => Cond::Pl, + Cond::Pl => Cond::Mi, + Cond::Vs => Cond::Vc, + Cond::Vc => Cond::Vs, + Cond::Hi => Cond::Ls, + Cond::Ls => Cond::Hi, + Cond::Ge => Cond::Lt, + Cond::Lt => Cond::Ge, + Cond::Gt => Cond::Le, + Cond::Le => Cond::Gt, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + Cond::Eq => "eq", + Cond::Ne => "ne", + Cond::Hs => "hs", + Cond::Lo => "lo", + Cond::Mi => "mi", + Cond::Pl => "pl", + Cond::Vs => "vs", + Cond::Vc => "vc", + Cond::Hi => "hi", + Cond::Ls => "ls", + Cond::Ge => "ge", + Cond::Lt => "lt", + Cond::Gt => "gt", + Cond::Le => "le", + } + } +} + +impl LoadKind { + pub(crate) fn mnemonic(self) -> &'static str { + match self { + LoadKind::Word => "ldr", + LoadKind::UByte => "ldrb", + LoadKind::SByte => "ldrsb", + LoadKind::UHalf => "ldrh", + LoadKind::SHalf => "ldrsh", + } + } +} + +impl StoreKind { + pub(crate) fn mnemonic(self) -> &'static str { + match self { + StoreKind::Word => "str", + StoreKind::Byte => "strb", + StoreKind::Half => "strh", + } + } +} + +/// Decode a 12-bit rotated data-processing immediate back to its value, for +/// pretty-printing. +pub fn decode_rotated_imm(enc: u32) -> u32 { + let rot = (enc >> 8) & 0xf; + let imm8 = enc & 0xff; + imm8.rotate_right(2 * rot) +} + +/// Encode `val` as an A32 data-processing immediate: an 8-bit value rotated +/// right by an even amount. Returns the 12-bit encoded operand (`rot << 8 | +/// imm8`) if representable. +pub fn encode_rotated_imm(val: u32) -> Option { + if val <= 0xff { + return Some(val); + } + for rot in 1..16u32 { + // The immediate is `imm8` rotated right by `2 * rot`; to recover imm8 we + // rotate `val` left by the same amount. + let rotated = val.rotate_left(2 * rot); + if rotated <= 0xff { + return Some((rot << 8) | rotated); + } + } + None +} diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index b5e7bc27910f..ebf8cab594c0 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -1,5 +1,6 @@ //! arm32 (A32) ISA: binary code emission. +use crate::binemit::Reloc; use crate::ir; use crate::isa::arm32::inst::*; use crate::settings; @@ -81,70 +82,130 @@ fn machreg_to_gpr(reg: Reg) -> u32 { u32::from(reg.to_real_reg().unwrap().hw_enc() & 0xf) } -/// `movw rd, #imm16` — load a 16-bit immediate into the low half of `rd`, -/// zeroing the high half. +//============================================================================= +// Instruction encoders. + +/// A data-processing instruction with an immediate operand2 (`op{s} rd, rn, +/// #imm12`). `imm12` is the already-encoded rotated immediate. +fn enc_dp_imm(opcode: u32, s: u32, rd: u32, rn: u32, imm12: u32) -> u32 { + COND_AL | (1 << 25) | (opcode << 21) | (s << 20) | (rn << 16) | (rd << 12) | (imm12 & 0xfff) +} + +/// A data-processing instruction with a register operand2 (`op{s} rd, rn, rm`). +fn enc_dp_reg(opcode: u32, s: u32, rd: u32, rn: u32, rm: u32) -> u32 { + COND_AL | (opcode << 21) | (s << 20) | (rn << 16) | (rd << 12) | rm +} + +/// `movw rd, #imm16` — load a 16-bit immediate into the low half of `rd`. fn enc_movw(rd: u32, imm16: u32) -> u32 { let imm4 = (imm16 >> 12) & 0xf; let imm12 = imm16 & 0xfff; COND_AL | 0x0300_0000 | (imm4 << 16) | (rd << 12) | imm12 } -/// `movt rd, #imm16` — load a 16-bit immediate into the high half of `rd`, -/// preserving the low half. +/// `movt rd, #imm16` — load a 16-bit immediate into the high half of `rd`. fn enc_movt(rd: u32, imm16: u32) -> u32 { let imm4 = (imm16 >> 12) & 0xf; let imm12 = imm16 & 0xfff; COND_AL | 0x0340_0000 | (imm4 << 16) | (rd << 12) | imm12 } -/// `mov rd, rm` (register). -fn enc_mov(rd: u32, rm: u32) -> u32 { - COND_AL | 0x01a0_0000 | (rd << 12) | rm +/// `bx rm` — branch and exchange. +fn enc_bx(rm: u32) -> u32 { + COND_AL | 0x012f_ff10 | rm +} + +/// `b ` — unconditional branch (offset patched in later). +fn enc_b(imm24: u32) -> u32 { + COND_AL | 0x0a00_0000 | (imm24 & 0x00ff_ffff) +} + +/// `b ` — conditional branch. +fn enc_bcond(cond: Cond, imm24: u32) -> u32 { + (cond.bits() << 28) | 0x0a00_0000 | (imm24 & 0x00ff_ffff) } -/// `bx lr` — branch and exchange to the link register. -fn enc_bx_lr() -> u32 { - COND_AL | 0x012f_ff10 | 14 +/// `bl ` — branch with link (direct call). +fn enc_bl(imm24: u32) -> u32 { + COND_AL | 0x0b00_0000 | (imm24 & 0x00ff_ffff) } -/// `add`/`sub sp, sp, #imm` for a small `imm` (< 256, encoded with rotation 0). +/// `add`/`sub sp, sp, #imm`. fn enc_sp_adjust(amount: i32) -> u32 { - let sp = 13u32; - let (base, mag) = if amount < 0 { - (0x0240_0000u32, (-amount) as u32) // sub + let (op, mag) = if amount < 0 { + (ALUOp::Sub, (-amount) as u32) } else { - (0x0280_0000u32, amount as u32) // add + (ALUOp::Add, amount as u32) }; - assert!( - mag < 256, - "arm32 sp adjust out of simple-immediate range: {amount}" - ); - COND_AL | base | (sp << 16) | (sp << 12) | mag + let imm12 = encode_rotated_imm(mag).expect("arm32 sp adjust not encodable as a single immediate"); + enc_dp_imm(op.opcode(), 0, 13, 13, imm12) } -/// `str`/`ldr rt, [base, #offset]` with a 12-bit immediate offset. -fn enc_ldr_str(load: bool, rt: u32, base: u32, offset: i32) -> u32 { +/// `ldr`/`str{b} rt, [base, #±offset]` (word or byte). +fn enc_ldr_str_imm(load: bool, byte: bool, rt: u32, base: u32, offset: i32) -> u32 { let (u_bit, mag) = if offset < 0 { (0u32, (-offset) as u32) } else { (1u32, offset as u32) }; assert!(mag < 4096, "arm32 ldr/str offset out of range: {offset}"); - // cond 01 I(0) P(1) U W(0) B(0) L rn rt imm12 - let l_bit = if load { 1u32 } else { 0 }; COND_AL | 0x0400_0000 | (1 << 24) // P = 1 (pre-indexed) | (u_bit << 23) - | (l_bit << 20) + | (if byte { 1 } else { 0 } << 22) + | (if load { 1 } else { 0 } << 20) | (base << 16) | (rt << 12) | mag } -/// `b label` — unconditional branch (offset patched in later). -fn enc_b_placeholder() -> u32 { - COND_AL | 0x0a00_0000 +/// `ldr`/`str{b} rt, [base, index]` (word or byte, register offset). +fn enc_ldr_str_reg(load: bool, byte: bool, rt: u32, base: u32, index: u32) -> u32 { + COND_AL + | 0x0600_0000 + | (1 << 24) // P = 1 + | (1 << 23) // U = 1 (add) + | (if byte { 1 } else { 0 } << 22) + | (if load { 1 } else { 0 } << 20) + | (base << 16) + | (rt << 12) + | index +} + +/// Extra load/store (halfword and signed byte) with an 8-bit immediate offset. +fn enc_ldrh_strh_imm(load: bool, s: u32, h: u32, rt: u32, base: u32, offset: i32) -> u32 { + let (u_bit, mag) = if offset < 0 { + (0u32, (-offset) as u32) + } else { + (1u32, offset as u32) + }; + assert!(mag < 256, "arm32 ldrh/strh offset out of range: {offset}"); + let imm_h = (mag >> 4) & 0xf; + let imm_l = mag & 0xf; + COND_AL + | (1 << 24) // P = 1 + | (u_bit << 23) + | (1 << 22) // immediate form + | (if load { 1 } else { 0 } << 20) + | (base << 16) + | (rt << 12) + | (imm_h << 8) + | (1 << 7) + | (s << 6) + | (h << 5) + | (1 << 4) + | imm_l +} + +/// `push {reglist}` (STMDB sp!). +fn enc_push(reg_list: u32) -> u32 { + COND_AL | 0x092d_0000 | (reg_list & 0xffff) +} + +/// `pop {reglist}` (LDMIA sp!). +fn enc_pop(reg_list: u32) -> u32 { + COND_AL | 0x08bd_0000 | (reg_list & 0xffff) } fn put_u32(sink: &mut MachBuffer, word: u32) { @@ -157,12 +218,13 @@ impl MachInstEmit for Inst { type State = EmitState; type Info = EmitInfo; - fn emit(&self, sink: &mut MachBuffer, _emit_info: &Self::Info, _state: &mut EmitState) { + fn emit(&self, sink: &mut MachBuffer, _emit_info: &Self::Info, state: &mut EmitState) { let start = sink.cur_offset(); match self { Inst::Nop0 => {} Inst::Nop4 => put_u32(sink, COND_AL | 0x0320_f000), - Inst::Ret => put_u32(sink, enc_bx_lr()), + Inst::Ret => put_u32(sink, enc_bx(14)), + Inst::MovImm { rd, imm } => { let rd = machreg_to_gpr(rd.to_reg()); put_u32(sink, enc_movw(rd, (imm & 0xffff) as u32)); @@ -170,27 +232,87 @@ impl MachInstEmit for Inst { put_u32(sink, enc_movt(rd, (imm >> 16) as u32)); } } - Inst::Mov { rd, rm } => { + Inst::MovRotImm { rd, imm12 } => { + let rd = machreg_to_gpr(rd.to_reg()); + put_u32(sink, enc_dp_imm(0b1101, 0, rd, 0, *imm12)); + } + Inst::MvnRotImm { rd, imm12 } => { + let rd = machreg_to_gpr(rd.to_reg()); + put_u32(sink, enc_dp_imm(0b1111, 0, rd, 0, *imm12)); + } + Inst::Movw { rd, imm16 } => { + put_u32(sink, enc_movw(machreg_to_gpr(rd.to_reg()), *imm16)); + } + Inst::Movt { rd, imm16 } => { + put_u32(sink, enc_movt(machreg_to_gpr(rd.to_reg()), *imm16)); + } + Inst::MovReg { rd, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_dp_reg(0b1101, 0, rd, 0, rm)); + } + + Inst::AluRRR { op, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_dp_reg(op.opcode(), 0, rd, rn, rm)); + } + Inst::AluRRImm { op, rd, rn, imm12 } => { let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_dp_imm(op.opcode(), 0, rd, rn, *imm12)); + } + Inst::CmpRR { op, rn, rm } => { + let rn = machreg_to_gpr(*rn); let rm = machreg_to_gpr(*rm); - put_u32(sink, enc_mov(rd, rm)); + put_u32(sink, enc_dp_reg(op.opcode(), 1, 0, rn, rm)); } + Inst::CmpRImm { op, rn, imm12 } => { + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_dp_imm(op.opcode(), 1, 0, rn, *imm12)); + } + Inst::AdjustSp { amount } => put_u32(sink, enc_sp_adjust(*amount)), - Inst::Store { rt, base, offset } => { + + Inst::Load { rt, mem, kind } => { + let rt = machreg_to_gpr(rt.to_reg()); + emit_load_store(sink, rt, mem, state, LoadStore::Load(*kind)); + } + Inst::Store { rt, mem, kind } => { let rt = machreg_to_gpr(*rt); - let base = machreg_to_gpr(*base); - put_u32(sink, enc_ldr_str(false, rt, base, *offset)); + emit_load_store(sink, rt, mem, state, LoadStore::Store(*kind)); } - Inst::Load { rt, base, offset } => { - let rt = machreg_to_gpr(rt.to_reg()); - let base = machreg_to_gpr(*base); - put_u32(sink, enc_ldr_str(true, rt, base, *offset)); + + Inst::Push { reg_list } => put_u32(sink, enc_push(*reg_list)), + Inst::Pop { reg_list } => put_u32(sink, enc_pop(*reg_list)), + + Inst::Call { info } => { + sink.add_reloc(Reloc::Arm32Call, &info.dest, 0); + put_u32(sink, enc_bl(0)); } + Inst::Jump { dest } => { sink.use_label_at_offset(sink.cur_offset(), *dest, LabelUse::Branch26); sink.add_uncond_branch(sink.cur_offset(), sink.cur_offset() + 4, *dest); - put_u32(sink, enc_b_placeholder()); + put_u32(sink, enc_b(0)); + } + Inst::CondBr { + cond, + taken, + not_taken, + } => { + // Conditional branch to `taken`. + let inverted = enc_bcond(cond.invert(), 0).to_le_bytes(); + sink.use_label_at_offset(sink.cur_offset(), *taken, LabelUse::Branch26); + sink.add_cond_branch(sink.cur_offset(), sink.cur_offset() + 4, *taken, &inverted); + put_u32(sink, enc_bcond(*cond, 0)); + // Unconditional branch to `not_taken`. + sink.use_label_at_offset(sink.cur_offset(), *not_taken, LabelUse::Branch26); + sink.add_uncond_branch(sink.cur_offset(), sink.cur_offset() + 4, *not_taken); + put_u32(sink, enc_b(0)); } + Inst::Args { .. } | Inst::Rets { .. } => { // Pseudo-instructions: no machine code. } @@ -207,3 +329,53 @@ impl MachInstEmit for Inst { self.print_with_state(state) } } + +enum LoadStore { + Load(LoadKind), + Store(StoreKind), +} + +fn emit_load_store( + sink: &mut MachBuffer, + rt: u32, + mem: &AMode, + state: &EmitState, + op: LoadStore, +) { + match mem.resolve(state) { + ResolvedAMode::Imm { base, offset } => { + let base = machreg_to_gpr(base); + let word = match op { + LoadStore::Load(LoadKind::Word) => enc_ldr_str_imm(true, false, rt, base, offset), + LoadStore::Load(LoadKind::UByte) => enc_ldr_str_imm(true, true, rt, base, offset), + LoadStore::Load(LoadKind::SByte) => { + enc_ldrh_strh_imm(true, 1, 0, rt, base, offset) + } + LoadStore::Load(LoadKind::UHalf) => { + enc_ldrh_strh_imm(true, 0, 1, rt, base, offset) + } + LoadStore::Load(LoadKind::SHalf) => { + enc_ldrh_strh_imm(true, 1, 1, rt, base, offset) + } + LoadStore::Store(StoreKind::Word) => enc_ldr_str_imm(false, false, rt, base, offset), + LoadStore::Store(StoreKind::Byte) => enc_ldr_str_imm(false, true, rt, base, offset), + LoadStore::Store(StoreKind::Half) => { + enc_ldrh_strh_imm(false, 0, 1, rt, base, offset) + } + }; + put_u32(sink, word); + } + ResolvedAMode::Reg { base, index } => { + let base = machreg_to_gpr(base); + let index = machreg_to_gpr(index); + let word = match op { + LoadStore::Load(LoadKind::Word) => enc_ldr_str_reg(true, false, rt, base, index), + LoadStore::Load(LoadKind::UByte) => enc_ldr_str_reg(true, true, rt, base, index), + LoadStore::Store(StoreKind::Word) => enc_ldr_str_reg(false, false, rt, base, index), + LoadStore::Store(StoreKind::Byte) => enc_ldr_str_reg(false, true, rt, base, index), + _ => unimplemented!("arm32: register-offset sub-word access not yet implemented"), + }; + put_u32(sink, word); + } + } +} diff --git a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs index 3db1d409fbfb..672eac84e042 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs @@ -27,36 +27,202 @@ fn u32_le(inst: Inst) -> u32 { u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) } -#[test] -fn bx_lr() { - assert_eq!(u32_le(Inst::Ret), 0xe12f_ff1e); +fn rot(v: u32) -> u32 { + encode_rotated_imm(v).unwrap() } #[test] -fn nop4() { +fn data_movement() { + assert_eq!(u32_le(Inst::Ret), 0xe12f_ff1e); // bx lr assert_eq!(u32_le(Inst::Nop4), 0xe320_f000); + assert_eq!( + u32_le(Inst::MovReg { + rd: writable_xreg(0), + rm: xreg(1), + }), + 0xe1a0_0001 + ); + assert_eq!( + u32_le(Inst::Movw { + rd: writable_xreg(0), + imm16: 42, + }), + 0xe300_002a + ); + assert_eq!( + u32_le(Inst::MovImm { + rd: writable_xreg(0), + imm: 42, + }), + 0xe300_002a + ); + assert_eq!( + u32_le(Inst::MovRotImm { + rd: writable_xreg(0), + imm12: rot(255), + }), + 0xe3a0_00ff // mov r0, #255 + ); + assert_eq!( + u32_le(Inst::MvnRotImm { + rd: writable_xreg(0), + imm12: rot(0), + }), + 0xe3e0_0000 // mvn r0, #0 + ); } #[test] -fn mov_reg() { - // mov r0, r1 +fn arithmetic() { assert_eq!( - u32_le(Inst::Mov { + u32_le(Inst::AluRRR { + op: ALUOp::Add, rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe081_0002 // add r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::AluRRImm { + op: ALUOp::Add, + rd: writable_xreg(0), + rn: xreg(1), + imm12: rot(1), + }), + 0xe281_0001 // add r0, r1, #1 + ); + assert_eq!( + u32_le(Inst::AluRRR { + op: ALUOp::Sub, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe041_0002 // sub r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::AluRRImm { + op: ALUOp::Rsb, + rd: writable_xreg(0), + rn: xreg(1), + imm12: rot(0), + }), + 0xe261_0000 // rsb r0, r1, #0 + ); +} + +#[test] +fn compares() { + assert_eq!( + u32_le(Inst::CmpRR { + op: CmpOp::Cmp, + rn: xreg(0), rm: xreg(1), }), - 0xe1a0_0001 + 0xe150_0001 // cmp r0, r1 + ); + assert_eq!( + u32_le(Inst::CmpRImm { + op: CmpOp::Cmp, + rn: xreg(0), + imm12: rot(0), + }), + 0xe350_0000 // cmp r0, #0 ); } #[test] -fn movw_small_imm() { - // movw r0, #42 +fn memory() { assert_eq!( - u32_le(Inst::MovImm { - rd: writable_xreg(0), - imm: 42, + u32_le(Inst::Load { + rt: writable_xreg(0), + mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + kind: LoadKind::Word, }), - 0xe300_002a + 0xe591_0004 // ldr r0, [r1, #4] ); + assert_eq!( + u32_le(Inst::Store { + rt: xreg(0), + mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + kind: StoreKind::Word, + }), + 0xe581_0004 // str r0, [r1, #4] + ); + assert_eq!( + u32_le(Inst::Load { + rt: writable_xreg(0), + mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + kind: LoadKind::UByte, + }), + 0xe5d1_0004 // ldrb r0, [r1, #4] + ); + assert_eq!( + u32_le(Inst::Store { + rt: xreg(0), + mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + kind: StoreKind::Byte, + }), + 0xe5c1_0004 // strb r0, [r1, #4] + ); + assert_eq!( + u32_le(Inst::Load { + rt: writable_xreg(0), + mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + kind: LoadKind::UHalf, + }), + 0xe1d1_00b4 // ldrh r0, [r1, #4] + ); + assert_eq!( + u32_le(Inst::Store { + rt: xreg(0), + mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + kind: StoreKind::Half, + }), + 0xe1c1_00b4 // strh r0, [r1, #4] + ); + assert_eq!( + u32_le(Inst::Load { + rt: writable_xreg(0), + mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + kind: LoadKind::SByte, + }), + 0xe1d1_00d4 // ldrsb r0, [r1, #4] + ); + assert_eq!( + u32_le(Inst::Load { + rt: writable_xreg(0), + mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + kind: LoadKind::SHalf, + }), + 0xe1d1_00f4 // ldrsh r0, [r1, #4] + ); + // Register-offset word forms. + assert_eq!( + u32_le(Inst::Load { + rt: writable_xreg(0), + mem: AMode::RegReg { rn: xreg(1), rm: xreg(2) }, + kind: LoadKind::Word, + }), + 0xe791_0002 // ldr r0, [r1, r2] + ); + assert_eq!( + u32_le(Inst::Store { + rt: xreg(0), + mem: AMode::RegReg { rn: xreg(1), rm: xreg(2) }, + kind: StoreKind::Word, + }), + 0xe781_0002 // str r0, [r1, r2] + ); +} + +#[test] +fn push_pop_and_sp() { + // push/pop {fp, lr} + let list = (1 << 11) | (1 << 14); + assert_eq!(u32_le(Inst::Push { reg_list: list }), 0xe92d_4800); + assert_eq!(u32_le(Inst::Pop { reg_list: list }), 0xe8bd_4800); + assert_eq!(u32_le(Inst::AdjustSp { amount: -8 }), 0xe24d_d008); + assert_eq!(u32_le(Inst::AdjustSp { amount: 8 }), 0xe28d_d008); } diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index 872a16d06642..07e89cff5ed4 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -13,6 +13,8 @@ use alloc::vec::Vec; use core::fmt::Write; use regalloc2::RegClass; +pub mod args; +pub use self::args::*; pub mod regs; pub use self::regs::*; pub mod emit; @@ -44,20 +46,63 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { } match inst { - Inst::Nop0 | Inst::Nop4 | Inst::Ret | Inst::AdjustSp { .. } | Inst::Jump { .. } => {} - Inst::MovImm { rd, .. } => def_if_virtual(collector, rd), - Inst::Mov { rd, rm } => { + Inst::Nop0 + | Inst::Nop4 + | Inst::Ret + | Inst::AdjustSp { .. } + | Inst::Jump { .. } + | Inst::CondBr { .. } + | Inst::Push { .. } + | Inst::Pop { .. } => {} + + Inst::MovImm { rd, .. } + | Inst::MovRotImm { rd, .. } + | Inst::MvnRotImm { rd, .. } + | Inst::Movw { rd, .. } + | Inst::Movt { rd, .. } => def_if_virtual(collector, rd), + + Inst::MovReg { rd, rm } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::AluRRR { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); use_if_virtual(collector, rm); def_if_virtual(collector, rd); } - Inst::Store { rt, base, .. } => { + Inst::AluRRImm { rd, rn, .. } => { + use_if_virtual(collector, rn); + def_if_virtual(collector, rd); + } + Inst::CmpRR { rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + } + Inst::CmpRImm { rn, .. } => use_if_virtual(collector, rn), + + Inst::Store { rt, mem, .. } => { use_if_virtual(collector, rt); - use_if_virtual(collector, base); + mem.get_operands(collector); } - Inst::Load { rt, base, .. } => { - use_if_virtual(collector, base); + Inst::Load { rt, mem, .. } => { + mem.get_operands(collector); def_if_virtual(collector, rt); } + + Inst::Call { info } => { + let CallInfo { uses, defs, .. } = &mut **info; + for CallArgPair { vreg, preg } in uses { + collector.reg_fixed_use(vreg, *preg); + } + for CallRetPair { vreg, location } in defs { + match location { + RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg), + RetLocation::Stack(..) => collector.any_def(vreg), + } + } + collector.reg_clobbers(info.clobbers); + } + Inst::Args { args } => { for ArgPair { vreg, preg } in args { collector.reg_fixed_def(vreg, *preg); @@ -93,7 +138,7 @@ impl MachInst for Inst { } fn is_safepoint(&self) -> bool { - false + matches!(self, Inst::Call { .. }) } fn get_operands(&mut self, collector: &mut impl OperandVisitor) { @@ -102,7 +147,7 @@ impl MachInst for Inst { fn is_move(&self) -> Option<(Writable, Reg)> { match self { - Inst::Mov { rd, rm } => Some((*rd, *rm)), + Inst::MovReg { rd, rm } => Some((*rd, *rm)), _ => None, } } @@ -120,13 +165,16 @@ impl MachInst for Inst { } fn call_type(&self) -> CallType { - CallType::None + match self { + Inst::Call { .. } => CallType::Regular, + _ => CallType::None, + } } fn is_term(&self) -> MachTerminator { match self { Inst::Rets { .. } => MachTerminator::Ret, - Inst::Jump { .. } => MachTerminator::Branch, + Inst::Jump { .. } | Inst::CondBr { .. } => MachTerminator::Branch, _ => MachTerminator::None, } } @@ -136,7 +184,7 @@ impl MachInst for Inst { } fn gen_move(to_reg: Writable, from_reg: Reg, _ty: Type) -> Inst { - Inst::Mov { + Inst::MovReg { rd: to_reg, rm: from_reg, } @@ -197,6 +245,13 @@ impl MachInst for Inst { impl Inst { pub(crate) fn print_with_state(&self, _state: &mut EmitState) -> String { let r = |reg: Reg| reg_name(reg); + let reglist = |mask: u32| -> String { + let names: Vec = (0..16) + .filter(|i| mask & (1 << i) != 0) + .map(|i| reg_name(xreg(i))) + .collect(); + alloc::format!("{{{}}}", names.join(", ")) + }; match self { Inst::Nop0 => "nop-zero-len".to_string(), Inst::Nop4 => "nop".to_string(), @@ -209,9 +264,35 @@ impl Inst { alloc::format!("movw {rd}, #{}; movt {rd}, #{}", imm & 0xffff, imm >> 16) } } - Inst::Mov { rd, rm } => { + Inst::MovRotImm { rd, imm12 } => { + alloc::format!("mov {}, #{}", r(rd.to_reg()), decode_rotated_imm(*imm12)) + } + Inst::MvnRotImm { rd, imm12 } => { + alloc::format!("mvn {}, #{}", r(rd.to_reg()), decode_rotated_imm(*imm12)) + } + Inst::Movw { rd, imm16 } => alloc::format!("movw {}, #{}", r(rd.to_reg()), imm16), + Inst::Movt { rd, imm16 } => alloc::format!("movt {}, #{}", r(rd.to_reg()), imm16), + Inst::MovReg { rd, rm } => { alloc::format!("mov {}, {}", r(rd.to_reg()), r(*rm)) } + Inst::AluRRR { op, rd, rn, rm } => { + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::AluRRImm { op, rd, rn, imm12 } => { + alloc::format!( + "{} {}, {}, #{}", + op.name(), + r(rd.to_reg()), + r(*rn), + decode_rotated_imm(*imm12) + ) + } + Inst::CmpRR { op, rn, rm } => { + alloc::format!("{} {}, {}", op.name(), r(*rn), r(*rm)) + } + Inst::CmpRImm { op, rn, imm12 } => { + alloc::format!("{} {}, #{}", op.name(), r(*rn), decode_rotated_imm(*imm12)) + } Inst::AdjustSp { amount } => { if *amount < 0 { alloc::format!("sub sp, sp, #{}", -*amount) @@ -219,13 +300,31 @@ impl Inst { alloc::format!("add sp, sp, #{amount}") } } - Inst::Store { rt, base, offset } => { - alloc::format!("str {}, [{}, #{}]", r(*rt), r(*base), offset) + Inst::Store { rt, mem, kind } => { + alloc::format!("{} {}, {}", kind.mnemonic(), r(*rt), mem.pretty_print()) } - Inst::Load { rt, base, offset } => { - alloc::format!("ldr {}, [{}, #{}]", r(rt.to_reg()), r(*base), offset) + Inst::Load { rt, mem, kind } => { + alloc::format!( + "{} {}, {}", + kind.mnemonic(), + r(rt.to_reg()), + mem.pretty_print() + ) } + Inst::Push { reg_list } => alloc::format!("push {}", reglist(*reg_list)), + Inst::Pop { reg_list } => alloc::format!("pop {}", reglist(*reg_list)), + Inst::Call { info } => alloc::format!("bl {}", info.dest.display(None)), Inst::Jump { dest } => alloc::format!("b {}", dest.to_string()), + Inst::CondBr { + cond, + taken, + not_taken, + } => alloc::format!( + "b{} {}; b {}", + cond.name(), + taken.to_string(), + not_taken.to_string() + ), Inst::Args { args } => { let mut s = "args".to_string(); for arg in args { diff --git a/cranelift/codegen/src/isa/arm32/inst/regs.rs b/cranelift/codegen/src/isa/arm32/inst/regs.rs index c7f252f51fcf..a9ef5b5bf1cd 100644 --- a/cranelift/codegen/src/isa/arm32/inst/regs.rs +++ b/cranelift/codegen/src/isa/arm32/inst/regs.rs @@ -71,12 +71,14 @@ pub fn writable_stack_reg() -> Writable { /// Link register (r14). #[inline] +#[allow(dead_code, reason = "part of the register API, used as the backend grows")] pub fn link_reg() -> Reg { xreg(14) } /// Writable link register. #[inline] +#[allow(dead_code, reason = "part of the register API, used as the backend grows")] pub fn writable_link_reg() -> Writable { Writable::from_reg(link_reg()) } @@ -112,9 +114,9 @@ pub fn reg_name(reg: Reg) -> alloc::string::String { pub const fn create_reg_environment() -> MachineEnv { // Preferred registers are the caller-saved (scratch) GPRs; the allocator // reaches for these first since they don't need to be saved/restored. - let preferred_int = PRegSet_int(&[0, 1, 2, 3]); + let preferred_int = preg_set_int(&[0, 1, 2, 3]); // Non-preferred registers are the callee-saved GPRs. - let non_preferred_int = PRegSet_int(&[4, 5, 6, 7, 8, 9, 10]); + let non_preferred_int = preg_set_int(&[4, 5, 6, 7, 8, 9, 10]); MachineEnv { preferred_regs_by_class: [preferred_int, empty(), empty()], @@ -124,8 +126,7 @@ pub const fn create_reg_environment() -> MachineEnv { } } -#[allow(non_snake_case)] -const fn PRegSet_int(encs: &[u8]) -> regalloc2::PRegSet { +const fn preg_set_int(encs: &[u8]) -> regalloc2::PRegSet { let mut set = regalloc2::PRegSet::empty(); let mut i = 0; while i < encs.len() { diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index c5e1ac9fe3b5..cec745058041 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -3,19 +3,75 @@ ;; The main lowering entry point. (decl partial lower (Inst) InstOutput) -;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (iconst (fits_in_32 ty) (u64_from_imm64 n))) (imm ty n)) -;;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; `iadd` / `isub` / `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule 2 (lower (iadd (fits_in_32 _) x (iconst _ (u64_from_imm64 n)))) + (add_imm (put_in_reg x) n)) +(rule 1 (lower (iadd (fits_in_32 _) (iconst _ (u64_from_imm64 n)) y)) + (add_imm (put_in_reg y) n)) +(rule (lower (iadd (fits_in_32 _) x y)) + (add_reg (put_in_reg x) (put_in_reg y))) + +(rule 1 (lower (isub (fits_in_32 _) x (iconst _ (u64_from_imm64 n)))) + (sub_imm (put_in_reg x) n)) +(rule (lower (isub (fits_in_32 _) x y)) + (sub_reg (put_in_reg x) (put_in_reg y))) + +;; Negation is a reverse-subtract from zero. +(rule (lower (ineg (fits_in_32 _) x)) + (rsb_imm (put_in_reg x) 0)) + +;;;; Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (load (fits_in_32 _) flags addr offset)) + (arm_load (amode addr offset) (LoadKind.Word))) + +(rule (lower (uload8 _ flags addr offset)) + (arm_load (amode addr offset) (LoadKind.UByte))) +(rule (lower (sload8 _ flags addr offset)) + (arm_load (amode addr offset) (LoadKind.SByte))) +(rule (lower (uload16 _ flags addr offset)) + (arm_load (amode addr offset) (LoadKind.UHalf))) +(rule (lower (sload16 _ flags addr offset)) + (arm_load (amode addr offset) (LoadKind.SHalf))) + +;;;; Stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (store flags val @ (value_type (fits_in_32 _)) addr offset)) + (arm_store (put_in_reg val) (amode addr offset) (StoreKind.Word))) + +(rule (lower (istore8 flags val addr offset)) + (arm_store (put_in_reg val) (amode addr offset) (StoreKind.Byte))) +(rule (lower (istore16 flags val addr offset)) + (arm_store (put_in_reg val) (amode addr offset) (StoreKind.Half))) + +;;;; `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (return args)) (lower_return args)) -;;;; Branch lowering ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; Branches ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl partial lower_branch (Inst MachLabelSlice) Unit) (rule (lower_branch (jump _) (single_target label)) (emit_side_effect (jump_impl label))) + +;; Fuse an `icmp` feeding a `brif` into a compare plus a conditional branch. +(rule 1 (lower_branch (brif (icmp _ cc a b) _ _) (two_targets taken not_taken)) + (emit_side_effect + (side_effect_concat + (cmp_rr (put_in_reg a) (put_in_reg b)) + (cond_br (cond_from_intcc cc) taken not_taken)))) + +;; Generic `brif`: branch when the value is non-zero. +(rule (lower_branch (brif v _ _) (two_targets taken not_taken)) + (emit_side_effect + (side_effect_concat + (cmp_imm (put_in_reg v) 0) + (cond_br (Cond.Ne) taken not_taken)))) diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs index d53421b44a03..406120ea043c 100644 --- a/cranelift/codegen/src/isa/arm32/lower/isle.rs +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -12,9 +12,10 @@ use crate::ir::{ BlockCall, ExternalName, Inst, InstructionData, MemFlags, Opcode, TrapCode, Value, ValueList, }; use crate::isa::arm32::Arm32Backend; +use crate::isa::arm32::inst::{Cond, encode_rotated_imm}; use crate::machinst::isle::*; use crate::machinst::{ - ArgPair, CallArgList, CallRetList, InstOutput, Lower, MachInst, MachLabel, RetPair, + ArgPair, CallArgList, CallInfo, CallRetList, InstOutput, Lower, MachInst, MachLabel, RetPair, VCodeConstant, VCodeConstantData, VCodeInst, }; use alloc::boxed::Box; @@ -23,6 +24,7 @@ use regalloc2::PReg; type VecArgPair = Vec; type VecRetPair = Vec; +type BoxCallInfo = Box>; /// The ISLE lowering context for arm32. pub(crate) struct Arm32IsleContext<'a, 'b, I, B> @@ -51,6 +53,49 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { fn emit(&mut self, inst: &MInst) -> Unit { self.lower_ctx.emit(inst.clone()); } + + /// Materialize a 32-bit constant into a register with the shortest sequence. + fn gen_constant(&mut self, val: u64) -> Reg { + let val = val as u32; + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + let inst = if let Some(imm12) = encode_rotated_imm(val) { + MInst::MovRotImm { rd, imm12 } + } else if let Some(imm12) = encode_rotated_imm(!val) { + MInst::MvnRotImm { rd, imm12 } + } else if val >> 16 == 0 { + MInst::Movw { + rd, + imm16: val, + } + } else { + MInst::MovImm { + rd, + imm: u64::from(val), + } + }; + self.lower_ctx.emit(inst); + rd.to_reg() + } + + /// Succeeds if the low 32 bits of `val` are encodable as a rotated imm12. + fn u64_from_rotated_imm12(&mut self, val: u64) -> Option { + encode_rotated_imm(val as u32) + } + + fn cond_from_intcc(&mut self, cc: &IntCC) -> Cond { + match cc { + IntCC::Equal => Cond::Eq, + IntCC::NotEqual => Cond::Ne, + IntCC::SignedLessThan => Cond::Lt, + IntCC::SignedGreaterThanOrEqual => Cond::Ge, + IntCC::SignedGreaterThan => Cond::Gt, + IntCC::SignedLessThanOrEqual => Cond::Le, + IntCC::UnsignedLessThan => Cond::Lo, + IntCC::UnsignedGreaterThanOrEqual => Cond::Hs, + IntCC::UnsignedGreaterThan => Cond::Hi, + IntCC::UnsignedLessThanOrEqual => Cond::Ls, + } + } } /// The main entry point for lowering with ISLE. diff --git a/cranelift/filetests/filetests/isa/arm32/arithmetic.clif b/cranelift/filetests/filetests/isa/arm32/arithmetic.clif new file mode 100644 index 000000000000..245e7955f028 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/arithmetic.clif @@ -0,0 +1,68 @@ +test compile precise-output +target arm + +function %add(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = iadd v0, v1 + return v2 +} + +; VCode: +; block0: +; add r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; add r0, r0, r1 +; bx lr + +function %add_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 10 + v2 = iadd v0, v1 + return v2 +} + +; VCode: +; block0: +; add r0, r0, #10 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; add r0, r0, #0xa +; bx lr + +function %sub(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = isub v0, v1 + return v2 +} + +; VCode: +; block0: +; sub r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; sub r0, r0, r1 +; bx lr + +function %neg(i32) -> i32 { +block0(v0: i32): + v1 = ineg v0 + return v1 +} + +; VCode: +; block0: +; rsb r0, r0, #0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; rsb r0, r0, #0 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/branch.clif b/cranelift/filetests/filetests/isa/arm32/branch.clif new file mode 100644 index 000000000000..985216b32e9c --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/branch.clif @@ -0,0 +1,39 @@ +test compile precise-output +target arm + +function %select_via_brif(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = icmp slt v0, v1 + brif v2, block1, block2 + +block1: + v3 = iconst.i32 1 + return v3 + +block2: + v4 = iconst.i32 0 + return v4 +} + +; VCode: +; block0: +; cmp r0, r1 +; blt label2; b label1 +; block1: +; mov r0, #0 +; bx lr +; block2: +; mov r0, #1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; cmp r0, r1 +; blt #0x10 +; block1: ; offset 0x8 +; mov r0, #0 +; bx lr +; block2: ; offset 0x10 +; mov r0, #1 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/iconst.clif b/cranelift/filetests/filetests/isa/arm32/iconst.clif index b7feaf52d425..eddd8d854d41 100644 --- a/cranelift/filetests/filetests/isa/arm32/iconst.clif +++ b/cranelift/filetests/filetests/isa/arm32/iconst.clif @@ -9,10 +9,11 @@ block0: ; VCode: ; block0: -; movw r0, #42 +; mov r0, #42 ; bx lr ; ; Disassembled: ; block0: ; offset 0x0 -; movw r0, #0x2a +; mov r0, #0x2a ; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/memory.clif b/cranelift/filetests/filetests/isa/arm32/memory.clif new file mode 100644 index 000000000000..b3be1e5b1b44 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/memory.clif @@ -0,0 +1,83 @@ +test compile precise-output +target arm + +function %load_word(i32) -> i32 { +block0(v0: i32): + v1 = load.i32 v0+4 + return v1 +} + +; VCode: +; block0: +; ldr r0, [r0, #4] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldr r0, [r0, #4] +; bx lr + +function %store_word(i32, i32) { +block0(v0: i32, v1: i32): + store.i32 v1, v0+8 + return +} + +; VCode: +; block0: +; str r1, [r0, #8] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; str r1, [r0, #8] +; bx lr + +function %load_u8(i32) -> i32 { +block0(v0: i32): + v1 = uload8.i32 v0 + return v1 +} + +; VCode: +; block0: +; ldrb r0, [r0, #0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldrb r0, [r0] +; bx lr + +function %store_u8(i32, i32) { +block0(v0: i32, v1: i32): + istore8 v1, v0 + return +} + +; VCode: +; block0: +; strb r1, [r0, #0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; strb r1, [r0] +; bx lr + +function %load_s16(i32) -> i32 { +block0(v0: i32): + v1 = sload16.i32 v0+2 + return v1 +} + +; VCode: +; block0: +; ldrsh r0, [r0, #2] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldrsh r0, [r0, #2] +; bx lr + From 41b2375bec7f3015e21d1a789e69b9a1d20f38a9 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Fri, 3 Jul 2026 03:44:56 +0300 Subject: [PATCH 03/18] arm32: add the integer ALU instructions Rounds out the data-processing group: the logical ops (and/orr/eor/bic and register mvn), the shifts and rotates in both immediate and register forms, mul/mla/umull/smull, the flag-setting arithmetic (adds/subs/adcs/ sbcs, plus the adc/sbc opcodes), and the tst/teq tests. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/inst.isle | 79 +++++++++- cranelift/codegen/src/isa/arm32/inst/args.rs | 31 +++- cranelift/codegen/src/isa/arm32/inst/emit.rs | 90 ++++++++++++ .../codegen/src/isa/arm32/inst/emit_tests.rs | 137 ++++++++++++++++++ cranelift/codegen/src/isa/arm32/inst/mod.rs | 109 +++++++++++++- cranelift/codegen/src/isa/arm32/lower.isle | 55 +++++++ cranelift/codegen/src/isa/arm32/lower/isle.rs | 21 ++- .../filetests/isa/arm32/logical.clif | 85 +++++++++++ .../filetests/filetests/isa/arm32/mul.clif | 19 +++ .../filetests/filetests/isa/arm32/shifts.clif | 91 ++++++++++++ 10 files changed, 709 insertions(+), 8 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/arm32/logical.clif create mode 100644 cranelift/filetests/filetests/isa/arm32/mul.clif create mode 100644 cranelift/filetests/filetests/isa/arm32/shifts.clif diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index 03f99935ef23..0543ee1bdae5 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -11,16 +11,28 @@ (Add) (Sub) (Rsb) + (Adc) + (Sbc) (And) (Orr) (Eor) (Bic))) -;; A flag-setting compare operation. +;; A flag-setting compare/test operation (destination is discarded). (type CmpOp (enum (Cmp) - (Cmn))) + (Cmn) + (Tst) + (Teq))) + +;; A shift/rotate operation. +(type ShiftOp + (enum + (Lsl) + (Lsr) + (Asr) + (Ror))) ;; An ARM condition code (the "always" and "never" codes are omitted). (type Cond @@ -81,15 +93,33 @@ (Movt (rd WritableReg) (imm16 u32)) ;; `mov rd, rm`. (MovReg (rd WritableReg) (rm Reg)) + ;; `mvn rd, rm` — bitwise-NOT of a register. + (MvnReg (rd WritableReg) (rm Reg)) ;; `op rd, rn, rm`. (AluRRR (op ALUOp) (rd WritableReg) (rn Reg) (rm Reg)) ;; `op rd, rn, #imm` (pre-encoded rotated imm12). (AluRRImm (op ALUOp) (rd WritableReg) (rn Reg) (imm12 u32)) - - ;; `cmp/cmn rn, rm`. + ;; `op{s} rd, rn, rm` — flag-setting form (adds/subs/adcs/sbcs). + (AluRRRFlags (op ALUOp) (rd WritableReg) (rn Reg) (rm Reg)) + + ;; A shift/rotate by a constant amount (`op rd, rm, #amount`). + (ShiftImm (op ShiftOp) (rd WritableReg) (rm Reg) (amount u8)) + ;; A shift/rotate by a register amount (`op rd, rm, rs`). + (ShiftReg (op ShiftOp) (rd WritableReg) (rm Reg) (rs Reg)) + + ;; `mul rd, rn, rm` — `rd = rn * rm` (low 32 bits). + (Mul (rd WritableReg) (rn Reg) (rm Reg)) + ;; `mla rd, rn, rm, ra` — `rd = rn * rm + ra` (low 32 bits). + (Mla (rd WritableReg) (rn Reg) (rm Reg) (ra Reg)) + ;; `umull rd_lo, rd_hi, rn, rm` — unsigned 32x32->64 multiply. + (Umull (rd_lo WritableReg) (rd_hi WritableReg) (rn Reg) (rm Reg)) + ;; `smull rd_lo, rd_hi, rn, rm` — signed 32x32->64 multiply. + (Smull (rd_lo WritableReg) (rd_hi WritableReg) (rn Reg) (rm Reg)) + + ;; `cmp/cmn/tst/teq rn, rm`. (CmpRR (op CmpOp) (rn Reg) (rm Reg)) - ;; `cmp/cmn rn, #imm` (pre-encoded rotated imm12). + ;; `cmp/cmn/tst/teq rn, #imm` (pre-encoded rotated imm12). (CmpRImm (op CmpOp) (rn Reg) (imm12 u32)) ;; A load of the given kind. @@ -181,6 +211,45 @@ (_ Unit (emit (MInst.AluRRImm (ALUOp.Rsb) rd a imm12)))) rd)) +;; A generic ALU op that prefers the immediate form when the RHS is an +;; encodable constant and otherwise materializes it into a register. +(decl alu_op_imm (ALUOp Reg u64) Reg) +(rule 1 (alu_op_imm op a b) + (if-let imm12 (u64_from_rotated_imm12 b)) + (alu_rr_imm12 op a imm12)) +(rule (alu_op_imm op a b) (alu_rrr op a (gen_constant b))) + +;;;; Multiplies ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl mul (Reg Reg) Reg) +(rule (mul a b) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.Mul rd a b)))) + rd)) + +;; Bitwise-NOT of a register (`mvn rd, rm`). +(decl mvn_reg (Reg) Reg) +(rule (mvn_reg rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.MvnReg rd rm)))) + rd)) + +;;;; Shifts ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Shift/rotate by a constant amount. Handles the Cranelift "modulo width" +;; semantics (and the shift-by-zero special case) in Rust. +(decl gen_shift_imm (ShiftOp Reg u64) Reg) +(extern constructor gen_shift_imm gen_shift_imm) + +;; Shift/rotate by a register amount. The amount is masked to 0..31 to match +;; Cranelift's modulo-width semantics. +(decl shift_reg (ShiftOp Reg Value) Reg) +(rule (shift_reg op rm amount) + (let ((masked Reg (alu_rr_imm12 (ALUOp.And) (put_in_reg amount) 31)) + (rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.ShiftReg op rd rm masked)))) + rd)) + ;;;; Loads and stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl arm_load (AMode LoadKind) Reg) diff --git a/cranelift/codegen/src/isa/arm32/inst/args.rs b/cranelift/codegen/src/isa/arm32/inst/args.rs index beb0ee208040..9452be0ced40 100644 --- a/cranelift/codegen/src/isa/arm32/inst/args.rs +++ b/cranelift/codegen/src/isa/arm32/inst/args.rs @@ -4,7 +4,7 @@ use crate::isa::arm32::inst::*; use crate::machinst::{OperandVisitor, Reg}; pub use crate::isa::arm32::lower::isle::generated_code::{ - ALUOp, AMode, CmpOp, Cond, LoadKind, StoreKind, + ALUOp, AMode, CmpOp, Cond, LoadKind, ShiftOp, StoreKind, }; /// A memory address resolved to a concrete base register and either an @@ -96,6 +96,8 @@ impl ALUOp { ALUOp::Sub => 0b0010, ALUOp::Rsb => 0b0011, ALUOp::Add => 0b0100, + ALUOp::Adc => 0b0101, + ALUOp::Sbc => 0b0110, ALUOp::Orr => 0b1100, ALUOp::Bic => 0b1110, } @@ -108,6 +110,8 @@ impl ALUOp { ALUOp::Sub => "sub", ALUOp::Rsb => "rsb", ALUOp::Add => "add", + ALUOp::Adc => "adc", + ALUOp::Sbc => "sbc", ALUOp::Orr => "orr", ALUOp::Bic => "bic", } @@ -118,6 +122,8 @@ impl CmpOp { /// The 4-bit data-processing opcode (with the S bit set separately). pub(crate) fn opcode(self) -> u32 { match self { + CmpOp::Tst => 0b1000, + CmpOp::Teq => 0b1001, CmpOp::Cmp => 0b1010, CmpOp::Cmn => 0b1011, } @@ -125,12 +131,35 @@ impl CmpOp { pub(crate) fn name(self) -> &'static str { match self { + CmpOp::Tst => "tst", + CmpOp::Teq => "teq", CmpOp::Cmp => "cmp", CmpOp::Cmn => "cmn", } } } +impl ShiftOp { + /// The 2-bit shift-type field used in a shifted operand2. + pub(crate) fn bits(self) -> u32 { + match self { + ShiftOp::Lsl => 0b00, + ShiftOp::Lsr => 0b01, + ShiftOp::Asr => 0b10, + ShiftOp::Ror => 0b11, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + ShiftOp::Lsl => "lsl", + ShiftOp::Lsr => "lsr", + ShiftOp::Asr => "asr", + ShiftOp::Ror => "ror", + } + } +} + impl Cond { /// The 4-bit condition-code field. pub(crate) fn bits(self) -> u32 { diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index ebf8cab594c0..b0987b08796e 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -208,6 +208,32 @@ fn enc_pop(reg_list: u32) -> u32 { COND_AL | 0x08bd_0000 | (reg_list & 0xffff) } +/// A shift/rotate by a constant amount: `mov rd, rm, #amount`. +fn enc_shift_imm(op: ShiftOp, rd: u32, rm: u32, amount: u32) -> u32 { + COND_AL | 0x01a0_0000 | (rd << 12) | ((amount & 0x1f) << 7) | (op.bits() << 5) | rm +} + +/// A shift/rotate by a register amount: `mov rd, rm, rs`. +fn enc_shift_reg(op: ShiftOp, rd: u32, rm: u32, rs: u32) -> u32 { + COND_AL | 0x01a0_0000 | (rd << 12) | (rs << 8) | (op.bits() << 5) | (1 << 4) | rm +} + +/// `mul rd, rn, rm` — `rd = rn * rm`. +fn enc_mul(rd: u32, rn: u32, rm: u32) -> u32 { + COND_AL | (rd << 16) | (rm << 8) | 0x90 | rn +} + +/// `mla rd, rn, rm, ra` — `rd = rn * rm + ra`. +fn enc_mla(rd: u32, rn: u32, rm: u32, ra: u32) -> u32 { + COND_AL | 0x0020_0000 | (rd << 16) | (ra << 12) | (rm << 8) | 0x90 | rn +} + +/// A 32x32->64 long multiply (`umull`/`smull`). `signed` selects `smull`. +fn enc_mull(signed: bool, rd_lo: u32, rd_hi: u32, rn: u32, rm: u32) -> u32 { + let base = if signed { 0x00c0_0000 } else { 0x0080_0000 }; + COND_AL | base | (rd_hi << 16) | (rd_lo << 12) | (rm << 8) | 0x90 | rn +} + fn put_u32(sink: &mut MachBuffer, word: u32) { for b in word.to_le_bytes() { sink.put1(b); @@ -251,6 +277,11 @@ impl MachInstEmit for Inst { let rm = machreg_to_gpr(*rm); put_u32(sink, enc_dp_reg(0b1101, 0, rd, 0, rm)); } + Inst::MvnReg { rd, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_dp_reg(0b1111, 0, rd, 0, rm)); + } Inst::AluRRR { op, rd, rn, rm } => { let rd = machreg_to_gpr(rd.to_reg()); @@ -263,6 +294,65 @@ impl MachInstEmit for Inst { let rn = machreg_to_gpr(*rn); put_u32(sink, enc_dp_imm(op.opcode(), 0, rd, rn, *imm12)); } + Inst::AluRRRFlags { op, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_dp_reg(op.opcode(), 1, rd, rn, rm)); + } + Inst::ShiftImm { + op, + rd, + rm, + amount, + } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_shift_imm(*op, rd, rm, u32::from(*amount))); + } + Inst::ShiftReg { op, rd, rm, rs } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + let rs = machreg_to_gpr(*rs); + put_u32(sink, enc_shift_reg(*op, rd, rm, rs)); + } + Inst::Mul { rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_mul(rd, rn, rm)); + } + Inst::Mla { rd, rn, rm, ra } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + let ra = machreg_to_gpr(*ra); + put_u32(sink, enc_mla(rd, rn, rm, ra)); + } + Inst::Umull { + rd_lo, + rd_hi, + rn, + rm, + } => { + let rd_lo = machreg_to_gpr(rd_lo.to_reg()); + let rd_hi = machreg_to_gpr(rd_hi.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_mull(false, rd_lo, rd_hi, rn, rm)); + } + Inst::Smull { + rd_lo, + rd_hi, + rn, + rm, + } => { + let rd_lo = machreg_to_gpr(rd_lo.to_reg()); + let rd_hi = machreg_to_gpr(rd_hi.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_mull(true, rd_lo, rd_hi, rn, rm)); + } Inst::CmpRR { op, rn, rm } => { let rn = machreg_to_gpr(*rn); let rm = machreg_to_gpr(*rm); diff --git a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs index 672eac84e042..63551a4e2782 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs @@ -112,6 +112,143 @@ fn arithmetic() { ); } +#[test] +fn logical() { + let cases = [ + (ALUOp::And, 0xe001_0002u32), // and r0, r1, r2 + (ALUOp::Orr, 0xe181_0002), // orr r0, r1, r2 + (ALUOp::Eor, 0xe021_0002), // eor r0, r1, r2 + (ALUOp::Bic, 0xe1c1_0002), // bic r0, r1, r2 + ]; + for (op, want) in cases { + assert_eq!( + u32_le(Inst::AluRRR { + op, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + want, + "{op:?}" + ); + } + assert_eq!( + u32_le(Inst::MvnReg { + rd: writable_xreg(0), + rm: xreg(1), + }), + 0xe1e0_0001 // mvn r0, r1 + ); +} + +#[test] +fn flag_setting() { + assert_eq!( + u32_le(Inst::AluRRRFlags { + op: ALUOp::Add, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe091_0002 // adds r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::AluRRRFlags { + op: ALUOp::Sbc, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe0d1_0002 // sbcs r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::CmpRR { + op: CmpOp::Tst, + rn: xreg(0), + rm: xreg(1), + }), + 0xe110_0001 // tst r0, r1 + ); + assert_eq!( + u32_le(Inst::CmpRR { + op: CmpOp::Teq, + rn: xreg(0), + rm: xreg(1), + }), + 0xe130_0001 // teq r0, r1 + ); +} + +#[test] +fn shifts() { + assert_eq!( + u32_le(Inst::ShiftImm { + op: ShiftOp::Lsl, + rd: writable_xreg(0), + rm: xreg(1), + amount: 2, + }), + 0xe1a0_0101 // lsl r0, r1, #2 + ); + assert_eq!( + u32_le(Inst::ShiftImm { + op: ShiftOp::Asr, + rd: writable_xreg(0), + rm: xreg(1), + amount: 3, + }), + 0xe1a0_01c1 // asr r0, r1, #3 + ); + assert_eq!( + u32_le(Inst::ShiftReg { + op: ShiftOp::Lsl, + rd: writable_xreg(0), + rm: xreg(1), + rs: xreg(2), + }), + 0xe1a0_0211 // lsl r0, r1, r2 + ); +} + +#[test] +fn multiplies() { + assert_eq!( + u32_le(Inst::Mul { + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe000_0291 // mul r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::Mla { + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + ra: xreg(3), + }), + 0xe020_3291 // mla r0, r1, r2, r3 + ); + assert_eq!( + u32_le(Inst::Umull { + rd_lo: writable_xreg(0), + rd_hi: writable_xreg(1), + rn: xreg(2), + rm: xreg(3), + }), + 0xe081_0392 // umull r0, r1, r2, r3 + ); + assert_eq!( + u32_le(Inst::Smull { + rd_lo: writable_xreg(0), + rd_hi: writable_xreg(1), + rn: xreg(2), + rm: xreg(3), + }), + 0xe0c1_0392 // smull r0, r1, r2, r3 + ); +} + #[test] fn compares() { assert_eq!( diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index 07e89cff5ed4..1fb12cafc633 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -61,7 +61,7 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { | Inst::Movw { rd, .. } | Inst::Movt { rd, .. } => def_if_virtual(collector, rd), - Inst::MovReg { rd, rm } => { + Inst::MovReg { rd, rm } | Inst::MvnReg { rd, rm } => { use_if_virtual(collector, rm); def_if_virtual(collector, rd); } @@ -74,6 +74,54 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { use_if_virtual(collector, rn); def_if_virtual(collector, rd); } + Inst::AluRRRFlags { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::ShiftImm { rd, rm, .. } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::ShiftReg { rd, rm, rs, .. } => { + use_if_virtual(collector, rm); + use_if_virtual(collector, rs); + def_if_virtual(collector, rd); + } + Inst::Mul { rd, rn, rm } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::Mla { rd, rn, rm, ra } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + use_if_virtual(collector, ra); + def_if_virtual(collector, rd); + } + Inst::Umull { + rd_lo, + rd_hi, + rn, + rm, + } + | Inst::Smull { + rd_lo, + rd_hi, + rn, + rm, + } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + // The two destination registers must be distinct from each other + // and from the inputs, so use early defs. + if rd_lo.to_reg().to_real_reg().is_none() { + collector.reg_early_def(rd_lo); + } + if rd_hi.to_reg().to_real_reg().is_none() { + collector.reg_early_def(rd_hi); + } + } Inst::CmpRR { rn, rm, .. } => { use_if_virtual(collector, rn); use_if_virtual(collector, rm); @@ -275,6 +323,9 @@ impl Inst { Inst::MovReg { rd, rm } => { alloc::format!("mov {}, {}", r(rd.to_reg()), r(*rm)) } + Inst::MvnReg { rd, rm } => { + alloc::format!("mvn {}, {}", r(rd.to_reg()), r(*rm)) + } Inst::AluRRR { op, rd, rn, rm } => { alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) } @@ -287,6 +338,62 @@ impl Inst { decode_rotated_imm(*imm12) ) } + Inst::AluRRRFlags { op, rd, rn, rm } => { + alloc::format!("{}s {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::ShiftImm { + op, + rd, + rm, + amount, + } => { + alloc::format!("{} {}, {}, #{}", op.name(), r(rd.to_reg()), r(*rm), amount) + } + Inst::ShiftReg { op, rd, rm, rs } => { + alloc::format!( + "{} {}, {}, {}", + op.name(), + r(rd.to_reg()), + r(*rm), + r(*rs) + ) + } + Inst::Mul { rd, rn, rm } => { + alloc::format!("mul {}, {}, {}", r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::Mla { rd, rn, rm, ra } => { + alloc::format!( + "mla {}, {}, {}, {}", + r(rd.to_reg()), + r(*rn), + r(*rm), + r(*ra) + ) + } + Inst::Umull { + rd_lo, + rd_hi, + rn, + rm, + } => alloc::format!( + "umull {}, {}, {}, {}", + r(rd_lo.to_reg()), + r(rd_hi.to_reg()), + r(*rn), + r(*rm) + ), + Inst::Smull { + rd_lo, + rd_hi, + rn, + rm, + } => alloc::format!( + "smull {}, {}, {}, {}", + r(rd_lo.to_reg()), + r(rd_hi.to_reg()), + r(*rn), + r(*rm) + ), Inst::CmpRR { op, rn, rm } => { alloc::format!("{} {}, {}", op.name(), r(*rn), r(*rm)) } diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index cec745058041..76c975d4cb71 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -26,6 +26,61 @@ (rule (lower (ineg (fits_in_32 _) x)) (rsb_imm (put_in_reg x) 0)) +;;;; Logical `band` / `bor` / `bxor` / `band_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule 1 (lower (band (fits_in_32 _) x (iconst _ (u64_from_imm64 n)))) + (alu_op_imm (ALUOp.And) (put_in_reg x) n)) +(rule (lower (band (fits_in_32 _) x y)) + (alu_rrr (ALUOp.And) (put_in_reg x) (put_in_reg y))) + +(rule 1 (lower (bor (fits_in_32 _) x (iconst _ (u64_from_imm64 n)))) + (alu_op_imm (ALUOp.Orr) (put_in_reg x) n)) +(rule (lower (bor (fits_in_32 _) x y)) + (alu_rrr (ALUOp.Orr) (put_in_reg x) (put_in_reg y))) + +(rule 1 (lower (bxor (fits_in_32 _) x (iconst _ (u64_from_imm64 n)))) + (alu_op_imm (ALUOp.Eor) (put_in_reg x) n)) +(rule (lower (bxor (fits_in_32 _) x y)) + (alu_rrr (ALUOp.Eor) (put_in_reg x) (put_in_reg y))) + +;; `x & ~y` maps directly to `bic` (Cranelift's `band_not` builder emits +;; `band` of `bnot`). +(rule 2 (lower (band (fits_in_32 _) x (bnot _ y))) + (alu_rrr (ALUOp.Bic) (put_in_reg x) (put_in_reg y))) +(rule 3 (lower (band (fits_in_32 _) (bnot _ y) x)) + (alu_rrr (ALUOp.Bic) (put_in_reg x) (put_in_reg y))) + +;; Bitwise NOT. +(rule (lower (bnot (fits_in_32 _) x)) + (mvn_reg (put_in_reg x))) + +;;;; `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (imul (fits_in_32 _) x y)) + (mul (put_in_reg x) (put_in_reg y))) + +;;;; Shifts: `ishl` / `ushr` / `sshr` / `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule 1 (lower (ishl $I32 x (iconst _ (u64_from_imm64 n)))) + (gen_shift_imm (ShiftOp.Lsl) (put_in_reg x) n)) +(rule (lower (ishl $I32 x y)) + (shift_reg (ShiftOp.Lsl) (put_in_reg x) y)) + +(rule 1 (lower (ushr $I32 x (iconst _ (u64_from_imm64 n)))) + (gen_shift_imm (ShiftOp.Lsr) (put_in_reg x) n)) +(rule (lower (ushr $I32 x y)) + (shift_reg (ShiftOp.Lsr) (put_in_reg x) y)) + +(rule 1 (lower (sshr $I32 x (iconst _ (u64_from_imm64 n)))) + (gen_shift_imm (ShiftOp.Asr) (put_in_reg x) n)) +(rule (lower (sshr $I32 x y)) + (shift_reg (ShiftOp.Asr) (put_in_reg x) y)) + +(rule 1 (lower (rotr $I32 x (iconst _ (u64_from_imm64 n)))) + (gen_shift_imm (ShiftOp.Ror) (put_in_reg x) n)) +(rule (lower (rotr $I32 x y)) + (shift_reg (ShiftOp.Ror) (put_in_reg x) y)) + ;;;; Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (load (fits_in_32 _) flags addr offset)) diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs index 406120ea043c..b4258f7ab913 100644 --- a/cranelift/codegen/src/isa/arm32/lower/isle.rs +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -12,7 +12,7 @@ use crate::ir::{ BlockCall, ExternalName, Inst, InstructionData, MemFlags, Opcode, TrapCode, Value, ValueList, }; use crate::isa::arm32::Arm32Backend; -use crate::isa::arm32::inst::{Cond, encode_rotated_imm}; +use crate::isa::arm32::inst::{Cond, ShiftOp, encode_rotated_imm}; use crate::machinst::isle::*; use crate::machinst::{ ArgPair, CallArgList, CallInfo, CallRetList, InstOutput, Lower, MachInst, MachLabel, RetPair, @@ -82,6 +82,25 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { encode_rotated_imm(val as u32) } + /// Shift/rotate by a constant, applying Cranelift's modulo-width semantics. + /// A masked amount of zero becomes a plain register move. + fn gen_shift_imm(&mut self, op: &ShiftOp, rm: Reg, amount: u64) -> Reg { + let amount = (amount & 31) as u8; + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + let inst = if amount == 0 { + MInst::MovReg { rd, rm } + } else { + MInst::ShiftImm { + op: *op, + rd, + rm, + amount, + } + }; + self.lower_ctx.emit(inst); + rd.to_reg() + } + fn cond_from_intcc(&mut self, cc: &IntCC) -> Cond { match cc { IntCC::Equal => Cond::Eq, diff --git a/cranelift/filetests/filetests/isa/arm32/logical.clif b/cranelift/filetests/filetests/isa/arm32/logical.clif new file mode 100644 index 000000000000..9f7c9a121b83 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/logical.clif @@ -0,0 +1,85 @@ +test compile precise-output +target arm + +function %and(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = band v0, v1 + return v2 +} + +; VCode: +; block0: +; and r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; and r0, r0, r1 +; bx lr + +function %or_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 0xff + v2 = bor v0, v1 + return v2 +} + +; VCode: +; block0: +; orr r0, r0, #255 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; orr r0, r0, #0xff +; bx lr + +function %xor(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = bxor v0, v1 + return v2 +} + +; VCode: +; block0: +; eor r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; eor r0, r0, r1 +; bx lr + +function %not(i32) -> i32 { +block0(v0: i32): + v1 = bnot v0 + return v1 +} + +; VCode: +; block0: +; mvn r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mvn r0, r0 +; bx lr + +function %and_not(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = bnot v1 + v3 = band v0, v2 + return v3 +} + +; VCode: +; block0: +; bic r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; bic r0, r0, r1 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/mul.clif b/cranelift/filetests/filetests/isa/arm32/mul.clif new file mode 100644 index 000000000000..d0677032bbca --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/mul.clif @@ -0,0 +1,19 @@ +test compile precise-output +target arm + +function %mul(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = imul v0, v1 + return v2 +} + +; VCode: +; block0: +; mul r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mul r0, r0, r1 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/shifts.clif b/cranelift/filetests/filetests/isa/arm32/shifts.clif new file mode 100644 index 000000000000..4c5ad3aaab86 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/shifts.clif @@ -0,0 +1,91 @@ +test compile precise-output +target arm + +function %shl_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 3 + v2 = ishl v0, v1 + return v2 +} + +; VCode: +; block0: +; lsl r0, r0, #3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; lsl r0, r0, #3 +; bx lr + +function %shl_reg(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = ishl v0, v1 + return v2 +} + +; VCode: +; block0: +; and r3, r1, #31 +; lsl r0, r0, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; and r3, r1, #0x1f +; lsl r0, r0, r3 +; bx lr + +function %ushr(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = ushr v0, v1 + return v2 +} + +; VCode: +; block0: +; and r3, r1, #31 +; lsr r0, r0, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; and r3, r1, #0x1f +; lsr r0, r0, r3 +; bx lr + +function %sshr_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 5 + v2 = sshr v0, v1 + return v2 +} + +; VCode: +; block0: +; asr r0, r0, #5 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; asr r0, r0, #5 +; bx lr + +function %rotr(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = rotr v0, v1 + return v2 +} + +; VCode: +; block0: +; and r3, r1, #31 +; ror r0, r0, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; and r3, r1, #0x1f +; ror r0, r0, r3 +; bx lr + From 3725a77aa21d6bfff3dafc6779aebe2a700ded15 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Fri, 3 Jul 2026 06:47:21 +0300 Subject: [PATCH 04/18] arm32: add division, clz, extends, and bit reversal The remaining core integer operations: clz; sdiv/udiv behind a new has_idiv ISA setting; the sxtb/sxth/uxtb/uxth extends; rev/rev16/rbit; the mls/smlal/umlal fused multiplies; and a conditional-select pseudo built from a mov plus a mov. Signed-off-by: Obei Sideg --- cranelift/codegen/meta/src/isa/arm32.rs | 9 ++ cranelift/codegen/src/isa/arm32/inst.isle | 84 ++++++++++++++ cranelift/codegen/src/isa/arm32/inst/args.rs | 45 +++++++- cranelift/codegen/src/isa/arm32/inst/emit.rs | 88 +++++++++++++++ .../codegen/src/isa/arm32/inst/emit_tests.rs | 105 ++++++++++++++++++ cranelift/codegen/src/isa/arm32/inst/mod.rs | 105 +++++++++++++++--- cranelift/codegen/src/isa/arm32/lower.isle | 55 ++++++++- cranelift/codegen/src/isa/arm32/lower/isle.rs | 6 +- .../filetests/filetests/isa/arm32/bitops.clif | 51 +++++++++ .../filetests/filetests/isa/arm32/div.clif | 35 ++++++ .../filetests/filetests/isa/arm32/extend.clif | 81 ++++++++++++++ .../filetests/filetests/isa/arm32/mul.clif | 17 +++ .../filetests/filetests/isa/arm32/select.clif | 42 +++++++ 13 files changed, 703 insertions(+), 20 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/arm32/bitops.clif create mode 100644 cranelift/filetests/filetests/isa/arm32/div.clif create mode 100644 cranelift/filetests/filetests/isa/arm32/extend.clif create mode 100644 cranelift/filetests/filetests/isa/arm32/select.clif diff --git a/cranelift/codegen/meta/src/isa/arm32.rs b/cranelift/codegen/meta/src/isa/arm32.rs index a1f0314c0bff..25aff482bb9b 100644 --- a/cranelift/codegen/meta/src/isa/arm32.rs +++ b/cranelift/codegen/meta/src/isa/arm32.rs @@ -12,5 +12,14 @@ pub(crate) fn define() -> TargetIsa { false, ); + settings.add_bool( + "has_idiv", + "Has hardware integer divide (the `sdiv`/`udiv` instructions, present \ + on ARMv7-R/M and ARMv7VE); when disabled, division must go through a \ + runtime library call.", + "", + false, + ); + TargetIsa::new("arm32", settings.build()) } diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index 0543ee1bdae5..e4c29aa18640 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -34,6 +34,22 @@ (Asr) (Ror))) +;; A single-operand bit-manipulation operation. +(type BitOp + (enum + (Clz) + (Rev) + (Rev16) + (Rbit))) + +;; A sign/zero extension operation. +(type ExtOp + (enum + (Sxtb) + (Sxth) + (Uxtb) + (Uxth))) + ;; An ARM condition code (the "always" and "never" codes are omitted). (type Cond (enum @@ -112,10 +128,29 @@ (Mul (rd WritableReg) (rn Reg) (rm Reg)) ;; `mla rd, rn, rm, ra` — `rd = rn * rm + ra` (low 32 bits). (Mla (rd WritableReg) (rn Reg) (rm Reg) (ra Reg)) + ;; `mls rd, rn, rm, ra` — `rd = ra - rn * rm` (low 32 bits). + (Mls (rd WritableReg) (rn Reg) (rm Reg) (ra Reg)) ;; `umull rd_lo, rd_hi, rn, rm` — unsigned 32x32->64 multiply. (Umull (rd_lo WritableReg) (rd_hi WritableReg) (rn Reg) (rm Reg)) ;; `smull rd_lo, rd_hi, rn, rm` — signed 32x32->64 multiply. (Smull (rd_lo WritableReg) (rd_hi WritableReg) (rn Reg) (rm Reg)) + ;; `umlal rd_lo, rd_hi, rn, rm` — unsigned multiply-accumulate into 64 bits. + (Umlal (rd_lo WritableReg) (rd_hi WritableReg) (rn Reg) (rm Reg)) + ;; `smlal rd_lo, rd_hi, rn, rm` — signed multiply-accumulate into 64 bits. + (Smlal (rd_lo WritableReg) (rd_hi WritableReg) (rn Reg) (rm Reg)) + + ;; `sdiv rd, rn, rm` — signed divide `rd = rn / rm`. + (SDiv (rd WritableReg) (rn Reg) (rm Reg)) + ;; `udiv rd, rn, rm` — unsigned divide. + (UDiv (rd WritableReg) (rn Reg) (rm Reg)) + + ;; A single-operand bit operation (`clz`/`rev`/`rev16`/`rbit`). + (BitRR (op BitOp) (rd WritableReg) (rm Reg)) + ;; A sign/zero extension (`sxtb`/`sxth`/`uxtb`/`uxth`). + (ExtRR (op ExtOp) (rd WritableReg) (rm Reg)) + + ;; A conditional select, emitted as `mov rd, rm; mov rd, rn`. + (CSel (cond Cond) (rd WritableReg) (rn Reg) (rm Reg)) ;; `cmp/cmn/tst/teq rn, rm`. (CmpRR (op CmpOp) (rn Reg) (rm Reg)) @@ -234,6 +269,55 @@ (_ Unit (emit (MInst.MvnReg rd rm)))) rd)) +;; `mls rd, rn, rm, ra` — `rd = ra - rn * rm`. +(decl mls (Reg Reg Reg) Reg) +(rule (mls rn rm ra) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.Mls rd rn rm ra)))) + rd)) + +;;;; Divides ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; True when the hardware integer-divide instructions are available. +(decl pure use_idiv () bool) +(extern constructor use_idiv use_idiv) + +(decl sdiv_reg (Reg Reg) Reg) +(rule (sdiv_reg rn rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.SDiv rd rn rm)))) + rd)) + +(decl udiv_reg (Reg Reg) Reg) +(rule (udiv_reg rn rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.UDiv rd rn rm)))) + rd)) + +;;;; Bit operations and extends ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl bit_rr (BitOp Reg) Reg) +(rule (bit_rr op rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.BitRR op rd rm)))) + rd)) + +(decl ext_rr (ExtOp Reg) Reg) +(rule (ext_rr op rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.ExtRR op rd rm)))) + rd)) + +;;;; Conditional select ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Select `if_true` when `cond` holds (given the current flags), else +;; `if_false`. Emitted as `mov rd, if_false; mov rd, if_true`. +(decl csel (Cond Reg Reg) Reg) +(rule (csel cond if_true if_false) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.CSel cond rd if_true if_false)))) + rd)) + ;;;; Shifts ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Shift/rotate by a constant amount. Handles the Cranelift "modulo width" diff --git a/cranelift/codegen/src/isa/arm32/inst/args.rs b/cranelift/codegen/src/isa/arm32/inst/args.rs index 9452be0ced40..03946741983a 100644 --- a/cranelift/codegen/src/isa/arm32/inst/args.rs +++ b/cranelift/codegen/src/isa/arm32/inst/args.rs @@ -4,7 +4,7 @@ use crate::isa::arm32::inst::*; use crate::machinst::{OperandVisitor, Reg}; pub use crate::isa::arm32::lower::isle::generated_code::{ - ALUOp, AMode, CmpOp, Cond, LoadKind, ShiftOp, StoreKind, + ALUOp, AMode, BitOp, CmpOp, Cond, ExtOp, LoadKind, ShiftOp, StoreKind, }; /// A memory address resolved to a concrete base register and either an @@ -139,6 +139,49 @@ impl CmpOp { } } +impl BitOp { + /// The full instruction word for `op rd, rm`, with `rd`/`rm` zeroed. + pub(crate) fn template(self) -> u32 { + match self { + BitOp::Clz => 0x016f_0f10, + BitOp::Rev => 0x06bf_0f30, + BitOp::Rev16 => 0x06bf_0fb0, + BitOp::Rbit => 0x06ff_0f30, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + BitOp::Clz => "clz", + BitOp::Rev => "rev", + BitOp::Rev16 => "rev16", + BitOp::Rbit => "rbit", + } + } +} + +impl ExtOp { + /// The full instruction word for `op rd, rm` (rotation 0), with `rd`/`rm` + /// zeroed. + pub(crate) fn template(self) -> u32 { + match self { + ExtOp::Sxtb => 0x06af_0070, + ExtOp::Sxth => 0x06bf_0070, + ExtOp::Uxtb => 0x06ef_0070, + ExtOp::Uxth => 0x06ff_0070, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + ExtOp::Sxtb => "sxtb", + ExtOp::Sxth => "sxth", + ExtOp::Uxtb => "uxtb", + ExtOp::Uxth => "uxth", + } + } +} + impl ShiftOp { /// The 2-bit shift-type field used in a shifted operand2. pub(crate) fn bits(self) -> u32 { diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index b0987b08796e..e59b151f0fed 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -234,6 +234,33 @@ fn enc_mull(signed: bool, rd_lo: u32, rd_hi: u32, rn: u32, rm: u32) -> u32 { COND_AL | base | (rd_hi << 16) | (rd_lo << 12) | (rm << 8) | 0x90 | rn } +/// A 32x32->64 multiply-accumulate (`umlal`/`smlal`). `signed` selects `smlal`. +fn enc_mlal(signed: bool, rd_lo: u32, rd_hi: u32, rn: u32, rm: u32) -> u32 { + let base = if signed { 0x00e0_0000 } else { 0x00a0_0000 }; + COND_AL | base | (rd_hi << 16) | (rd_lo << 12) | (rm << 8) | 0x90 | rn +} + +/// `mls rd, rn, rm, ra` — `rd = ra - rn * rm`. +fn enc_mls(rd: u32, rn: u32, rm: u32, ra: u32) -> u32 { + COND_AL | 0x0060_0090 | (rd << 16) | (ra << 12) | (rm << 8) | rn +} + +/// `sdiv`/`udiv rd, rn, rm`. `signed` selects `sdiv`. +fn enc_div(signed: bool, rd: u32, rn: u32, rm: u32) -> u32 { + let base = if signed { 0x0710_f010 } else { 0x0730_f010 }; + COND_AL | base | (rd << 16) | (rm << 8) | rn +} + +/// A single-operand `op rd, rm` built from an opcode template. +fn enc_op_rd_rm(template: u32, rd: u32, rm: u32) -> u32 { + COND_AL | template | (rd << 12) | rm +} + +/// A conditional register move: `mov rd, rm`. +fn enc_mov_cond(cond: Cond, rd: u32, rm: u32) -> u32 { + (cond.bits() << 28) | 0x01a0_0000 | (rd << 12) | rm +} + fn put_u32(sink: &mut MachBuffer, word: u32) { for b in word.to_le_bytes() { sink.put1(b); @@ -353,6 +380,67 @@ impl MachInstEmit for Inst { let rm = machreg_to_gpr(*rm); put_u32(sink, enc_mull(true, rd_lo, rd_hi, rn, rm)); } + Inst::Umlal { + rd_lo, + rd_hi, + rn, + rm, + } => { + let rd_lo = machreg_to_gpr(rd_lo.to_reg()); + let rd_hi = machreg_to_gpr(rd_hi.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_mlal(false, rd_lo, rd_hi, rn, rm)); + } + Inst::Smlal { + rd_lo, + rd_hi, + rn, + rm, + } => { + let rd_lo = machreg_to_gpr(rd_lo.to_reg()); + let rd_hi = machreg_to_gpr(rd_hi.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_mlal(true, rd_lo, rd_hi, rn, rm)); + } + Inst::Mls { rd, rn, rm, ra } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + let ra = machreg_to_gpr(*ra); + put_u32(sink, enc_mls(rd, rn, rm, ra)); + } + Inst::SDiv { rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_div(true, rd, rn, rm)); + } + Inst::UDiv { rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_div(false, rd, rn, rm)); + } + Inst::BitRR { op, rd, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_op_rd_rm(op.template(), rd, rm)); + } + Inst::ExtRR { op, rd, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_op_rd_rm(op.template(), rd, rm)); + } + Inst::CSel { cond, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + // Default to the "false" value, then conditionally overwrite. + put_u32(sink, enc_dp_reg(0b1101, 0, rd, 0, rm)); + put_u32(sink, enc_mov_cond(*cond, rd, rn)); + } Inst::CmpRR { op, rn, rm } => { let rn = machreg_to_gpr(*rn); let rm = machreg_to_gpr(*rm); diff --git a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs index 63551a4e2782..dd73ab46aedf 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs @@ -249,6 +249,111 @@ fn multiplies() { ); } +#[test] +fn bit_ops() { + let bit_cases = [ + (BitOp::Clz, 0xe16f_0f11u32), + (BitOp::Rev, 0xe6bf_0f31), + (BitOp::Rev16, 0xe6bf_0fb1), + (BitOp::Rbit, 0xe6ff_0f31), + ]; + for (op, want) in bit_cases { + assert_eq!( + u32_le(Inst::BitRR { + op, + rd: writable_xreg(0), + rm: xreg(1), + }), + want, + "{op:?}" + ); + } + let ext_cases = [ + (ExtOp::Sxtb, 0xe6af_0071u32), + (ExtOp::Sxth, 0xe6bf_0071), + (ExtOp::Uxtb, 0xe6ef_0071), + (ExtOp::Uxth, 0xe6ff_0071), + ]; + for (op, want) in ext_cases { + assert_eq!( + u32_le(Inst::ExtRR { + op, + rd: writable_xreg(0), + rm: xreg(1), + }), + want, + "{op:?}" + ); + } +} + +#[test] +fn divides() { + assert_eq!( + u32_le(Inst::SDiv { + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe710_f211 // sdiv r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::UDiv { + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe730_f211 // udiv r0, r1, r2 + ); +} + +#[test] +fn fused_multiply() { + assert_eq!( + u32_le(Inst::Mls { + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + ra: xreg(3), + }), + 0xe060_3291 // mls r0, r1, r2, r3 + ); + assert_eq!( + u32_le(Inst::Umlal { + rd_lo: writable_xreg(0), + rd_hi: writable_xreg(1), + rn: xreg(2), + rm: xreg(3), + }), + 0xe0a1_0392 // umlal r0, r1, r2, r3 + ); + assert_eq!( + u32_le(Inst::Smlal { + rd_lo: writable_xreg(0), + rd_hi: writable_xreg(1), + rn: xreg(2), + rm: xreg(3), + }), + 0xe0e1_0392 // smlal r0, r1, r2, r3 + ); +} + +#[test] +fn conditional_select() { + // csel ne r0, r1, r2 => mov r0, r2 ; movne r0, r1 + let bytes = encode(Inst::CSel { + cond: Cond::Ne, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }); + assert_eq!(bytes.len(), 8); + let w0 = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + let w1 = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]); + assert_eq!(w0, 0xe1a0_0002); // mov r0, r2 + assert_eq!(w1, 0x11a0_0001); // movne r0, r1 +} + #[test] fn compares() { assert_eq!( diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index 1fb12cafc633..bb715925f6d9 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -93,12 +93,30 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { use_if_virtual(collector, rm); def_if_virtual(collector, rd); } - Inst::Mla { rd, rn, rm, ra } => { + Inst::Mla { rd, rn, rm, ra } | Inst::Mls { rd, rn, rm, ra } => { use_if_virtual(collector, rn); use_if_virtual(collector, rm); use_if_virtual(collector, ra); def_if_virtual(collector, rd); } + Inst::SDiv { rd, rn, rm } | Inst::UDiv { rd, rn, rm } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::BitRR { rd, rm, .. } | Inst::ExtRR { rd, rm, .. } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::CSel { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + // `rd` is written before the inputs are all consumed (the default + // move), so keep it distinct from the operands. + if rd.to_reg().to_real_reg().is_none() { + collector.reg_early_def(rd); + } + } Inst::Umull { rd_lo, rd_hi, @@ -110,11 +128,25 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { rd_hi, rn, rm, + } + | Inst::Umlal { + rd_lo, + rd_hi, + rn, + rm, + } + | Inst::Smlal { + rd_lo, + rd_hi, + rn, + rm, } => { use_if_virtual(collector, rn); use_if_virtual(collector, rm); // The two destination registers must be distinct from each other - // and from the inputs, so use early defs. + // and from the inputs, so use early defs. (For the accumulating + // `umlal`/`smlal` the destinations are also inputs; those are only + // constructed once i64 support wires them up.) if rd_lo.to_reg().to_real_reg().is_none() { collector.reg_early_def(rd_lo); } @@ -370,30 +402,69 @@ impl Inst { r(*ra) ) } + Inst::Mls { rd, rn, rm, ra } => { + alloc::format!( + "mls {}, {}, {}, {}", + r(rd.to_reg()), + r(*rn), + r(*rm), + r(*ra) + ) + } Inst::Umull { rd_lo, rd_hi, rn, rm, - } => alloc::format!( - "umull {}, {}, {}, {}", - r(rd_lo.to_reg()), - r(rd_hi.to_reg()), - r(*rn), - r(*rm) - ), - Inst::Smull { + } + | Inst::Smull { rd_lo, rd_hi, rn, rm, - } => alloc::format!( - "smull {}, {}, {}, {}", - r(rd_lo.to_reg()), - r(rd_hi.to_reg()), - r(*rn), - r(*rm) - ), + } + | Inst::Umlal { + rd_lo, + rd_hi, + rn, + rm, + } + | Inst::Smlal { + rd_lo, + rd_hi, + rn, + rm, + } => { + let mnem = match self { + Inst::Umull { .. } => "umull", + Inst::Smull { .. } => "smull", + Inst::Umlal { .. } => "umlal", + _ => "smlal", + }; + alloc::format!( + "{mnem} {}, {}, {}, {}", + r(rd_lo.to_reg()), + r(rd_hi.to_reg()), + r(*rn), + r(*rm) + ) + } + Inst::SDiv { rd, rn, rm } => { + alloc::format!("sdiv {}, {}, {}", r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::UDiv { rd, rn, rm } => { + alloc::format!("udiv {}, {}, {}", r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::BitRR { op, rd, rm } => { + alloc::format!("{} {}, {}", op.name(), r(rd.to_reg()), r(*rm)) + } + Inst::ExtRR { op, rd, rm } => { + alloc::format!("{} {}, {}", op.name(), r(rd.to_reg()), r(*rm)) + } + Inst::CSel { cond, rd, rn, rm } => { + let rd = r(rd.to_reg()); + alloc::format!("mov {rd}, {}; mov{} {rd}, {}", r(*rm), cond.name(), r(*rn)) + } Inst::CmpRR { op, rn, rm } => { alloc::format!("{} {}, {}", op.name(), r(*rn), r(*rm)) } diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index 76c975d4cb71..426e1e9715a9 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -54,11 +54,64 @@ (rule (lower (bnot (fits_in_32 _) x)) (mvn_reg (put_in_reg x))) -;;;; `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; `imul` and multiply-fused `isub` -> `mls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (imul (fits_in_32 _) x y)) (mul (put_in_reg x) (put_in_reg y))) +;; `a - x*y` fuses into a single `mls`. +(rule 2 (lower (isub (fits_in_32 _) a (imul _ x y))) + (mls (put_in_reg x) (put_in_reg y) (put_in_reg a))) + +;;;; Divides (only when hardware `sdiv`/`udiv` is available) ;;;;;;;;;;;;;;;;;;; +;; NOTE: trap-on-zero / INT_MIN overflow checks are not yet emitted. + +(rule (lower (sdiv $I32 x y)) + (if-let true (use_idiv)) + (sdiv_reg (put_in_reg x) (put_in_reg y))) +(rule (lower (udiv $I32 x y)) + (if-let true (use_idiv)) + (udiv_reg (put_in_reg x) (put_in_reg y))) + +;;;; Counting / byte-swap / bit-reverse ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (clz $I32 x)) + (bit_rr (BitOp.Clz) (put_in_reg x))) +(rule (lower (bswap $I32 x)) + (bit_rr (BitOp.Rev) (put_in_reg x))) +(rule (lower (bswap $I16 x)) + (bit_rr (BitOp.Rev16) (put_in_reg x))) +(rule (lower (bitrev $I32 x)) + (bit_rr (BitOp.Rbit) (put_in_reg x))) + +;;;; Integer extends / reduces ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (sextend $I32 x @ (value_type $I8))) + (ext_rr (ExtOp.Sxtb) (put_in_reg x))) +(rule (lower (sextend $I32 x @ (value_type $I16))) + (ext_rr (ExtOp.Sxth) (put_in_reg x))) +(rule (lower (uextend $I32 x @ (value_type $I8))) + (ext_rr (ExtOp.Uxtb) (put_in_reg x))) +(rule (lower (uextend $I32 x @ (value_type $I16))) + (ext_rr (ExtOp.Uxth) (put_in_reg x))) + +;; Reducing to a narrower integer is free: the value already lives in a +;; 32-bit register and consumers read only the low bits. +(rule (lower (ireduce (fits_in_32 _) x)) + (put_in_reg x)) + +;;;; `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Fuse an `icmp` condition into the compare that precedes the select. +(rule 1 (lower (select (fits_in_32 _) (icmp _ cc x y) a b)) + (let ((_ Unit (emit_side_effect (cmp_rr (put_in_reg x) (put_in_reg y))))) + (csel (cond_from_intcc cc) (put_in_reg a) (put_in_reg b)))) + +;; Generic select: branch on whether the condition value is non-zero. +(rule (lower (select (fits_in_32 _) c a b)) + (let ((_ Unit (emit_side_effect (cmp_imm (put_in_reg c) 0)))) + (csel (Cond.Ne) (put_in_reg a) (put_in_reg b)))) + ;;;; Shifts: `ishl` / `ushr` / `sshr` / `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule 1 (lower (ishl $I32 x (iconst _ (u64_from_imm64 n)))) diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs index b4258f7ab913..e8efb0b87aff 100644 --- a/cranelift/codegen/src/isa/arm32/lower/isle.rs +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -33,7 +33,6 @@ where B: LowerBackend, { pub lower_ctx: &'a mut Lower<'b, I>, - #[allow(dead_code, reason = "kept for symmetry with other backends")] pub backend: &'a B, } @@ -101,6 +100,11 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { rd.to_reg() } + /// Whether the hardware integer-divide instructions may be used. + fn use_idiv(&mut self) -> bool { + self.backend.isa_flags.has_idiv() + } + fn cond_from_intcc(&mut self, cc: &IntCC) -> Cond { match cc { IntCC::Equal => Cond::Eq, diff --git a/cranelift/filetests/filetests/isa/arm32/bitops.clif b/cranelift/filetests/filetests/isa/arm32/bitops.clif new file mode 100644 index 000000000000..bcac6de99d14 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/bitops.clif @@ -0,0 +1,51 @@ +test compile precise-output +target arm + +function %clz(i32) -> i32 { +block0(v0: i32): + v1 = clz v0 + return v1 +} + +; VCode: +; block0: +; clz r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; clz r0, r0 +; bx lr + +function %bswap(i32) -> i32 { +block0(v0: i32): + v1 = bswap v0 + return v1 +} + +; VCode: +; block0: +; rev r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; rev r0, r0 +; bx lr + +function %bitrev(i32) -> i32 { +block0(v0: i32): + v1 = bitrev v0 + return v1 +} + +; VCode: +; block0: +; rbit r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; rbit r0, r0 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/div.clif b/cranelift/filetests/filetests/isa/arm32/div.clif new file mode 100644 index 000000000000..b35cda4b9d61 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/div.clif @@ -0,0 +1,35 @@ +test compile precise-output +target arm has_idiv + +function %sdiv(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = sdiv v0, v1 + return v2 +} + +; VCode: +; block0: +; sdiv r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; sdiv r0, r0, r1 +; bx lr + +function %udiv(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = udiv v0, v1 + return v2 +} + +; VCode: +; block0: +; udiv r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; udiv r0, r0, r1 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/extend.clif b/cranelift/filetests/filetests/isa/arm32/extend.clif new file mode 100644 index 000000000000..6165525df5ab --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/extend.clif @@ -0,0 +1,81 @@ +test compile precise-output +target arm + +function %sext8(i8) -> i32 { +block0(v0: i8): + v1 = sextend.i32 v0 + return v1 +} + +; VCode: +; block0: +; sxtb r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; sxtb r0, r0 +; bx lr + +function %sext16(i16) -> i32 { +block0(v0: i16): + v1 = sextend.i32 v0 + return v1 +} + +; VCode: +; block0: +; sxth r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; sxth r0, r0 +; bx lr + +function %uext8(i8) -> i32 { +block0(v0: i8): + v1 = uextend.i32 v0 + return v1 +} + +; VCode: +; block0: +; uxtb r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; uxtb r0, r0 +; bx lr + +function %uext16(i16) -> i32 { +block0(v0: i16): + v1 = uextend.i32 v0 + return v1 +} + +; VCode: +; block0: +; uxth r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; uxth r0, r0 +; bx lr + +function %reduce(i32) -> i8 { +block0(v0: i32): + v1 = ireduce.i8 v0 + return v1 +} + +; VCode: +; block0: +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/mul.clif b/cranelift/filetests/filetests/isa/arm32/mul.clif index d0677032bbca..4bb6371f1ba7 100644 --- a/cranelift/filetests/filetests/isa/arm32/mul.clif +++ b/cranelift/filetests/filetests/isa/arm32/mul.clif @@ -17,3 +17,20 @@ block0(v0: i32, v1: i32): ; mul r0, r0, r1 ; bx lr +function %mls(i32, i32, i32) -> i32 { +block0(v0: i32, v1: i32, v2: i32): + v3 = imul v1, v2 + v4 = isub v0, v3 + return v4 +} + +; VCode: +; block0: +; mls r0, r1, r2, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mls r0, r1, r2, r0 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/select.clif b/cranelift/filetests/filetests/isa/arm32/select.clif new file mode 100644 index 000000000000..6045c800a447 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/select.clif @@ -0,0 +1,42 @@ +test compile precise-output +target arm + +function %select_icmp(i32, i32, i32, i32) -> i32 { +block0(v0: i32, v1: i32, v2: i32, v3: i32): + v4 = icmp slt v0, v1 + v5 = select v4, v2, v3 + return v5 +} + +; VCode: +; block0: +; cmp r0, r1 +; mov r0, r3; movlt r0, r2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; cmp r0, r1 +; mov r0, r3 +; movlt r0, r2 +; bx lr + +function %select_val(i32, i32, i32) -> i32 { +block0(v0: i32, v1: i32, v2: i32): + v3 = select v0, v1, v2 + return v3 +} + +; VCode: +; block0: +; cmp r0, #0 +; mov r0, r2; movne r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; cmp r0, #0 +; mov r0, r2 +; movne r0, r1 +; bx lr + From 6604ac4a180b6714b34bd2a8ee8e0b0ce3736c63 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Fri, 3 Jul 2026 08:36:28 +0300 Subject: [PATCH 05/18] arm32: fill in the rest of the scalar-integer A32 encodings Adds encoders, operand handling, printing, and unit tests for the scalar-integer instructions the backend was still missing: the bitfield ops (bfc/bfi/sbfx/ubfx), the saturating arithmetic, sel/pkh/revsh/rrx/ udf, the full parallel add/sub family, the 16-bit and accumulating extends, the DSP multiplies, ldm/stm, the exclusive and acquire/release memory accesses, and the barriers. Most of these have no CLIF opcode that maps to them yet; they are here so the encoder is complete and the tests pin the bit patterns down. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/inst.isle | 150 +++++++- cranelift/codegen/src/isa/arm32/inst/args.rs | 323 +++++++++++++++- cranelift/codegen/src/isa/arm32/inst/emit.rs | 224 +++++++++++ .../codegen/src/isa/arm32/inst/emit_tests.rs | 360 ++++++++++++++++++ cranelift/codegen/src/isa/arm32/inst/mod.rs | 172 ++++++++- cranelift/codegen/src/isa/arm32/lower.isle | 10 + .../filetests/filetests/isa/arm32/fence.clif | 19 + .../filetests/filetests/isa/arm32/trap.clif | 16 + 8 files changed, 1269 insertions(+), 5 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/arm32/fence.clif create mode 100644 cranelift/filetests/filetests/isa/arm32/trap.clif diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index e4c29aa18640..a638c847246d 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -40,7 +40,47 @@ (Clz) (Rev) (Rev16) - (Rbit))) + (Rbit) + (Revsh))) + +;; A saturating add/subtract (`qadd`/`qsub`/`qdadd`/`qdsub`). +(type QAluOp + (enum + (Qadd) + (Qsub) + (Qdadd) + (Qdsub))) + +;; A saturate operation (`ssat`/`usat`/`ssat16`/`usat16`). +(type SatOp + (enum + (Ssat) + (Usat) + (Ssat16) + (Usat16))) + +;; A bitfield extract (`sbfx`/`ubfx`). +(type BfxOp + (enum + (Sbfx) + (Ubfx))) + +;; A halfword-pack operation (`pkhbt`/`pkhtb`). +(type PkhOp + (enum + (Pkhbt) + (Pkhtb))) + +;; A parallel (SIMD-in-GPR) 8/16-bit add/subtract operation. Names follow the +;; ARM mnemonics: prefix S/Q/SH/U/UQ/UH, op ADD8/ADD16/SUB8/SUB16/ASX/SAX. +(type ParAluOp + (enum + (Sadd8) (Sadd16) (Ssub8) (Ssub16) (Sasx) (Ssax) + (Qadd8) (Qadd16) (Qsub8) (Qsub16) (Qasx) (Qsax) + (Shadd8) (Shadd16) (Shsub8) (Shsub16) (Shasx) (Shsax) + (Uadd8) (Uadd16) (Usub8) (Usub16) (Uasx) (Usax) + (Uqadd8) (Uqadd16) (Uqsub8) (Uqsub16) (Uqasx) (Uqsax) + (Uhadd8) (Uhadd16) (Uhsub8) (Uhsub16) (Uhasx) (Uhsax))) ;; A sign/zero extension operation. (type ExtOp @@ -48,7 +88,50 @@ (Sxtb) (Sxth) (Uxtb) - (Uxth))) + (Uxth) + (Sxtb16) + (Uxtb16))) + +;; A sign/zero extend-and-add operation (`sxtab`/`uxtah`/...). +(type ExtAddOp + (enum + (Sxtab) + (Sxtah) + (Sxtab16) + (Uxtab) + (Uxtah) + (Uxtab16))) + +;; DSP multiplies with a single 32-bit result (`rd = f(rn, rm)`). +(type DspMul3Op + (enum + (Smulbb) (Smulbt) (Smultb) (Smultt) + (Smulwb) (Smulwt) + (Smmul) (Smuad) (Smusd))) + +;; DSP multiply-accumulates with a single 32-bit result (`rd = f(rn, rm) + ra`). +(type DspMul4Op + (enum + (Smlabb) (Smlabt) (Smlatb) (Smlatt) + (Smlawb) (Smlawt) + (Smmla) (Smmls) + (Smlad) (Smlsd))) + +;; DSP multiplies with a 64-bit accumulator (`{rd_hi:rd_lo} = ...`). +(type DspMulLOp + (enum + (Smlalbb) (Smlalbt) (Smlaltb) (Smlaltt) + (Smlald) (Smlsld) + (Umaal))) + +;; A memory-barrier / exclusive-monitor-clear instruction (all use the full +;; system option). +(type BarrierOp + (enum + (Dmb) + (Dsb) + (Isb) + (Clrex))) ;; An ARM condition code (the "always" and "never" codes are omitted). (type Cond @@ -152,6 +235,57 @@ ;; A conditional select, emitted as `mov rd, rm; mov rd, rn`. (CSel (cond Cond) (rd WritableReg) (rn Reg) (rm Reg)) + ;; Bitfield clear: `bfc rd, #lsb, #width`. + (Bfc (rd WritableReg) (lsb u8) (width u8)) + ;; Bitfield insert: `bfi rd, rn, #lsb, #width`. + (Bfi (rd WritableReg) (rn Reg) (lsb u8) (width u8)) + ;; Bitfield extract: `sbfx`/`ubfx rd, rn, #lsb, #width`. + (Bfx (op BfxOp) (rd WritableReg) (rn Reg) (lsb u8) (width u8)) + + ;; A saturating add/subtract (`op rd, rm, rn`). + (QAlu (op QAluOp) (rd WritableReg) (rm Reg) (rn Reg)) + ;; A saturate: `ssat/usat rd, #sat_bits, rm`. + (Sat (op SatOp) (rd WritableReg) (sat_bits u8) (rm Reg)) + + ;; A parallel 8/16-bit add/subtract (`op rd, rn, rm`). + (ParAlu (op ParAluOp) (rd WritableReg) (rn Reg) (rm Reg)) + + ;; A sign/zero extend-and-add (`op rd, rn, rm`). + (ExtAdd (op ExtAddOp) (rd WritableReg) (rn Reg) (rm Reg)) + + ;; A DSP multiply producing a single 32-bit result. + (DspMul3 (op DspMul3Op) (rd WritableReg) (rn Reg) (rm Reg)) + ;; A DSP multiply-accumulate producing a single 32-bit result. + (DspMul4 (op DspMul4Op) (rd WritableReg) (rn Reg) (rm Reg) (ra Reg)) + ;; A DSP multiply with a 64-bit accumulator pair. + (DspMulL (op DspMulLOp) (rd_lo WritableReg) (rd_hi WritableReg) (rn Reg) (rm Reg)) + + ;; Byte-wise select using the GE flags (`sel rd, rn, rm`). + (Sel (rd WritableReg) (rn Reg) (rm Reg)) + ;; Halfword pack (`pkhbt`/`pkhtb rd, rn, rm`). + (Pkh (op PkhOp) (rd WritableReg) (rn Reg) (rm Reg)) + ;; Rotate right with extend (`rrx rd, rm`). + (Rrx (rd WritableReg) (rm Reg)) + + ;; A permanently-undefined instruction used to encode a trap. + (Udf (code TrapCode)) + + ;; Load/store multiple, increment-after (`ldmia`/`stmia rn{!}, {list}`). + (LdmStm (load bool) (rn Reg) (writeback bool) (reg_list u32)) + + ;; Load-exclusive (`ldrex`) or load-acquire-exclusive (`ldaex`). + (LoadEx (acquire bool) (rt WritableReg) (rn Reg)) + ;; Store-exclusive (`strex`) or store-release-exclusive (`stlex`); `rd` + ;; receives the success flag. + (StoreEx (acquire bool) (rd WritableReg) (rt Reg) (rn Reg)) + ;; Load-acquire (`lda rt, [rn]`). + (LoadAcq (rt WritableReg) (rn Reg)) + ;; Store-release (`stl rt, [rn]`). + (StoreRel (rt Reg) (rn Reg)) + + ;; A memory barrier / clear-exclusive. + (Barrier (op BarrierOp)) + ;; `cmp/cmn/tst/teq rn, rm`. (CmpRR (op CmpOp) (rn Reg) (rm Reg)) ;; `cmp/cmn/tst/teq rn, #imm` (pre-encoded rotated imm12). @@ -318,6 +452,18 @@ (_ Unit (emit (MInst.CSel cond rd if_true if_false)))) rd)) +;;;; Traps ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl udf (TrapCode) SideEffectNoResult) +(rule (udf code) + (SideEffectNoResult.Inst (MInst.Udf code))) + +;;;; Barriers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl dmb () SideEffectNoResult) +(rule (dmb) + (SideEffectNoResult.Inst (MInst.Barrier (BarrierOp.Dmb)))) + ;;;; Shifts ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Shift/rotate by a constant amount. Handles the Cranelift "modulo width" diff --git a/cranelift/codegen/src/isa/arm32/inst/args.rs b/cranelift/codegen/src/isa/arm32/inst/args.rs index 03946741983a..95dc74786249 100644 --- a/cranelift/codegen/src/isa/arm32/inst/args.rs +++ b/cranelift/codegen/src/isa/arm32/inst/args.rs @@ -4,7 +4,8 @@ use crate::isa::arm32::inst::*; use crate::machinst::{OperandVisitor, Reg}; pub use crate::isa::arm32::lower::isle::generated_code::{ - ALUOp, AMode, BitOp, CmpOp, Cond, ExtOp, LoadKind, ShiftOp, StoreKind, + ALUOp, AMode, BarrierOp, BfxOp, BitOp, CmpOp, Cond, DspMul3Op, DspMul4Op, DspMulLOp, ExtAddOp, + ExtOp, LoadKind, ParAluOp, PkhOp, QAluOp, SatOp, ShiftOp, StoreKind, }; /// A memory address resolved to a concrete base register and either an @@ -147,6 +148,7 @@ impl BitOp { BitOp::Rev => 0x06bf_0f30, BitOp::Rev16 => 0x06bf_0fb0, BitOp::Rbit => 0x06ff_0f30, + BitOp::Revsh => 0x06ff_0fb0, } } @@ -156,6 +158,181 @@ impl BitOp { BitOp::Rev => "rev", BitOp::Rev16 => "rev16", BitOp::Rbit => "rbit", + BitOp::Revsh => "revsh", + } + } +} + +impl QAluOp { + /// The instruction word for `op rd, rm, rn`, with the registers zeroed. + pub(crate) fn template(self) -> u32 { + match self { + QAluOp::Qadd => 0x0100_0050, + QAluOp::Qsub => 0x0120_0050, + QAluOp::Qdadd => 0x0140_0050, + QAluOp::Qdsub => 0x0160_0050, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + QAluOp::Qadd => "qadd", + QAluOp::Qsub => "qsub", + QAluOp::Qdadd => "qdadd", + QAluOp::Qdsub => "qdsub", + } + } +} + +impl SatOp { + /// The instruction word for `op rd, #sat, rm` (no shift), registers zeroed. + pub(crate) fn template(self) -> u32 { + match self { + SatOp::Ssat => 0x06a0_0010, + SatOp::Usat => 0x06e0_0010, + SatOp::Ssat16 => 0x06a0_0f30, + SatOp::Usat16 => 0x06e0_0f30, + } + } + + /// Signed operations encode `sat_bits - 1`; unsigned encode `sat_bits`. + pub(crate) fn is_signed(self) -> bool { + matches!(self, SatOp::Ssat | SatOp::Ssat16) + } + + pub(crate) fn name(self) -> &'static str { + match self { + SatOp::Ssat => "ssat", + SatOp::Usat => "usat", + SatOp::Ssat16 => "ssat16", + SatOp::Usat16 => "usat16", + } + } +} + +impl BfxOp { + pub(crate) fn template(self) -> u32 { + match self { + BfxOp::Sbfx => 0x07a0_0050, + BfxOp::Ubfx => 0x07e0_0050, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + BfxOp::Sbfx => "sbfx", + BfxOp::Ubfx => "ubfx", + } + } +} + +impl PkhOp { + pub(crate) fn template(self) -> u32 { + match self { + PkhOp::Pkhbt => 0x0680_0010, + PkhOp::Pkhtb => 0x0680_0050, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + PkhOp::Pkhbt => "pkhbt", + PkhOp::Pkhtb => "pkhtb", + } + } +} + +impl ParAluOp { + /// The `(op1, op2)` fields of the parallel add/subtract encoding: `op1` + /// (bits 22:20) selects the prefix (signedness/saturation) and `op2` (bits + /// 7:5) selects the operation. + fn parts(self) -> (u32, u32) { + // Prefixes: S=1, Q=2, SH=3, U=5, UQ=6, UH=7. + // Ops: ADD16=0, ASX=1, SAX=2, SUB16=3, ADD8=4, SUB8=7. + match self { + ParAluOp::Sadd16 => (1, 0), + ParAluOp::Sasx => (1, 1), + ParAluOp::Ssax => (1, 2), + ParAluOp::Ssub16 => (1, 3), + ParAluOp::Sadd8 => (1, 4), + ParAluOp::Ssub8 => (1, 7), + ParAluOp::Qadd16 => (2, 0), + ParAluOp::Qasx => (2, 1), + ParAluOp::Qsax => (2, 2), + ParAluOp::Qsub16 => (2, 3), + ParAluOp::Qadd8 => (2, 4), + ParAluOp::Qsub8 => (2, 7), + ParAluOp::Shadd16 => (3, 0), + ParAluOp::Shasx => (3, 1), + ParAluOp::Shsax => (3, 2), + ParAluOp::Shsub16 => (3, 3), + ParAluOp::Shadd8 => (3, 4), + ParAluOp::Shsub8 => (3, 7), + ParAluOp::Uadd16 => (5, 0), + ParAluOp::Uasx => (5, 1), + ParAluOp::Usax => (5, 2), + ParAluOp::Usub16 => (5, 3), + ParAluOp::Uadd8 => (5, 4), + ParAluOp::Usub8 => (5, 7), + ParAluOp::Uqadd16 => (6, 0), + ParAluOp::Uqasx => (6, 1), + ParAluOp::Uqsax => (6, 2), + ParAluOp::Uqsub16 => (6, 3), + ParAluOp::Uqadd8 => (6, 4), + ParAluOp::Uqsub8 => (6, 7), + ParAluOp::Uhadd16 => (7, 0), + ParAluOp::Uhasx => (7, 1), + ParAluOp::Uhsax => (7, 2), + ParAluOp::Uhsub16 => (7, 3), + ParAluOp::Uhadd8 => (7, 4), + ParAluOp::Uhsub8 => (7, 7), + } + } + + /// The instruction word for `op rd, rn, rm`, with the registers zeroed. + pub(crate) fn template(self) -> u32 { + let (op1, op2) = self.parts(); + 0x0600_0000 | (op1 << 20) | 0x0000_0f00 | (op2 << 5) | 0x10 + } + + pub(crate) fn name(self) -> &'static str { + match self { + ParAluOp::Sadd8 => "sadd8", + ParAluOp::Sadd16 => "sadd16", + ParAluOp::Ssub8 => "ssub8", + ParAluOp::Ssub16 => "ssub16", + ParAluOp::Sasx => "sasx", + ParAluOp::Ssax => "ssax", + ParAluOp::Qadd8 => "qadd8", + ParAluOp::Qadd16 => "qadd16", + ParAluOp::Qsub8 => "qsub8", + ParAluOp::Qsub16 => "qsub16", + ParAluOp::Qasx => "qasx", + ParAluOp::Qsax => "qsax", + ParAluOp::Shadd8 => "shadd8", + ParAluOp::Shadd16 => "shadd16", + ParAluOp::Shsub8 => "shsub8", + ParAluOp::Shsub16 => "shsub16", + ParAluOp::Shasx => "shasx", + ParAluOp::Shsax => "shsax", + ParAluOp::Uadd8 => "uadd8", + ParAluOp::Uadd16 => "uadd16", + ParAluOp::Usub8 => "usub8", + ParAluOp::Usub16 => "usub16", + ParAluOp::Uasx => "uasx", + ParAluOp::Usax => "usax", + ParAluOp::Uqadd8 => "uqadd8", + ParAluOp::Uqadd16 => "uqadd16", + ParAluOp::Uqsub8 => "uqsub8", + ParAluOp::Uqsub16 => "uqsub16", + ParAluOp::Uqasx => "uqasx", + ParAluOp::Uqsax => "uqsax", + ParAluOp::Uhadd8 => "uhadd8", + ParAluOp::Uhadd16 => "uhadd16", + ParAluOp::Uhsub8 => "uhsub8", + ParAluOp::Uhsub16 => "uhsub16", + ParAluOp::Uhasx => "uhasx", + ParAluOp::Uhsax => "uhsax", } } } @@ -169,6 +346,8 @@ impl ExtOp { ExtOp::Sxth => 0x06bf_0070, ExtOp::Uxtb => 0x06ef_0070, ExtOp::Uxth => 0x06ff_0070, + ExtOp::Sxtb16 => 0x068f_0070, + ExtOp::Uxtb16 => 0x06cf_0070, } } @@ -178,6 +357,148 @@ impl ExtOp { ExtOp::Sxth => "sxth", ExtOp::Uxtb => "uxtb", ExtOp::Uxth => "uxth", + ExtOp::Sxtb16 => "sxtb16", + ExtOp::Uxtb16 => "uxtb16", + } + } +} + +impl ExtAddOp { + /// The instruction word for `op rd, rn, rm` (rotation 0), registers zeroed. + pub(crate) fn template(self) -> u32 { + match self { + ExtAddOp::Sxtab => 0x06a0_0070, + ExtAddOp::Sxtah => 0x06b0_0070, + ExtAddOp::Sxtab16 => 0x0680_0070, + ExtAddOp::Uxtab => 0x06e0_0070, + ExtAddOp::Uxtah => 0x06f0_0070, + ExtAddOp::Uxtab16 => 0x06c0_0070, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + ExtAddOp::Sxtab => "sxtab", + ExtAddOp::Sxtah => "sxtah", + ExtAddOp::Sxtab16 => "sxtab16", + ExtAddOp::Uxtab => "uxtab", + ExtAddOp::Uxtah => "uxtah", + ExtAddOp::Uxtab16 => "uxtab16", + } + } +} + +impl DspMul3Op { + /// The instruction word for `op rd, rn, rm`, with `rd`(19:16), `rm`(11:8), + /// `rn`(3:0) zeroed. The half-select (M/N) and pre-select bits are baked in. + pub(crate) fn template(self) -> u32 { + match self { + DspMul3Op::Smulbb => 0x0160_0080, + DspMul3Op::Smulbt => 0x0160_00c0, + DspMul3Op::Smultb => 0x0160_00a0, + DspMul3Op::Smultt => 0x0160_00e0, + DspMul3Op::Smulwb => 0x0120_00a0, + DspMul3Op::Smulwt => 0x0120_00e0, + DspMul3Op::Smmul => 0x0750_f010, + DspMul3Op::Smuad => 0x0700_f010, + DspMul3Op::Smusd => 0x0700_f050, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + DspMul3Op::Smulbb => "smulbb", + DspMul3Op::Smulbt => "smulbt", + DspMul3Op::Smultb => "smultb", + DspMul3Op::Smultt => "smultt", + DspMul3Op::Smulwb => "smulwb", + DspMul3Op::Smulwt => "smulwt", + DspMul3Op::Smmul => "smmul", + DspMul3Op::Smuad => "smuad", + DspMul3Op::Smusd => "smusd", + } + } +} + +impl DspMul4Op { + /// The instruction word for `op rd, rn, rm, ra`, with `rd`(19:16), + /// `ra`(15:12), `rm`(11:8), `rn`(3:0) zeroed. + pub(crate) fn template(self) -> u32 { + match self { + DspMul4Op::Smlabb => 0x0100_0080, + DspMul4Op::Smlabt => 0x0100_00c0, + DspMul4Op::Smlatb => 0x0100_00a0, + DspMul4Op::Smlatt => 0x0100_00e0, + DspMul4Op::Smlawb => 0x0120_0080, + DspMul4Op::Smlawt => 0x0120_00c0, + DspMul4Op::Smmla => 0x0750_0010, + DspMul4Op::Smmls => 0x0750_00d0, + DspMul4Op::Smlad => 0x0700_0010, + DspMul4Op::Smlsd => 0x0700_0050, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + DspMul4Op::Smlabb => "smlabb", + DspMul4Op::Smlabt => "smlabt", + DspMul4Op::Smlatb => "smlatb", + DspMul4Op::Smlatt => "smlatt", + DspMul4Op::Smlawb => "smlawb", + DspMul4Op::Smlawt => "smlawt", + DspMul4Op::Smmla => "smmla", + DspMul4Op::Smmls => "smmls", + DspMul4Op::Smlad => "smlad", + DspMul4Op::Smlsd => "smlsd", + } + } +} + +impl DspMulLOp { + /// The instruction word for `op rd_lo, rd_hi, rn, rm`, with `rd_hi`(19:16), + /// `rd_lo`(15:12), `rm`(11:8), `rn`(3:0) zeroed. + pub(crate) fn template(self) -> u32 { + match self { + DspMulLOp::Smlalbb => 0x0140_0080, + DspMulLOp::Smlalbt => 0x0140_00c0, + DspMulLOp::Smlaltb => 0x0140_00a0, + DspMulLOp::Smlaltt => 0x0140_00e0, + DspMulLOp::Smlald => 0x0740_0010, + DspMulLOp::Smlsld => 0x0740_0050, + DspMulLOp::Umaal => 0x0040_0090, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + DspMulLOp::Smlalbb => "smlalbb", + DspMulLOp::Smlalbt => "smlalbt", + DspMulLOp::Smlaltb => "smlaltb", + DspMulLOp::Smlaltt => "smlaltt", + DspMulLOp::Smlald => "smlald", + DspMulLOp::Smlsld => "smlsld", + DspMulLOp::Umaal => "umaal", + } + } +} + +impl BarrierOp { + /// The full (unconditional) instruction word, with the system option (SY). + pub(crate) fn encoding(self) -> u32 { + match self { + BarrierOp::Dmb => 0xf57f_f05f, + BarrierOp::Dsb => 0xf57f_f04f, + BarrierOp::Isb => 0xf57f_f06f, + BarrierOp::Clrex => 0xf57f_f01f, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + BarrierOp::Dmb => "dmb sy", + BarrierOp::Dsb => "dsb sy", + BarrierOp::Isb => "isb sy", + BarrierOp::Clrex => "clrex", } } } diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index e59b151f0fed..e43bc46beec0 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -261,6 +261,96 @@ fn enc_mov_cond(cond: Cond, rd: u32, rm: u32) -> u32 { (cond.bits() << 28) | 0x01a0_0000 | (rd << 12) | rm } +/// `bfc rd, #lsb, #width`. +fn enc_bfc(rd: u32, lsb: u32, width: u32) -> u32 { + let msb = lsb + width - 1; + COND_AL | 0x07c0_001f | (msb << 16) | (rd << 12) | (lsb << 7) +} + +/// `bfi rd, rn, #lsb, #width`. +fn enc_bfi(rd: u32, rn: u32, lsb: u32, width: u32) -> u32 { + let msb = lsb + width - 1; + COND_AL | 0x07c0_0010 | (msb << 16) | (rd << 12) | (lsb << 7) | rn +} + +/// `sbfx`/`ubfx rd, rn, #lsb, #width` from the op template. +fn enc_bfx(template: u32, rd: u32, rn: u32, lsb: u32, width: u32) -> u32 { + COND_AL | template | ((width - 1) << 16) | (rd << 12) | (lsb << 7) | rn +} + +/// A three-register op `template rd, rn, rm` with `rn`/`rd`/`rm` zeroed +/// (used by the saturating, parallel, sel, pkh and extend-add families). +fn enc_rd_rn_rm(template: u32, rd: u32, rn: u32, rm: u32) -> u32 { + COND_AL | template | (rn << 16) | (rd << 12) | rm +} + +/// `ssat`/`usat rd, #sat_bits, rm`. +fn enc_sat(op: SatOp, rd: u32, sat_bits: u32, rm: u32) -> u32 { + let field = if op.is_signed() { + sat_bits - 1 + } else { + sat_bits + }; + COND_AL | op.template() | (field << 16) | (rd << 12) | rm +} + +/// `rrx rd, rm` (a `mov` with a rotate-right-extend shift). +fn enc_rrx(rd: u32, rm: u32) -> u32 { + COND_AL | 0x01a0_0060 | (rd << 12) | rm +} + +/// The permanently-undefined `udf` encoding (`udf #0`). +fn enc_udf() -> u32 { + 0xe7f0_00f0 +} + +/// A DSP multiply with a single result: `template` plus `rd`(19:16), +/// `rm`(11:8), `rn`(3:0). +fn enc_dsp3(template: u32, rd: u32, rn: u32, rm: u32) -> u32 { + COND_AL | template | (rd << 16) | (rm << 8) | rn +} + +/// A DSP multiply-accumulate: `template` plus `rd`(19:16), `ra`(15:12), +/// `rm`(11:8), `rn`(3:0). +fn enc_dsp4(template: u32, rd: u32, rn: u32, rm: u32, ra: u32) -> u32 { + COND_AL | template | (rd << 16) | (ra << 12) | (rm << 8) | rn +} + +/// A DSP long multiply: `template` plus `rd_hi`(19:16), `rd_lo`(15:12), +/// `rm`(11:8), `rn`(3:0). +fn enc_dsp_long(template: u32, rd_lo: u32, rd_hi: u32, rn: u32, rm: u32) -> u32 { + COND_AL | template | (rd_hi << 16) | (rd_lo << 12) | (rm << 8) | rn +} + +/// `ldmia`/`stmia rn{!}, {reglist}` (increment-after). +fn enc_ldm_stm(load: bool, rn: u32, writeback: bool, reg_list: u32) -> u32 { + let base = if load { 0x0890_0000 } else { 0x0880_0000 }; + let wb = if writeback { 0x0020_0000 } else { 0 }; + COND_AL | base | wb | (rn << 16) | (reg_list & 0xffff) +} + +/// `ldrex`/`ldaex rt, [rn]`. +fn enc_load_ex(acquire: bool, rt: u32, rn: u32) -> u32 { + let base = if acquire { 0x0190_0e9f } else { 0x0190_0f9f }; + COND_AL | base | (rn << 16) | (rt << 12) +} + +/// `strex`/`stlex rd, rt, [rn]`. +fn enc_store_ex(acquire: bool, rd: u32, rt: u32, rn: u32) -> u32 { + let base = if acquire { 0x0180_0e90 } else { 0x0180_0f90 }; + COND_AL | base | (rn << 16) | (rd << 12) | rt +} + +/// `lda rt, [rn]`. +fn enc_lda(rt: u32, rn: u32) -> u32 { + COND_AL | 0x0190_0c9f | (rn << 16) | (rt << 12) +} + +/// `stl rt, [rn]`. +fn enc_stl(rt: u32, rn: u32) -> u32 { + COND_AL | 0x0180_fc90 | (rn << 16) | rt +} + fn put_u32(sink: &mut MachBuffer, word: u32) { for b in word.to_le_bytes() { sink.put1(b); @@ -441,6 +531,140 @@ impl MachInstEmit for Inst { put_u32(sink, enc_dp_reg(0b1101, 0, rd, 0, rm)); put_u32(sink, enc_mov_cond(*cond, rd, rn)); } + Inst::Bfc { rd, lsb, width } => { + let rd = machreg_to_gpr(rd.to_reg()); + put_u32(sink, enc_bfc(rd, u32::from(*lsb), u32::from(*width))); + } + Inst::Bfi { rd, rn, lsb, width } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_bfi(rd, rn, u32::from(*lsb), u32::from(*width))); + } + Inst::Bfx { + op, + rd, + rn, + lsb, + width, + } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + put_u32( + sink, + enc_bfx(op.template(), rd, rn, u32::from(*lsb), u32::from(*width)), + ); + } + Inst::QAlu { op, rd, rm, rn } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_rd_rn_rm(op.template(), rd, rn, rm)); + } + Inst::Sat { + op, + rd, + sat_bits, + rm, + } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_sat(*op, rd, u32::from(*sat_bits), rm)); + } + Inst::ParAlu { op, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_rd_rn_rm(op.template(), rd, rn, rm)); + } + Inst::ExtAdd { op, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_rd_rn_rm(op.template(), rd, rn, rm)); + } + Inst::DspMul3 { op, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_dsp3(op.template(), rd, rn, rm)); + } + Inst::DspMul4 { op, rd, rn, rm, ra } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + let ra = machreg_to_gpr(*ra); + put_u32(sink, enc_dsp4(op.template(), rd, rn, rm, ra)); + } + Inst::DspMulL { + op, + rd_lo, + rd_hi, + rn, + rm, + } => { + let rd_lo = machreg_to_gpr(rd_lo.to_reg()); + let rd_hi = machreg_to_gpr(rd_hi.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_dsp_long(op.template(), rd_lo, rd_hi, rn, rm)); + } + Inst::Sel { rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_rd_rn_rm(0x0680_0fb0, rd, rn, rm)); + } + Inst::Pkh { op, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_rd_rn_rm(op.template(), rd, rn, rm)); + } + Inst::Rrx { rd, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_rrx(rd, rm)); + } + Inst::Udf { code } => { + sink.add_trap(*code); + put_u32(sink, enc_udf()); + } + Inst::LdmStm { + load, + rn, + writeback, + reg_list, + } => { + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_ldm_stm(*load, rn, *writeback, *reg_list)); + } + Inst::LoadEx { acquire, rt, rn } => { + let rt = machreg_to_gpr(rt.to_reg()); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_load_ex(*acquire, rt, rn)); + } + Inst::StoreEx { + acquire, + rd, + rt, + rn, + } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rt = machreg_to_gpr(*rt); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_store_ex(*acquire, rd, rt, rn)); + } + Inst::LoadAcq { rt, rn } => { + let rt = machreg_to_gpr(rt.to_reg()); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_lda(rt, rn)); + } + Inst::StoreRel { rt, rn } => { + let rt = machreg_to_gpr(*rt); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_stl(rt, rn)); + } + Inst::Barrier { op } => put_u32(sink, op.encoding()), Inst::CmpRR { op, rn, rm } => { let rn = machreg_to_gpr(*rn); let rm = machreg_to_gpr(*rm); diff --git a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs index dd73ab46aedf..633d4fe04a9d 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs @@ -354,6 +354,366 @@ fn conditional_select() { assert_eq!(w1, 0x11a0_0001); // movne r0, r1 } +#[test] +fn bitfield() { + assert_eq!( + u32_le(Inst::Bfc { + rd: writable_xreg(0), + lsb: 4, + width: 8, + }), + 0xe7cb_021f // bfc r0, #4, #8 + ); + assert_eq!( + u32_le(Inst::Bfi { + rd: writable_xreg(0), + rn: xreg(1), + lsb: 4, + width: 8, + }), + 0xe7cb_0211 // bfi r0, r1, #4, #8 + ); + assert_eq!( + u32_le(Inst::Bfx { + op: BfxOp::Sbfx, + rd: writable_xreg(0), + rn: xreg(1), + lsb: 4, + width: 8, + }), + 0xe7a7_0251 // sbfx r0, r1, #4, #8 + ); + assert_eq!( + u32_le(Inst::Bfx { + op: BfxOp::Ubfx, + rd: writable_xreg(0), + rn: xreg(1), + lsb: 4, + width: 8, + }), + 0xe7e7_0251 // ubfx r0, r1, #4, #8 + ); +} + +#[test] +fn saturating() { + let q_cases = [ + (QAluOp::Qadd, 0xe102_0051u32), + (QAluOp::Qsub, 0xe122_0051), + (QAluOp::Qdadd, 0xe142_0051), + (QAluOp::Qdsub, 0xe162_0051), + ]; + for (op, want) in q_cases { + // qadd r0, r1, r2 => Rd=0, Rm=1, Rn=2 + assert_eq!( + u32_le(Inst::QAlu { + op, + rd: writable_xreg(0), + rm: xreg(1), + rn: xreg(2), + }), + want, + "{op:?}" + ); + } + assert_eq!( + u32_le(Inst::Sat { + op: SatOp::Ssat, + rd: writable_xreg(0), + sat_bits: 8, + rm: xreg(1), + }), + 0xe6a7_0011 // ssat r0, #8, r1 + ); + assert_eq!( + u32_le(Inst::Sat { + op: SatOp::Usat, + rd: writable_xreg(0), + sat_bits: 8, + rm: xreg(1), + }), + 0xe6e8_0011 // usat r0, #8, r1 + ); +} + +#[test] +fn misc_alu() { + assert_eq!( + u32_le(Inst::Sel { + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe681_0fb2 // sel r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::Pkh { + op: PkhOp::Pkhbt, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe681_0012 // pkhbt r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::Pkh { + op: PkhOp::Pkhtb, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe681_0052 // pkhtb r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::Rrx { + rd: writable_xreg(0), + rm: xreg(1), + }), + 0xe1a0_0061 // rrx r0, r1 + ); + assert_eq!( + u32_le(Inst::BitRR { + op: BitOp::Revsh, + rd: writable_xreg(0), + rm: xreg(1), + }), + 0xe6ff_0fb1 // revsh r0, r1 + ); + assert_eq!( + u32_le(Inst::Udf { + code: crate::ir::TrapCode::STACK_OVERFLOW, + }), + 0xe7f0_00f0 // udf #0 + ); +} + +#[test] +fn extend_variants() { + assert_eq!( + u32_le(Inst::ExtRR { + op: ExtOp::Sxtb16, + rd: writable_xreg(0), + rm: xreg(1), + }), + 0xe68f_0071 // sxtb16 r0, r1 + ); + assert_eq!( + u32_le(Inst::ExtRR { + op: ExtOp::Uxtb16, + rd: writable_xreg(0), + rm: xreg(1), + }), + 0xe6cf_0071 // uxtb16 r0, r1 + ); + let add_cases = [ + (ExtAddOp::Sxtab, 0xe6a1_0072u32), + (ExtAddOp::Sxtah, 0xe6b1_0072), + (ExtAddOp::Sxtab16, 0xe681_0072), + (ExtAddOp::Uxtab, 0xe6e1_0072), + (ExtAddOp::Uxtah, 0xe6f1_0072), + (ExtAddOp::Uxtab16, 0xe6c1_0072), + ]; + for (op, want) in add_cases { + assert_eq!( + u32_le(Inst::ExtAdd { + op, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + want, + "{op:?}" + ); + } +} + +#[test] +fn parallel_add_sub() { + let par = |op| { + u32_le(Inst::ParAlu { + op, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }) + }; + // Hand-verified reference encodings (`op r0, r1, r2`). + assert_eq!(par(ParAluOp::Sadd8), 0xe611_0f92); + assert_eq!(par(ParAluOp::Ssub8), 0xe611_0ff2); + assert_eq!(par(ParAluOp::Uadd8), 0xe651_0f92); + assert_eq!(par(ParAluOp::Uadd16), 0xe651_0f12); + assert_eq!(par(ParAluOp::Uhadd16), 0xe671_0f12); + + // All 36 members must encode to distinct, well-formed words that share the + // parallel add/sub encoding skeleton. + let all = [ + ParAluOp::Sadd8, ParAluOp::Sadd16, ParAluOp::Ssub8, ParAluOp::Ssub16, + ParAluOp::Sasx, ParAluOp::Ssax, ParAluOp::Qadd8, ParAluOp::Qadd16, + ParAluOp::Qsub8, ParAluOp::Qsub16, ParAluOp::Qasx, ParAluOp::Qsax, + ParAluOp::Shadd8, ParAluOp::Shadd16, ParAluOp::Shsub8, ParAluOp::Shsub16, + ParAluOp::Shasx, ParAluOp::Shsax, ParAluOp::Uadd8, ParAluOp::Uadd16, + ParAluOp::Usub8, ParAluOp::Usub16, ParAluOp::Uasx, ParAluOp::Usax, + ParAluOp::Uqadd8, ParAluOp::Uqadd16, ParAluOp::Uqsub8, ParAluOp::Uqsub16, + ParAluOp::Uqasx, ParAluOp::Uqsax, ParAluOp::Uhadd8, ParAluOp::Uhadd16, + ParAluOp::Uhsub8, ParAluOp::Uhsub16, ParAluOp::Uhasx, ParAluOp::Uhsax, + ]; + let mut seen = alloc::collections::BTreeSet::new(); + for op in all { + let w = par(op); + // Fixed skeleton: cond=AL + [27:24]=0110, bit23=0, [11:8]=1111 & bit4=1, + // and the register fields Rn=1, Rd=0, Rm=2. + assert_eq!(w & 0xff80_0000, 0xe600_0000, "{op:?}"); + assert_eq!(w & 0x0000_0f10, 0x0000_0f10, "{op:?}"); + assert_eq!(w & 0x000f_f00f, 0x0001_0002, "{op:?}"); + assert!(seen.insert(w), "duplicate encoding for {op:?}"); + } + assert_eq!(seen.len(), 36); +} + +#[test] +fn dsp_multiplies() { + let m3 = |op| { + u32_le(Inst::DspMul3 { + op, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }) + }; + // `op r0, r1, r2` + assert_eq!(m3(DspMul3Op::Smulbb), 0xe160_0281); + assert_eq!(m3(DspMul3Op::Smulbt), 0xe160_02c1); + assert_eq!(m3(DspMul3Op::Smultb), 0xe160_02a1); + assert_eq!(m3(DspMul3Op::Smultt), 0xe160_02e1); + assert_eq!(m3(DspMul3Op::Smulwb), 0xe120_02a1); + assert_eq!(m3(DspMul3Op::Smulwt), 0xe120_02e1); + assert_eq!(m3(DspMul3Op::Smmul), 0xe750_f211); + assert_eq!(m3(DspMul3Op::Smuad), 0xe700_f211); + assert_eq!(m3(DspMul3Op::Smusd), 0xe700_f251); + + let m4 = |op| { + u32_le(Inst::DspMul4 { + op, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + ra: xreg(3), + }) + }; + // `op r0, r1, r2, r3` + assert_eq!(m4(DspMul4Op::Smlabb), 0xe100_3281); + assert_eq!(m4(DspMul4Op::Smlabt), 0xe100_32c1); + assert_eq!(m4(DspMul4Op::Smlatb), 0xe100_32a1); + assert_eq!(m4(DspMul4Op::Smlatt), 0xe100_32e1); + assert_eq!(m4(DspMul4Op::Smlawb), 0xe120_3281); + assert_eq!(m4(DspMul4Op::Smlawt), 0xe120_32c1); + assert_eq!(m4(DspMul4Op::Smmla), 0xe750_3211); + assert_eq!(m4(DspMul4Op::Smmls), 0xe750_32d1); + assert_eq!(m4(DspMul4Op::Smlad), 0xe700_3211); + assert_eq!(m4(DspMul4Op::Smlsd), 0xe700_3251); + + let ml = |op| { + u32_le(Inst::DspMulL { + op, + rd_lo: writable_xreg(0), + rd_hi: writable_xreg(1), + rn: xreg(2), + rm: xreg(3), + }) + }; + // `op r0, r1, r2, r3` + assert_eq!(ml(DspMulLOp::Smlalbb), 0xe141_0382); + assert_eq!(ml(DspMulLOp::Smlalbt), 0xe141_03c2); + assert_eq!(ml(DspMulLOp::Smlaltb), 0xe141_03a2); + assert_eq!(ml(DspMulLOp::Smlaltt), 0xe141_03e2); + assert_eq!(ml(DspMulLOp::Smlald), 0xe741_0312); + assert_eq!(ml(DspMulLOp::Smlsld), 0xe741_0352); + assert_eq!(ml(DspMulLOp::Umaal), 0xe041_0392); +} + +#[test] +fn memory_multiple_and_exclusive() { + assert_eq!( + u32_le(Inst::LdmStm { + load: true, + rn: xreg(0), + writeback: true, + reg_list: (1 << 1) | (1 << 2), + }), + 0xe8b0_0006 // ldmia r0!, {r1, r2} + ); + assert_eq!( + u32_le(Inst::LdmStm { + load: false, + rn: xreg(0), + writeback: false, + reg_list: (1 << 1) | (1 << 2), + }), + 0xe880_0006 // stmia r0, {r1, r2} + ); + assert_eq!( + u32_le(Inst::LoadEx { + acquire: false, + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe190_1f9f // ldrex r1, [r0] + ); + assert_eq!( + u32_le(Inst::LoadEx { + acquire: true, + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe190_1e9f // ldaex r1, [r0] + ); + assert_eq!( + u32_le(Inst::StoreEx { + acquire: false, + rd: writable_xreg(0), + rt: xreg(1), + rn: xreg(2), + }), + 0xe182_0f91 // strex r0, r1, [r2] + ); + assert_eq!( + u32_le(Inst::StoreEx { + acquire: true, + rd: writable_xreg(0), + rt: xreg(1), + rn: xreg(2), + }), + 0xe182_0e91 // stlex r0, r1, [r2] + ); + assert_eq!( + u32_le(Inst::LoadAcq { + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe190_1c9f // lda r1, [r0] + ); + assert_eq!( + u32_le(Inst::StoreRel { + rt: xreg(1), + rn: xreg(0), + }), + 0xe180_fc91 // stl r1, [r0] + ); +} + +#[test] +fn barriers() { + assert_eq!(u32_le(Inst::Barrier { op: BarrierOp::Dmb }), 0xf57f_f05f); + assert_eq!(u32_le(Inst::Barrier { op: BarrierOp::Dsb }), 0xf57f_f04f); + assert_eq!(u32_le(Inst::Barrier { op: BarrierOp::Isb }), 0xf57f_f06f); + assert_eq!( + u32_le(Inst::Barrier { + op: BarrierOp::Clrex + }), + 0xf57f_f01f + ); +} + #[test] fn compares() { assert_eq!( diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index bb715925f6d9..03fe4271b465 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -93,7 +93,14 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { use_if_virtual(collector, rm); def_if_virtual(collector, rd); } - Inst::Mla { rd, rn, rm, ra } | Inst::Mls { rd, rn, rm, ra } => { + Inst::DspMul3 { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::Mla { rd, rn, rm, ra } + | Inst::Mls { rd, rn, rm, ra } + | Inst::DspMul4 { rd, rn, rm, ra, .. } => { use_if_virtual(collector, rn); use_if_virtual(collector, rm); use_if_virtual(collector, ra); @@ -117,6 +124,43 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { collector.reg_early_def(rd); } } + Inst::Bfc { rd, .. } => def_if_virtual(collector, rd), + Inst::Sat { rd, rm, .. } | Inst::Rrx { rd, rm } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::Bfi { rd, rn, .. } | Inst::Bfx { rd, rn, .. } => { + use_if_virtual(collector, rn); + def_if_virtual(collector, rd); + } + Inst::QAlu { rd, rm, rn, .. } => { + use_if_virtual(collector, rm); + use_if_virtual(collector, rn); + def_if_virtual(collector, rd); + } + Inst::Sel { rd, rn, rm } + | Inst::Pkh { rd, rn, rm, .. } + | Inst::ParAlu { rd, rn, rm, .. } + | Inst::ExtAdd { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::Udf { .. } | Inst::Barrier { .. } => {} + Inst::LdmStm { rn, .. } => use_if_virtual(collector, rn), + Inst::LoadEx { rt, rn, .. } | Inst::LoadAcq { rt, rn } => { + use_if_virtual(collector, rn); + def_if_virtual(collector, rt); + } + Inst::StoreEx { rd, rt, rn, .. } => { + use_if_virtual(collector, rt); + use_if_virtual(collector, rn); + def_if_virtual(collector, rd); + } + Inst::StoreRel { rt, rn } => { + use_if_virtual(collector, rt); + use_if_virtual(collector, rn); + } Inst::Umull { rd_lo, rd_hi, @@ -140,6 +184,13 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { rd_hi, rn, rm, + } + | Inst::DspMulL { + rd_lo, + rd_hi, + rn, + rm, + .. } => { use_if_virtual(collector, rn); use_if_virtual(collector, rm); @@ -237,7 +288,7 @@ impl MachInst for Inst { } fn is_trap(&self) -> bool { - false + matches!(self, Inst::Udf { .. }) } fn is_args(&self) -> bool { @@ -465,6 +516,123 @@ impl Inst { let rd = r(rd.to_reg()); alloc::format!("mov {rd}, {}; mov{} {rd}, {}", r(*rm), cond.name(), r(*rn)) } + Inst::Bfc { rd, lsb, width } => { + alloc::format!("bfc {}, #{}, #{}", r(rd.to_reg()), lsb, width) + } + Inst::Bfi { rd, rn, lsb, width } => { + alloc::format!("bfi {}, {}, #{}, #{}", r(rd.to_reg()), r(*rn), lsb, width) + } + Inst::Bfx { + op, + rd, + rn, + lsb, + width, + } => alloc::format!( + "{} {}, {}, #{}, #{}", + op.name(), + r(rd.to_reg()), + r(*rn), + lsb, + width + ), + Inst::QAlu { op, rd, rm, rn } => { + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rm), r(*rn)) + } + Inst::Sat { + op, + rd, + sat_bits, + rm, + } => alloc::format!( + "{} {}, #{}, {}", + op.name(), + r(rd.to_reg()), + sat_bits, + r(*rm) + ), + Inst::ParAlu { op, rd, rn, rm } => { + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::ExtAdd { op, rd, rn, rm } => { + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::DspMul3 { op, rd, rn, rm } => { + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::DspMul4 { op, rd, rn, rm, ra } => alloc::format!( + "{} {}, {}, {}, {}", + op.name(), + r(rd.to_reg()), + r(*rn), + r(*rm), + r(*ra) + ), + Inst::DspMulL { + op, + rd_lo, + rd_hi, + rn, + rm, + } => alloc::format!( + "{} {}, {}, {}, {}", + op.name(), + r(rd_lo.to_reg()), + r(rd_hi.to_reg()), + r(*rn), + r(*rm) + ), + Inst::Sel { rd, rn, rm } => { + alloc::format!("sel {}, {}, {}", r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::Pkh { op, rd, rn, rm } => { + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::Rrx { rd, rm } => { + alloc::format!("rrx {}, {}", r(rd.to_reg()), r(*rm)) + } + Inst::Udf { code } => alloc::format!("udf ; {code}"), + Inst::LdmStm { + load, + rn, + writeback, + reg_list, + } => { + let names: Vec = (0..16) + .filter(|i| reg_list & (1 << i) != 0) + .map(|i| reg_name(xreg(i))) + .collect(); + alloc::format!( + "{} {}{}, {{{}}}", + if *load { "ldmia" } else { "stmia" }, + r(*rn), + if *writeback { "!" } else { "" }, + names.join(", ") + ) + } + Inst::LoadEx { acquire, rt, rn } => alloc::format!( + "{} {}, [{}]", + if *acquire { "ldaex" } else { "ldrex" }, + r(rt.to_reg()), + r(*rn) + ), + Inst::StoreEx { + acquire, + rd, + rt, + rn, + } => alloc::format!( + "{} {}, {}, [{}]", + if *acquire { "stlex" } else { "strex" }, + r(rd.to_reg()), + r(*rt), + r(*rn) + ), + Inst::LoadAcq { rt, rn } => { + alloc::format!("lda {}, [{}]", r(rt.to_reg()), r(*rn)) + } + Inst::StoreRel { rt, rn } => alloc::format!("stl {}, [{}]", r(*rt), r(*rn)), + Inst::Barrier { op } => op.name().to_string(), Inst::CmpRR { op, rn, rm } => { alloc::format!("{} {}, {}", op.name(), r(*rn), r(*rm)) } diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index 426e1e9715a9..4c6d8398fc9a 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -63,6 +63,16 @@ (rule 2 (lower (isub (fits_in_32 _) a (imul _ x y))) (mls (put_in_reg x) (put_in_reg y) (put_in_reg a))) +;;;; Traps ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (trap code)) + (side_effect (udf code))) + +;;;; Fences ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (fence)) + (side_effect (dmb))) + ;;;; Divides (only when hardware `sdiv`/`udiv` is available) ;;;;;;;;;;;;;;;;;;; ;; NOTE: trap-on-zero / INT_MIN overflow checks are not yet emitted. diff --git a/cranelift/filetests/filetests/isa/arm32/fence.clif b/cranelift/filetests/filetests/isa/arm32/fence.clif new file mode 100644 index 000000000000..29658e86dfcc --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/fence.clif @@ -0,0 +1,19 @@ +test compile precise-output +target arm + +function %fence() { +block0: + fence + return +} + +; VCode: +; block0: +; dmb sy +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; dmb sy +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/trap.clif b/cranelift/filetests/filetests/isa/arm32/trap.clif new file mode 100644 index 000000000000..cf667fcd36f7 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/trap.clif @@ -0,0 +1,16 @@ +test compile precise-output +target arm + +function %trap() { +block0: + trap user1 +} + +; VCode: +; block0: +; udf ; user1 +; +; Disassembled: +; block0: ; offset 0x0 +; udf #0 ; trap: user1 + From acbc24cbb7664dcfef278bc34b33d60c45b91bba Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Fri, 3 Jul 2026 20:41:13 +0300 Subject: [PATCH 06/18] arm32: lower direct and indirect calls A direct call to a near/colocated callee becomes a bl with an Arm32Call relocation; call_indirect becomes a blx through a register. The call and indirect-call instructions record the safepoint and stack map, pop the callee's stack arguments, and load any stack-returned values. Both forms are treated as calls by call_type/is_safepoint so the frame is set up and lr is preserved around them -- getting that wrong for the indirect case corrupts the return address. Far calls, return_call, and try_call are left for later. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/inst.isle | 19 +++++++ cranelift/codegen/src/isa/arm32/inst/emit.rs | 45 ++++++++++++++- cranelift/codegen/src/isa/arm32/inst/mod.rs | 21 ++++++- cranelift/codegen/src/isa/arm32/lower.isle | 23 ++++++++ cranelift/codegen/src/isa/arm32/lower/isle.rs | 42 +++++++++++++- .../filetests/filetests/isa/arm32/call.clif | 57 +++++++++++++++++++ 6 files changed, 202 insertions(+), 5 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/arm32/call.clif diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index a638c847246d..f35eb808b21b 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -170,6 +170,7 @@ (IncomingArg (offset i64)))) (type BoxCallInfo (primitive BoxCallInfo)) +(type BoxCallIndInfo (primitive BoxCallIndInfo)) ;;;; Instruction formats ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -306,6 +307,8 @@ ;; A direct call (`bl`). (Call (info BoxCallInfo)) + ;; An indirect call (`blx rm`). + (CallInd (info BoxCallIndInfo)) ;; An unconditional branch to a label (`b`). (Jump (dest MachLabel)) @@ -518,6 +521,22 @@ (rule (jump_impl label) (SideEffectNoResult.Inst (MInst.Jump label))) +;;;; Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl gen_call_info (Sig ExternalName CallArgList CallRetList OptionTryCallInfo bool) BoxCallInfo) +(extern constructor gen_call_info gen_call_info) + +(decl gen_call_ind_info (Sig Reg CallArgList CallRetList OptionTryCallInfo) BoxCallIndInfo) +(extern constructor gen_call_ind_info gen_call_ind_info) + +(decl call_impl (BoxCallInfo) SideEffectNoResult) +(rule (call_impl info) + (SideEffectNoResult.Inst (MInst.Call info))) + +(decl call_ind_impl (BoxCallIndInfo) SideEffectNoResult) +(rule (call_ind_impl info) + (SideEffectNoResult.Inst (MInst.CallInd info))) + ;; Map an `IntCC` to the corresponding ARM condition code. (decl cond_from_intcc (IntCC) Cond) (extern constructor cond_from_intcc cond_from_intcc) diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index e43bc46beec0..db15684f4b48 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -38,7 +38,6 @@ pub struct EmitState { } impl EmitState { - #[expect(dead_code, reason = "will be used once safepoints are supported")] fn take_stack_map(&mut self) -> Option { self.user_stack_map.take() } @@ -130,6 +129,42 @@ fn enc_bl(imm24: u32) -> u32 { COND_AL | 0x0b00_0000 | (imm24 & 0x00ff_ffff) } +/// `blx rm` — branch with link and exchange to a register (indirect call). +fn enc_blx(rm: u32) -> u32 { + COND_AL | 0x012f_ff30 | rm +} + +/// Common post-call sequence: record the safepoint/call site, pop callee stack +/// args, and load any stack-carried return values. +fn emit_call_epilogue( + sink: &mut MachBuffer, + emit_info: &EmitInfo, + state: &mut EmitState, + info: &crate::machinst::CallInfo, +) { + use crate::isa::arm32::abi::Arm32MachineDeps; + use crate::machinst::ABIMachineSpec; + + if let Some(s) = state.take_stack_map() { + let offset = sink.cur_offset(); + sink.push_user_stack_map(state, offset, s); + } + sink.add_call_site(); + + let callee_pop_size = i32::try_from(info.callee_pop_size).unwrap(); + if callee_pop_size > 0 { + for inst in Arm32MachineDeps::gen_sp_reg_adjust(-callee_pop_size) { + inst.emit(sink, emit_info, state); + } + } + + info.emit_retval_loads::( + state.frame_layout().stackslots_size, + |inst| inst.emit(sink, emit_info, state), + |_needed_space| None, + ); +} + /// `add`/`sub sp, sp, #imm`. fn enc_sp_adjust(amount: i32) -> u32 { let (op, mag) = if amount < 0 { @@ -361,7 +396,7 @@ impl MachInstEmit for Inst { type State = EmitState; type Info = EmitInfo; - fn emit(&self, sink: &mut MachBuffer, _emit_info: &Self::Info, state: &mut EmitState) { + fn emit(&self, sink: &mut MachBuffer, emit_info: &Self::Info, state: &mut EmitState) { let start = sink.cur_offset(); match self { Inst::Nop0 => {} @@ -692,6 +727,12 @@ impl MachInstEmit for Inst { Inst::Call { info } => { sink.add_reloc(Reloc::Arm32Call, &info.dest, 0); put_u32(sink, enc_bl(0)); + emit_call_epilogue(sink, emit_info, state, info); + } + Inst::CallInd { info } => { + let rm = machreg_to_gpr(info.dest); + put_u32(sink, enc_blx(rm)); + emit_call_epilogue(sink, emit_info, state, info); } Inst::Jump { dest } => { diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index 03fe4271b465..459884f3d773 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -233,6 +233,22 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { } collector.reg_clobbers(info.clobbers); } + Inst::CallInd { info } => { + let CallInfo { + dest, uses, defs, .. + } = &mut **info; + collector.reg_use(dest); + for CallArgPair { vreg, preg } in uses { + collector.reg_fixed_use(vreg, *preg); + } + for CallRetPair { vreg, location } in defs { + match location { + RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg), + RetLocation::Stack(..) => collector.any_def(vreg), + } + } + collector.reg_clobbers(info.clobbers); + } Inst::Args { args } => { for ArgPair { vreg, preg } in args { @@ -269,7 +285,7 @@ impl MachInst for Inst { } fn is_safepoint(&self) -> bool { - matches!(self, Inst::Call { .. }) + matches!(self, Inst::Call { .. } | Inst::CallInd { .. }) } fn get_operands(&mut self, collector: &mut impl OperandVisitor) { @@ -297,7 +313,7 @@ impl MachInst for Inst { fn call_type(&self) -> CallType { match self { - Inst::Call { .. } => CallType::Regular, + Inst::Call { .. } | Inst::CallInd { .. } => CallType::Regular, _ => CallType::None, } } @@ -660,6 +676,7 @@ impl Inst { Inst::Push { reg_list } => alloc::format!("push {}", reglist(*reg_list)), Inst::Pop { reg_list } => alloc::format!("pop {}", reglist(*reg_list)), Inst::Call { info } => alloc::format!("bl {}", info.dest.display(None)), + Inst::CallInd { info } => alloc::format!("blx {}", r(info.dest)), Inst::Jump { dest } => alloc::format!("b {}", dest.to_string()), Inst::CondBr { cond, diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index 4c6d8398fc9a..0fc9cf1721c5 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -173,6 +173,29 @@ (rule (lower (return args)) (lower_return args)) +;;;; Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Direct call to an in-range function (`bl`). +(rule (lower (call (func_ref_data sig_ref name (RelocDistance.Near) patchable) args)) + (let ((output ValueRegsVec (gen_call_output sig_ref)) + (abi Sig (abi_sig sig_ref)) + (uses CallArgList (gen_call_args abi args)) + (defs CallRetList (gen_call_rets abi output)) + (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none) patchable)) + (_ Unit (emit_side_effect (call_impl info)))) + output)) + +;; Indirect call through a register (`blx`). +(rule (lower (call_indirect sig_ref ptr args)) + (let ((output ValueRegsVec (gen_call_output sig_ref)) + (abi Sig (abi_sig sig_ref)) + (target Reg (put_in_reg ptr)) + (uses CallArgList (gen_call_args abi args)) + (defs CallRetList (gen_call_rets abi output)) + (info BoxCallIndInfo (gen_call_ind_info abi target uses defs (try_call_none))) + (_ Unit (emit_side_effect (call_ind_impl info)))) + output)) + ;;;; Branches ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl partial lower_branch (Inst MachLabelSlice) Unit) diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs index e8efb0b87aff..936c258dfa1c 100644 --- a/cranelift/codegen/src/isa/arm32/lower/isle.rs +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -16,7 +16,7 @@ use crate::isa::arm32::inst::{Cond, ShiftOp, encode_rotated_imm}; use crate::machinst::isle::*; use crate::machinst::{ ArgPair, CallArgList, CallInfo, CallRetList, InstOutput, Lower, MachInst, MachLabel, RetPair, - VCodeConstant, VCodeConstantData, VCodeInst, + Sig, TryCallInfo, VCodeConstant, VCodeConstantData, VCodeInst, }; use alloc::boxed::Box; use alloc::vec::Vec; @@ -25,6 +25,7 @@ use regalloc2::PReg; type VecArgPair = Vec; type VecRetPair = Vec; type BoxCallInfo = Box>; +type BoxCallIndInfo = Box>; /// The ISLE lowering context for arm32. pub(crate) struct Arm32IsleContext<'a, 'b, I, B> @@ -53,6 +54,45 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { self.lower_ctx.emit(inst.clone()); } + fn gen_call_info( + &mut self, + sig: Sig, + dest: ExternalName, + uses: CallArgList, + defs: CallRetList, + try_call_info: Option, + patchable: bool, + ) -> BoxCallInfo { + let stack_ret_space = self.lower_ctx.sigs()[sig].sized_stack_ret_space(); + let stack_arg_space = self.lower_ctx.sigs()[sig].sized_stack_arg_space(); + self.lower_ctx + .abi_mut() + .accumulate_outgoing_args_size(stack_ret_space + stack_arg_space); + Box::new( + self.lower_ctx + .gen_call_info(sig, dest, uses, defs, try_call_info, patchable), + ) + } + + fn gen_call_ind_info( + &mut self, + sig: Sig, + dest: Reg, + uses: CallArgList, + defs: CallRetList, + try_call_info: Option, + ) -> BoxCallIndInfo { + let stack_ret_space = self.lower_ctx.sigs()[sig].sized_stack_ret_space(); + let stack_arg_space = self.lower_ctx.sigs()[sig].sized_stack_arg_space(); + self.lower_ctx + .abi_mut() + .accumulate_outgoing_args_size(stack_ret_space + stack_arg_space); + Box::new( + self.lower_ctx + .gen_call_info(sig, dest, uses, defs, try_call_info, false), + ) + } + /// Materialize a 32-bit constant into a register with the shortest sequence. fn gen_constant(&mut self, val: u64) -> Reg { let val = val as u32; diff --git a/cranelift/filetests/filetests/isa/arm32/call.clif b/cranelift/filetests/filetests/isa/arm32/call.clif new file mode 100644 index 000000000000..a76d9436ce3a --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/call.clif @@ -0,0 +1,57 @@ +test compile precise-output +target arm + +function %call_direct(i32) -> i32 { + fn0 = colocated %g(i32) -> i32 +block0(v0: i32): + v1 = call fn0(v0) + return v1 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; block0: +; bl %g +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; block1: ; offset 0x8 +; bl #0x10 ; reloc_external Call %g 0 +; pop {fp, lr} +; bx lr + +function %call_indirect(i32, i32) -> i32 { + sig0 = (i32) -> i32 +block0(v0: i32, v1: i32): + v2 = call_indirect sig0, v0(v1) + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; block0: +; mov r2, r0 +; mov r0, r1 +; mov r1, r2 +; blx r1 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; block1: ; offset 0x8 +; mov r2, r0 +; mov r0, r1 +; mov r1, r2 +; blx r1 +; pop {fp, lr} +; bx lr + From d5548fc8745bcfb5fb2aca54096eb9f8daf78d0c Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Fri, 3 Jul 2026 22:07:01 +0300 Subject: [PATCH 07/18] arm32: represent i64 as a register pair An i64 lives in a (lo, hi) pair of 32-bit registers. rc_for_type returns two Int registers for I64, and the ABI passes 64-bit arguments in even-aligned pairs (r0:r1 / r2:r3) per AAPCS, spilling to the 8-aligned stack when they run out. Lowers iconst, add/sub via the adds/adc and subs/sbc carry chains, the bitwise ops per half, iconcat/isplit, the extends and reduce between i32 and i64, and 64-bit load/store as two word accesses. The i64 rules take an explicit priority so ISLE doesn't have to prove them disjoint from the fits_in_32 rules. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/abi.rs | 13 ++ cranelift/codegen/src/isa/arm32/inst.isle | 48 ++++++++ cranelift/codegen/src/isa/arm32/inst/mod.rs | 6 +- cranelift/codegen/src/isa/arm32/lower.isle | 46 ++++++++ cranelift/codegen/src/isa/arm32/lower/isle.rs | 10 ++ .../filetests/filetests/isa/arm32/i64.clif | 111 ++++++++++++++++++ 6 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/arm32/i64.clif diff --git a/cranelift/codegen/src/isa/arm32/abi.rs b/cranelift/codegen/src/isa/arm32/abi.rs index 70c60c274849..478ce86271cd 100644 --- a/cranelift/codegen/src/isa/arm32/abi.rs +++ b/cranelift/codegen/src/isa/arm32/abi.rs @@ -76,6 +76,19 @@ impl ABIMachineSpec for Arm32MachineDeps { } let (rcs, reg_tys) = Inst::rc_for_type(param.value_type)?; + + // AAPCS: a 64-bit value passed in registers must start in an + // even-numbered register (an r0:r1 / r2:r3 pair). If the aligned + // pair would not fully fit, it goes entirely on the (8-aligned) + // stack, so bump `next_x_reg` past the argument registers. + if rcs.len() == 2 { + next_x_reg = align_to(u32::from(next_x_reg), 2) as u8; + if next_x_reg + 1 > x_end { + next_x_reg = x_end + 1; + next_stack = align_to(next_stack, 8); + } + } + let mut slots = ABIArgSlotVec::new(); for (rc, reg_ty) in rcs.iter().zip(reg_tys.iter()) { assert_eq!(*rc, RegClass::Int, "arm32 only supports integer values"); diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index f35eb808b21b..c45cc767d8a5 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -352,6 +352,13 @@ (_ Unit (emit (MInst.AluRRImm op rd rn imm12)))) rd)) +;; `op{s} rd, rn, rm` — flag-setting form (adds/subs/adcs/sbcs). +(decl alu_rrr_flags (ALUOp Reg Reg) Reg) +(rule (alu_rrr_flags op rn rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AluRRRFlags op rd rn rm)))) + rd)) + ;; A constant is encodable as an A32 data-processing immediate; the result is ;; its pre-encoded 12-bit form. (decl pure partial u64_from_rotated_imm12 (u64) u32) @@ -483,6 +490,40 @@ (_ Unit (emit (MInst.ShiftReg op rd rm masked)))) rd)) +;;;; 64-bit helpers (values held as a (lo, hi) register pair) ;;;;;;;;;;;;;;;;;; + +(decl vr_lo (ValueRegs) Reg) +(rule (vr_lo regs) (value_regs_get regs 0)) +(decl vr_hi (ValueRegs) Reg) +(rule (vr_hi regs) (value_regs_get regs 1)) + +;; The high 32 bits of a 64-bit constant. +(decl pure u64_high32 (u64) u64) +(extern constructor u64_high32 u64_high32) + +;; `adds`/`adc` and `subs`/`sbc` carry chains over the halves. +(decl add_i64 (ValueRegs ValueRegs) ValueRegs) +(rule (add_i64 x y) + (let ((lo Reg (alu_rrr_flags (ALUOp.Add) (vr_lo x) (vr_lo y))) + (hi Reg (alu_rrr (ALUOp.Adc) (vr_hi x) (vr_hi y)))) + (value_regs lo hi))) + +(decl sub_i64 (ValueRegs ValueRegs) ValueRegs) +(rule (sub_i64 x y) + (let ((lo Reg (alu_rrr_flags (ALUOp.Sub) (vr_lo x) (vr_lo y))) + (hi Reg (alu_rrr (ALUOp.Sbc) (vr_hi x) (vr_hi y)))) + (value_regs lo hi))) + +;; A bitwise op applied independently to each half. +(decl logical_i64 (ALUOp ValueRegs ValueRegs) ValueRegs) +(rule (logical_i64 op x y) + (value_regs (alu_rrr op (vr_lo x) (vr_lo y)) + (alu_rrr op (vr_hi x) (vr_hi y)))) + +(decl not_i64 (ValueRegs) ValueRegs) +(rule (not_i64 x) + (value_regs (mvn_reg (vr_lo x)) (mvn_reg (vr_hi x)))) + ;;;; Loads and stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl arm_load (AMode LoadKind) Reg) @@ -499,6 +540,13 @@ (decl amode (Value Offset32) AMode) (rule (amode base offset) (AMode.RegOffset base (offset32_to_i32 offset))) +;; Addressing mode for the high word of a 64-bit access (offset + 4). +(decl amode_hi (Value Offset32) AMode) +(rule (amode_hi base offset) (AMode.RegOffset base (offset32_plus4 offset))) + +(decl pure offset32_plus4 (Offset32) i32) +(extern constructor offset32_plus4 offset32_plus4) + ;;;; Comparisons and branches ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Emit a compare (`cmp rn, rm`) as a side effect. diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index 459884f3d773..9cbce86cdf53 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -1,7 +1,7 @@ //! This module defines arm32 (AArch32 / A32) machine instruction types. use crate::binemit::{Addend, CodeOffset, Reloc}; -use crate::ir::types::{I8, I16, I32}; +use crate::ir::types::{I8, I16, I32, I64}; use crate::ir::{Type, types}; use crate::isa::FunctionAlignment; use crate::isa::arm32::abi::Arm32MachineDeps; @@ -354,8 +354,10 @@ impl MachInst for Inst { I8 => Ok((&[RegClass::Int], &[I8])), I16 => Ok((&[RegClass::Int], &[I16])), I32 => Ok((&[RegClass::Int], &[I32])), + // 64-bit values are held in a pair of 32-bit integer registers. + I64 => Ok((&[RegClass::Int, RegClass::Int], &[I32, I32])), _ => Err(CodegenError::Unsupported(alloc::format!( - "Unsupported type on arm32 (only i8/i16/i32 are implemented so far): {ty}" + "Unsupported type on arm32 (only i8/i16/i32/i64 are implemented so far): {ty}" ))), } } diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index 0fc9cf1721c5..99acd6cda939 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -8,6 +8,52 @@ (rule (lower (iconst (fits_in_32 ty) (u64_from_imm64 n))) (imm ty n)) +;;;; 64-bit integer operations (register pairs) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; These use an explicit priority so they don't overlap the `fits_in_32` rules +;; (ISLE can't prove `$I64` and `fits_in_32` are disjoint on its own). + +(rule 4 (lower (iconst $I64 (u64_from_imm64 n))) + (value_regs (gen_constant n) (gen_constant (u64_high32 n)))) + +(rule 4 (lower (iadd $I64 x y)) + (add_i64 (put_in_regs x) (put_in_regs y))) +(rule 4 (lower (isub $I64 x y)) + (sub_i64 (put_in_regs x) (put_in_regs y))) + +(rule 4 (lower (band $I64 x y)) + (logical_i64 (ALUOp.And) (put_in_regs x) (put_in_regs y))) +(rule 4 (lower (bor $I64 x y)) + (logical_i64 (ALUOp.Orr) (put_in_regs x) (put_in_regs y))) +(rule 4 (lower (bxor $I64 x y)) + (logical_i64 (ALUOp.Eor) (put_in_regs x) (put_in_regs y))) +(rule 4 (lower (bnot $I64 x)) + (not_i64 (put_in_regs x))) + +;; `iconcat(lo, hi)` builds the pair; `isplit` takes it apart. +(rule 4 (lower (iconcat $I64 lo hi)) + (value_regs (put_in_reg lo) (put_in_reg hi))) +(rule 4 (lower (isplit $I64 x)) + (let ((regs ValueRegs (put_in_regs x))) + (output_pair (value_reg (vr_lo regs)) (value_reg (vr_hi regs))))) + +;; Widen/narrow between i32 and i64. +(rule 4 (lower (uextend $I64 x @ (value_type $I32))) + (value_regs (put_in_reg x) (gen_constant 0))) +(rule 4 (lower (sextend $I64 x @ (value_type $I32))) + (let ((lo Reg (put_in_reg x))) + (value_regs lo (gen_shift_imm (ShiftOp.Asr) lo 31)))) +(rule 4 (lower (ireduce $I32 x @ (value_type $I64))) + (vr_lo (put_in_regs x))) + +;; 64-bit loads/stores as two 32-bit accesses. +(rule 4 (lower (load $I64 flags addr offset)) + (value_regs (arm_load (amode addr offset) (LoadKind.Word)) + (arm_load (amode_hi addr offset) (LoadKind.Word)))) +(rule 4 (lower (store flags val @ (value_type $I64) addr offset)) + (let ((regs ValueRegs (put_in_regs val)) + (_ InstOutput (arm_store (vr_lo regs) (amode addr offset) (StoreKind.Word)))) + (arm_store (vr_hi regs) (amode_hi addr offset) (StoreKind.Word)))) + ;;;; `iadd` / `isub` / `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule 2 (lower (iadd (fits_in_32 _) x (iconst _ (u64_from_imm64 n)))) diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs index 936c258dfa1c..8b99839388db 100644 --- a/cranelift/codegen/src/isa/arm32/lower/isle.rs +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -121,6 +121,16 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { encode_rotated_imm(val as u32) } + /// The high 32 bits of a 64-bit constant. + fn u64_high32(&mut self, val: u64) -> u64 { + val >> 32 + } + + /// The offset of the high word of a 64-bit memory access. + fn offset32_plus4(&mut self, offset: Offset32) -> i32 { + i32::from(offset) + 4 + } + /// Shift/rotate by a constant, applying Cranelift's modulo-width semantics. /// A masked amount of zero becomes a plain register move. fn gen_shift_imm(&mut self, op: &ShiftOp, rm: Reg, amount: u64) -> Reg { diff --git a/cranelift/filetests/filetests/isa/arm32/i64.clif b/cranelift/filetests/filetests/isa/arm32/i64.clif new file mode 100644 index 000000000000..e57c32ddd838 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/i64.clif @@ -0,0 +1,111 @@ +test compile precise-output +target arm + +function %add64(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = iadd v0, v1 + return v2 +} + +; VCode: +; block0: +; adds r0, r0, r2 +; adc r1, r1, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; adds r0, r0, r2 +; adc r1, r1, r3 +; bx lr + +function %sub64(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = isub v0, v1 + return v2 +} + +; VCode: +; block0: +; subs r0, r0, r2 +; sbc r1, r1, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; subs r0, r0, r2 +; sbc r1, r1, r3 +; bx lr + +function %const64() -> i64 { +block0: + v0 = iconst.i64 0x1234_5678_9abc_def0 + return v0 +} + +; VCode: +; block0: +; movw r0, #57072; movt r0, #39612 +; movw r1, #22136; movt r1, #4660 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; movw r0, #0xdef0 +; movt r0, #0x9abc +; movw r1, #0x5678 +; movt r1, #0x1234 +; bx lr + +function %uext(i32) -> i64 { +block0(v0: i32): + v1 = uextend.i64 v0 + return v1 +} + +; VCode: +; block0: +; mov r1, #0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mov r1, #0 +; bx lr + +function %sext(i32) -> i64 { +block0(v0: i32): + v1 = sextend.i64 v0 + return v1 +} + +; VCode: +; block0: +; asr r1, r0, #31 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; asr r1, r0, #0x1f +; bx lr + +function %load64(i32) -> i64 { +block0(v0: i32): + v1 = load.i64 v0 + return v1 +} + +; VCode: +; block0: +; ldr r2, [r0, #0] +; ldr r1, [r0, #4] +; mov r0, r2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldr r2, [r0] +; ldr r1, [r0, #4] +; mov r0, r2 +; bx lr + From f8ff46ec4205f28edb9e1f80c9191a9634e76b02 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Fri, 3 Jul 2026 22:07:01 +0300 Subject: [PATCH 08/18] arm32: lower i64 multiply and constant-amount shifts imul.i64 expands to umull plus two mla for the low 64 bits of the product. Constant-amount ishl/ushr/sshr use a funnel sequence: for shifts below 32 an orr of two 32-bit shifts, and for shifts of 32 or more a single shift with a zero or sign fill. Variable-amount i64 shifts are still unimplemented. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/inst.isle | 26 +++++ cranelift/codegen/src/isa/arm32/lower.isle | 11 ++ cranelift/codegen/src/isa/arm32/lower/isle.rs | 86 ++++++++++++++ .../filetests/filetests/isa/arm32/i64.clif | 109 ++++++++++++++++++ 4 files changed, 232 insertions(+) diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index c45cc767d8a5..4cebe1983349 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -524,6 +524,32 @@ (rule (not_i64 x) (value_regs (mvn_reg (vr_lo x)) (mvn_reg (vr_hi x)))) +;; `mla rd, rn, rm, ra` — `rd = rn * rm + ra`. +(decl mla (Reg Reg Reg) Reg) +(rule (mla rn rm ra) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.Mla rd rn rm ra)))) + rd)) + +;; 64-bit multiply (low 64 bits): umull for the lo*lo product, then fold in the +;; two cross terms with mla. +(decl mul_i64 (ValueRegs ValueRegs) ValueRegs) +(rule (mul_i64 x y) + (let ((xlo Reg (vr_lo x)) + (xhi Reg (vr_hi x)) + (ylo Reg (vr_lo y)) + (yhi Reg (vr_hi y)) + (lo WritableReg (temp_writable_reg $I32)) + (hi0 WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.Umull lo hi0 xlo ylo))) + (hi1 Reg (mla xlo yhi (writable_reg_to_reg hi0))) + (hi2 Reg (mla xhi ylo hi1))) + (value_regs (writable_reg_to_reg lo) hi2))) + +;; Constant shift/rotate of a 64-bit value; the funnel logic is in Rust. +(decl gen_i64_shift_imm (ShiftOp ValueRegs u64) ValueRegs) +(extern constructor gen_i64_shift_imm gen_i64_shift_imm) + ;;;; Loads and stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl arm_load (AMode LoadKind) Reg) diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index 99acd6cda939..fc59a0e2265e 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -45,6 +45,17 @@ (rule 4 (lower (ireduce $I32 x @ (value_type $I64))) (vr_lo (put_in_regs x))) +(rule 4 (lower (imul $I64 x y)) + (mul_i64 (put_in_regs x) (put_in_regs y))) + +;; 64-bit shifts by a constant amount. +(rule 4 (lower (ishl $I64 x (iconst _ (u64_from_imm64 n)))) + (gen_i64_shift_imm (ShiftOp.Lsl) (put_in_regs x) n)) +(rule 4 (lower (ushr $I64 x (iconst _ (u64_from_imm64 n)))) + (gen_i64_shift_imm (ShiftOp.Lsr) (put_in_regs x) n)) +(rule 4 (lower (sshr $I64 x (iconst _ (u64_from_imm64 n)))) + (gen_i64_shift_imm (ShiftOp.Asr) (put_in_regs x) n)) + ;; 64-bit loads/stores as two 32-bit accesses. (rule 4 (lower (load $I64 flags addr offset)) (value_regs (arm_load (amode addr offset) (LoadKind.Word)) diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs index 8b99839388db..4c9fedbdf881 100644 --- a/cranelift/codegen/src/isa/arm32/lower/isle.rs +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -45,6 +45,25 @@ impl<'a, 'b> Arm32IsleContext<'a, 'b, MInst, Arm32Backend> { pub(crate) fn dfg(&self) -> &crate::ir::DataFlowGraph { &self.lower_ctx.f.dfg } + + /// Emit a single 32-bit shift `op rd, rm, #amount` (1 <= amount <= 31). + fn shift_raw(&mut self, op: ShiftOp, rm: Reg, amount: u8) -> Reg { + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::ShiftImm { op, rd, rm, amount }); + rd.to_reg() + } + + /// Emit `orr rd, a, b`. + fn orr_raw(&mut self, a: Reg, b: Reg) -> Reg { + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::AluRRR { + op: crate::isa::arm32::inst::ALUOp::Orr, + rd, + rn: a, + rm: b, + }); + rd.to_reg() + } } impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { @@ -131,6 +150,73 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { i32::from(offset) + 4 } + /// Shift/rotate a 64-bit `(lo, hi)` pair by a constant amount, emitting the + /// funnel-shift sequence. Only Lsl/Lsr/Asr are supported. + fn gen_i64_shift_imm(&mut self, op: &ShiftOp, x: ValueRegs, amount: u64) -> ValueRegs { + let n = (amount & 63) as u8; + let xlo = x.regs()[0]; + let xhi = x.regs()[1]; + if n == 0 { + return ValueRegs::two(xlo, xhi); + } + match *op { + ShiftOp::Lsl => { + if n < 32 { + let lo = self.shift_raw(ShiftOp::Lsl, xlo, n); + let hi_hi = self.shift_raw(ShiftOp::Lsl, xhi, n); + let hi_lo = self.shift_raw(ShiftOp::Lsr, xlo, 32 - n); + let hi = self.orr_raw(hi_hi, hi_lo); + ValueRegs::two(lo, hi) + } else { + let zero = self.gen_constant(0); + let hi = if n == 32 { + xlo + } else { + self.shift_raw(ShiftOp::Lsl, xlo, n - 32) + }; + ValueRegs::two(zero, hi) + } + } + ShiftOp::Lsr => { + if n < 32 { + let hi = self.shift_raw(ShiftOp::Lsr, xhi, n); + let lo_lo = self.shift_raw(ShiftOp::Lsr, xlo, n); + let lo_hi = self.shift_raw(ShiftOp::Lsl, xhi, 32 - n); + let lo = self.orr_raw(lo_lo, lo_hi); + ValueRegs::two(lo, hi) + } else { + let zero = self.gen_constant(0); + let lo = if n == 32 { + xhi + } else { + self.shift_raw(ShiftOp::Lsr, xhi, n - 32) + }; + ValueRegs::two(lo, zero) + } + } + ShiftOp::Asr => { + if n < 32 { + let hi = self.shift_raw(ShiftOp::Asr, xhi, n); + let lo_lo = self.shift_raw(ShiftOp::Lsr, xlo, n); + let lo_hi = self.shift_raw(ShiftOp::Lsl, xhi, 32 - n); + let lo = self.orr_raw(lo_lo, lo_hi); + ValueRegs::two(lo, hi) + } else { + // The high word becomes all sign bits; the low word is the + // old high word arithmetically shifted by `n - 32`. + let hi = self.shift_raw(ShiftOp::Asr, xhi, 31); + let lo = if n == 32 { + xhi + } else { + self.shift_raw(ShiftOp::Asr, xhi, n - 32) + }; + ValueRegs::two(lo, hi) + } + } + ShiftOp::Ror => unimplemented!("arm32: 64-bit rotate is not implemented"), + } + } + /// Shift/rotate by a constant, applying Cranelift's modulo-width semantics. /// A masked amount of zero becomes a plain register move. fn gen_shift_imm(&mut self, op: &ShiftOp, rm: Reg, amount: u64) -> Reg { diff --git a/cranelift/filetests/filetests/isa/arm32/i64.clif b/cranelift/filetests/filetests/isa/arm32/i64.clif index e57c32ddd838..b29d30f4c499 100644 --- a/cranelift/filetests/filetests/isa/arm32/i64.clif +++ b/cranelift/filetests/filetests/isa/arm32/i64.clif @@ -109,3 +109,112 @@ block0(v0: i32): ; mov r0, r2 ; bx lr + +function %mul64(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = imul v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r5, [sp, #0] +; str r6, [sp, #4] +; block0: +; umull r5, r6, r0, r2 +; mla r0, r0, r3, r6 +; mla r1, r1, r2, r0 +; mov r0, r5 +; ldr r5, [sp, #0] +; ldr r6, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r5, [sp] +; str r6, [sp, #4] +; block1: ; offset 0x14 +; umull r5, r6, r0, r2 +; mla r0, r0, r3, r6 +; mla r1, r1, r2, r0 +; mov r0, r5 +; ldr r5, [sp] +; ldr r6, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +function %shl64(i64) -> i64 { +block0(v0: i64): + v1 = iconst.i32 5 + v2 = ishl v0, v1 + return v2 +} + +; VCode: +; block0: +; lsl r3, r0, #5 +; lsl r1, r1, #5 +; lsr r0, r0, #27 +; orr r1, r1, r0 +; mov r0, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; lsl r3, r0, #5 +; lsl r1, r1, #5 +; lsr r0, r0, #0x1b +; orr r1, r1, r0 +; mov r0, r3 +; bx lr + +function %shl64_big(i64) -> i64 { +block0(v0: i64): + v1 = iconst.i32 40 + v2 = ishl v0, v1 + return v2 +} + +; VCode: +; block0: +; mov r1, r0 +; mov r0, #0 +; lsl r1, r1, #8 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mov r1, r0 +; mov r0, #0 +; lsl r1, r1, #8 +; bx lr + +function %sshr64(i64) -> i64 { +block0(v0: i64): + v1 = iconst.i32 33 + v2 = sshr v0, v1 + return v2 +} + +; VCode: +; block0: +; asr r3, r1, #31 +; asr r0, r1, #1 +; mov r1, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; asr r3, r1, #0x1f +; asr r0, r1, #1 +; mov r1, r3 +; bx lr + From b9b3b0623f933af2e244928c5530db2f9740ffd1 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Sat, 4 Jul 2026 06:05:17 +0300 Subject: [PATCH 09/18] arm32: lower i64 comparisons icmp on i64 operands, both as a value and fused into brif. eq/ne reduce the halves with eor/eor/orrs; the ordering comparisons subtract the pair with subs/sbcs and test a Z-independent condition, swapping operands for the gt/le/hi/ls cases. Also adds a standalone 32-bit icmp-to-bool lowering, which until now only existed fused into brif or select. Note that icmp's controlling type is its result type, so the rules match the operand width with value_type. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/inst.isle | 10 ++ cranelift/codegen/src/isa/arm32/lower.isle | 22 +++- cranelift/codegen/src/isa/arm32/lower/isle.rs | 63 ++++++++-- .../filetests/isa/arm32/i64-cmp.clif | 116 ++++++++++++++++++ 4 files changed, 201 insertions(+), 10 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/arm32/i64-cmp.clif diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index 4cebe1983349..d4397ce8f0ca 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -614,3 +614,13 @@ ;; Map an `IntCC` to the corresponding ARM condition code. (decl cond_from_intcc (IntCC) Cond) (extern constructor cond_from_intcc cond_from_intcc) + +;; Emit the compare sequence for a 64-bit `icmp` and return the ARM condition +;; that then tests it (the compare itself is emitted as a side effect). +(decl lower_icmp_i64 (IntCC ValueRegs ValueRegs) Cond) +(extern constructor lower_icmp_i64 lower_icmp_i64) + +;; Materialize a boolean 0/1 from the current flags and a condition +;; (`mov rd, #0; mov rd, #1`). +(decl csetv (Cond) Reg) +(rule (csetv cond) (csel cond (gen_constant 1) (gen_constant 0))) diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index fc59a0e2265e..bb2ab2ab46f4 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -167,10 +167,21 @@ (rule (lower (ireduce (fits_in_32 _) x)) (put_in_reg x)) +;;;; `icmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; 32-bit comparison producing a 0/1 boolean. +(rule (lower (icmp _ cc a @ (value_type (fits_in_32 _)) b)) + (let ((_ Unit (emit_side_effect (cmp_rr (put_in_reg a) (put_in_reg b))))) + (csetv (cond_from_intcc cc)))) + +;; 64-bit comparison producing a 0/1 boolean. +(rule 4 (lower (icmp _ cc a @ (value_type $I64) b)) + (csetv (lower_icmp_i64 cc (put_in_regs a) (put_in_regs b)))) + ;;;; `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Fuse an `icmp` condition into the compare that precedes the select. -(rule 1 (lower (select (fits_in_32 _) (icmp _ cc x y) a b)) +(rule 1 (lower (select (fits_in_32 _) (icmp _ cc x @ (value_type (fits_in_32 _)) y) a b)) (let ((_ Unit (emit_side_effect (cmp_rr (put_in_reg x) (put_in_reg y))))) (csel (cond_from_intcc cc) (put_in_reg a) (put_in_reg b)))) @@ -260,13 +271,18 @@ (rule (lower_branch (jump _) (single_target label)) (emit_side_effect (jump_impl label))) -;; Fuse an `icmp` feeding a `brif` into a compare plus a conditional branch. -(rule 1 (lower_branch (brif (icmp _ cc a b) _ _) (two_targets taken not_taken)) +;; Fuse a 32-bit `icmp` feeding a `brif` into a compare plus conditional branch. +(rule 1 (lower_branch (brif (icmp _ cc a @ (value_type (fits_in_32 _)) b) _ _) (two_targets taken not_taken)) (emit_side_effect (side_effect_concat (cmp_rr (put_in_reg a) (put_in_reg b)) (cond_br (cond_from_intcc cc) taken not_taken)))) +;; Fuse a 64-bit `icmp` feeding a `brif`. +(rule 2 (lower_branch (brif (icmp _ cc a @ (value_type $I64) b) _ _) (two_targets taken not_taken)) + (emit_side_effect + (cond_br (lower_icmp_i64 cc (put_in_regs a) (put_in_regs b)) taken not_taken))) + ;; Generic `brif`: branch when the value is non-zero. (rule (lower_branch (brif v _ _) (two_targets taken not_taken)) (emit_side_effect diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs index 4c9fedbdf881..1cc10654bf74 100644 --- a/cranelift/codegen/src/isa/arm32/lower/isle.rs +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -12,7 +12,7 @@ use crate::ir::{ BlockCall, ExternalName, Inst, InstructionData, MemFlags, Opcode, TrapCode, Value, ValueList, }; use crate::isa::arm32::Arm32Backend; -use crate::isa::arm32::inst::{Cond, ShiftOp, encode_rotated_imm}; +use crate::isa::arm32::inst::{ALUOp, Cond, ShiftOp, encode_rotated_imm}; use crate::machinst::isle::*; use crate::machinst::{ ArgPair, CallArgList, CallInfo, CallRetList, InstOutput, Lower, MachInst, MachLabel, RetPair, @@ -55,15 +55,21 @@ impl<'a, 'b> Arm32IsleContext<'a, 'b, MInst, Arm32Backend> { /// Emit `orr rd, a, b`. fn orr_raw(&mut self, a: Reg, b: Reg) -> Reg { + self.alu_raw(ALUOp::Orr, false, a, b) + } + + /// Emit `op{s} rd, rn, rm` into a fresh register (which is returned). + fn alu_raw(&mut self, op: ALUOp, set_flags: bool, rn: Reg, rm: Reg) -> Reg { let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); - self.lower_ctx.emit(MInst::AluRRR { - op: crate::isa::arm32::inst::ALUOp::Orr, - rd, - rn: a, - rm: b, - }); + let inst = if set_flags { + MInst::AluRRRFlags { op, rd, rn, rm } + } else { + MInst::AluRRR { op, rd, rn, rm } + }; + self.lower_ctx.emit(inst); rd.to_reg() } + } impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { @@ -241,6 +247,49 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { self.backend.isa_flags.has_idiv() } + /// Emit the compare sequence for a 64-bit `icmp` and return the ARM + /// condition code that tests its result. + /// + /// A 64-bit `subs`/`sbcs` sets N, V and C correctly for the full value but + /// its Z flag only reflects the high word, so it's only usable for the + /// conditions that don't depend on Z (lt/ge/lo/hs). The gt/le/hi/ls forms + /// are obtained by swapping the operands, and eq/ne use a separate + /// `eor`/`eor`/`orrs` reduction. + fn lower_icmp_i64(&mut self, cc: &IntCC, a: ValueRegs, b: ValueRegs) -> Cond { + let (alo, ahi) = (a.regs()[0], a.regs()[1]); + let (blo, bhi) = (b.regs()[0], b.regs()[1]); + match cc { + IntCC::Equal | IntCC::NotEqual => { + let tlo = self.alu_raw(ALUOp::Eor, false, alo, blo); + let thi = self.alu_raw(ALUOp::Eor, false, ahi, bhi); + // `orrs` sets Z iff both halves are equal, i.e. a == b. + let _ = self.alu_raw(ALUOp::Orr, true, tlo, thi); + if matches!(cc, IntCC::Equal) { + Cond::Eq + } else { + Cond::Ne + } + } + _ => { + // (lo1, hi1) - (lo2, hi2), then test `cond`. + let (lo1, hi1, lo2, hi2, cond) = match cc { + IntCC::SignedLessThan => (alo, ahi, blo, bhi, Cond::Lt), + IntCC::SignedGreaterThanOrEqual => (alo, ahi, blo, bhi, Cond::Ge), + IntCC::SignedGreaterThan => (blo, bhi, alo, ahi, Cond::Lt), + IntCC::SignedLessThanOrEqual => (blo, bhi, alo, ahi, Cond::Ge), + IntCC::UnsignedLessThan => (alo, ahi, blo, bhi, Cond::Lo), + IntCC::UnsignedGreaterThanOrEqual => (alo, ahi, blo, bhi, Cond::Hs), + IntCC::UnsignedGreaterThan => (blo, bhi, alo, ahi, Cond::Lo), + IntCC::UnsignedLessThanOrEqual => (blo, bhi, alo, ahi, Cond::Hs), + _ => unreachable!(), + }; + let _ = self.alu_raw(ALUOp::Sub, true, lo1, lo2); // subs + let _ = self.alu_raw(ALUOp::Sbc, true, hi1, hi2); // sbcs + cond + } + } + } + fn cond_from_intcc(&mut self, cc: &IntCC) -> Cond { match cc { IntCC::Equal => Cond::Eq, diff --git a/cranelift/filetests/filetests/isa/arm32/i64-cmp.clif b/cranelift/filetests/filetests/isa/arm32/i64-cmp.clif new file mode 100644 index 000000000000..33e92b2dedc0 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/i64-cmp.clif @@ -0,0 +1,116 @@ +test compile precise-output +target arm + +function %eq(i64, i64) -> i8 { +block0(v0: i64, v1: i64): + v2 = icmp eq v0, v1 + return v2 +} + +; VCode: +; block0: +; eor r0, r0, r2 +; eor r1, r1, r3 +; orrs r0, r0, r1 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; moveq r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; eor r0, r0, r2 +; eor r1, r1, r3 +; orrs r0, r0, r1 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; moveq r0, r1 +; bx lr + +function %slt(i64, i64) -> i8 { +block0(v0: i64, v1: i64): + v2 = icmp slt v0, v1 + return v2 +} + +; VCode: +; block0: +; subs r0, r0, r2 +; sbcs r0, r1, r3 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; movlt r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; subs r0, r0, r2 +; sbcs r0, r1, r3 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; movlt r0, r1 +; bx lr + +function %ugt(i64, i64) -> i8 { +block0(v0: i64, v1: i64): + v2 = icmp ugt v0, v1 + return v2 +} + +; VCode: +; block0: +; subs r0, r2, r0 +; sbcs r0, r3, r1 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; movlo r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; subs r0, r2, r0 +; sbcs r0, r3, r1 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; movlo r0, r1 +; bx lr + +function %br_sle(i64, i64) -> i32 { +block0(v0: i64, v1: i64): + v2 = icmp sle v0, v1 + brif v2, block1, block2 +block1: + v3 = iconst.i32 1 + return v3 +block2: + v4 = iconst.i32 0 + return v4 +} + +; VCode: +; block0: +; subs r0, r2, r0 +; sbcs r0, r3, r1 +; bge label2; b label1 +; block1: +; mov r0, #0 +; bx lr +; block2: +; mov r0, #1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; subs r0, r2, r0 +; sbcs r0, r3, r1 +; bge #0x14 +; block1: ; offset 0xc +; mov r0, #0 +; bx lr +; block2: ; offset 0x14 +; mov r0, #1 +; bx lr + From d9ec67ced8c96bdd8c6b77b06664fb6acf010fc0 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Sat, 4 Jul 2026 08:41:16 +0300 Subject: [PATCH 10/18] arm32: add VFP floating point Introduces f32/f64 on the VFP unit. regalloc2 can't model the S/D sub-register aliasing, so the float class is the D registers and an f32 lives in the low single half of its D register -- every value owns a whole D register, which also keeps the encodings uniform (F32 vs F64 is just the size bit). D0-D7 are caller-saved, D8-D15 callee-saved; moves and spills are class-aware (vmov.f64, vldr/vstr). The ABI is a simplified hard-float (args in D0-D7, return in D0). Lowers the arithmetic, negate/abs/sqrt, loads and stores, and the const materialization through GPRs. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/abi.rs | 154 +++++++++++++----- cranelift/codegen/src/isa/arm32/inst.isle | 80 +++++++++ cranelift/codegen/src/isa/arm32/inst/args.rs | 61 ++++++- cranelift/codegen/src/isa/arm32/inst/emit.rs | 90 ++++++++++ cranelift/codegen/src/isa/arm32/inst/mod.rs | 87 +++++++++- cranelift/codegen/src/isa/arm32/inst/regs.rs | 60 +++++-- cranelift/codegen/src/isa/arm32/lower.isle | 48 ++++++ cranelift/codegen/src/isa/arm32/lower/isle.rs | 5 + .../filetests/filetests/isa/arm32/float.clif | 129 +++++++++++++++ 9 files changed, 649 insertions(+), 65 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/arm32/float.clif diff --git a/cranelift/codegen/src/isa/arm32/abi.rs b/cranelift/codegen/src/isa/arm32/abi.rs index 478ce86271cd..2aa7248a8c24 100644 --- a/cranelift/codegen/src/isa/arm32/abi.rs +++ b/cranelift/codegen/src/isa/arm32/abi.rs @@ -49,11 +49,15 @@ impl ABIMachineSpec for Arm32MachineDeps { ) -> CodegenResult<(u32, Option)> { // Integer arguments/returns go in r0-r3 (r0-r1 for returns), the rest on // the stack. This is a simplified AAPCS. - let (x_start, x_end) = match args_or_rets { - ArgsOrRets::Args => (0u8, 3u8), - ArgsOrRets::Rets => (0u8, 1u8), + let (x_start, x_end, d_end) = match args_or_rets { + ArgsOrRets::Args => (0u8, 3u8, 7u8), + ArgsOrRets::Rets => (0u8, 1u8, 1u8), }; let mut next_x_reg = x_start; + // VFP (hard-float) register counter: each float value takes one D reg + // (an f32 uses the low S-half). This is a simplified AAPCS-VFP that does + // not back-fill S registers. + let mut next_d_reg = 0u8; let mut next_stack: u32 = 0; let ret_area_ptr = if add_ret_area_ptr { @@ -91,14 +95,26 @@ impl ABIMachineSpec for Arm32MachineDeps { let mut slots = ABIArgSlotVec::new(); for (rc, reg_ty) in rcs.iter().zip(reg_tys.iter()) { - assert_eq!(*rc, RegClass::Int, "arm32 only supports integer values"); - if next_x_reg <= x_end { + let reg = match rc { + RegClass::Float if next_d_reg <= d_end => { + let r = dreg(next_d_reg); + next_d_reg += 1; + Some(r) + } + RegClass::Int if next_x_reg <= x_end => { + let r = xreg(next_x_reg); + next_x_reg += 1; + Some(r) + } + RegClass::Int | RegClass::Float => None, + RegClass::Vector => unreachable!("arm32 has no vector registers"), + }; + if let Some(reg) = reg { slots.push(ABIArgSlot::Reg { - reg: xreg(next_x_reg).to_real_reg().unwrap(), + reg: reg.to_real_reg().unwrap(), ty: *reg_ty, extension: param.extension, }); - next_x_reg += 1; } else { if args_or_rets == ArgsOrRets::Rets && !flags.enable_multi_ret_implicit_sret() { return Err(crate::CodegenError::Unsupported( @@ -133,18 +149,34 @@ impl ABIMachineSpec for Arm32MachineDeps { } fn gen_load_stack(mem: StackAMode, into_reg: Writable, _ty: Type) -> Inst { - Inst::Load { - rt: into_reg, - mem: amode_from_stack(mem), - kind: LoadKind::Word, + if into_reg.to_reg().class() == RegClass::Float { + Inst::FpuLoad { + size: FpuSize::F64, + rd: into_reg, + mem: amode_from_stack(mem), + } + } else { + Inst::Load { + rt: into_reg, + mem: amode_from_stack(mem), + kind: LoadKind::Word, + } } } fn gen_store_stack(mem: StackAMode, from_reg: Reg, _ty: Type) -> Inst { - Inst::Store { - rt: from_reg, - mem: amode_from_stack(mem), - kind: StoreKind::Word, + if from_reg.class() == RegClass::Float { + Inst::FpuStore { + size: FpuSize::F64, + rt: from_reg, + mem: amode_from_stack(mem), + } + } else { + Inst::Store { + rt: from_reg, + mem: amode_from_stack(mem), + kind: StoreKind::Word, + } } } @@ -292,14 +324,25 @@ impl ABIMachineSpec for Arm32MachineDeps { insts.extend(Self::gen_sp_reg_adjust(-(stack_size as i32))); let mut cur_offset = 0i32; for reg in &frame_layout.clobbered_callee_saves { - insts.push(Inst::Store { - rt: Reg::from(reg.to_reg()), - mem: AMode::SPOffset { - offset: i64::from(cur_offset), - }, - kind: StoreKind::Word, - }); - cur_offset += 4; + let r = Reg::from(reg.to_reg()); + let mem = AMode::SPOffset { + offset: i64::from(cur_offset), + }; + if r.class() == RegClass::Float { + insts.push(Inst::FpuStore { + size: FpuSize::F64, + rt: r, + mem, + }); + cur_offset += 8; + } else { + insts.push(Inst::Store { + rt: r, + mem, + kind: StoreKind::Word, + }); + cur_offset += 4; + } } } insts @@ -316,14 +359,24 @@ impl ABIMachineSpec for Arm32MachineDeps { + frame_layout.outgoing_args_size; let mut cur_offset = 0i32; for reg in &frame_layout.clobbered_callee_saves { - insts.push(Inst::Load { - rt: reg.map(Reg::from), - mem: AMode::SPOffset { - offset: i64::from(cur_offset), - }, - kind: LoadKind::Word, - }); - cur_offset += 4; + let mem = AMode::SPOffset { + offset: i64::from(cur_offset), + }; + if reg.to_reg().class() == RegClass::Float { + insts.push(Inst::FpuLoad { + size: FpuSize::F64, + rd: reg.map(Reg::from), + mem, + }); + cur_offset += 8; + } else { + insts.push(Inst::Load { + rt: reg.map(Reg::from), + mem, + kind: LoadKind::Word, + }); + cur_offset += 4; + } } if stack_size > 0 { insts.extend(Self::gen_sp_reg_adjust(stack_size as i32)); @@ -348,9 +401,9 @@ impl ABIMachineSpec for Arm32MachineDeps { ) -> u32 { match rc { RegClass::Int => 1, - RegClass::Float | RegClass::Vector => { - unimplemented!("arm32: no float/vector registers yet") - } + // A D register is 8 bytes = two 4-byte spill slots. + RegClass::Float => 2, + RegClass::Vector => unimplemented!("arm32 has no vector registers"), } } @@ -438,7 +491,7 @@ fn amode_from_stack(mem: StackAMode) -> AMode { } } -/// Callee-saved GPRs under AAPCS: r4-r11. +/// Callee-saved registers under AAPCS: GPRs r4-r11 and VFP D8-D15. const DEFAULT_CALLEE_SAVES: PRegSet = PRegSet::empty() .with(preg(4)) .with(preg(5)) @@ -447,21 +500,40 @@ const DEFAULT_CALLEE_SAVES: PRegSet = PRegSet::empty() .with(preg(8)) .with(preg(9)) .with(preg(10)) - .with(preg(11)); - -/// Caller-saved (clobbered) GPRs: r0-r3 and r12. + .with(preg(11)) + .with(pdreg(8)) + .with(pdreg(9)) + .with(pdreg(10)) + .with(pdreg(11)) + .with(pdreg(12)) + .with(pdreg(13)) + .with(pdreg(14)) + .with(pdreg(15)); + +/// Caller-saved (clobbered) registers: GPRs r0-r3, r12 and VFP D0-D7. const DEFAULT_CLOBBERS: PRegSet = PRegSet::empty() .with(preg(0)) .with(preg(1)) .with(preg(2)) .with(preg(3)) - .with(preg(12)); + .with(preg(12)) + .with(pdreg(0)) + .with(pdreg(1)) + .with(pdreg(2)) + .with(pdreg(3)) + .with(pdreg(4)) + .with(pdreg(5)) + .with(pdreg(6)) + .with(pdreg(7)); fn compute_clobber_size(clobbers: &[Writable]) -> u32 { let mut size = 0; for reg in clobbers { - debug_assert_eq!(reg.to_reg().class(), RegClass::Int); - size += 4; + match reg.to_reg().class() { + RegClass::Int => size += 4, + RegClass::Float => size += 8, + RegClass::Vector => unreachable!("arm32 has no vector registers"), + } } align_to(size, 8) } diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index d4397ce8f0ca..9fd5ac284191 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -133,6 +133,28 @@ (Isb) (Clrex))) +;; The precision of a VFP floating-point operation. +(type FpuSize + (enum + (F32) + (F64))) + +;; A three-register VFP data-processing operation. +(type FpuOp3 + (enum + (Vadd) + (Vsub) + (Vmul) + (Vdiv))) + +;; A two-register VFP data-processing operation. +(type FpuOp2 + (enum + (Vmov) + (Vneg) + (Vabs) + (Vsqrt))) + ;; An ARM condition code (the "always" and "never" codes are omitted). (type Cond (enum @@ -287,6 +309,19 @@ ;; A memory barrier / clear-exclusive. (Barrier (op BarrierOp)) + ;; A three-register VFP op (`vadd`/`vsub`/`vmul`/`vdiv`). + (FpuRRR (op FpuOp3) (size FpuSize) (rd WritableReg) (rn Reg) (rm Reg)) + ;; A two-register VFP op (`vmov`/`vneg`/`vabs`/`vsqrt`). + (FpuRR (op FpuOp2) (size FpuSize) (rd WritableReg) (rm Reg)) + ;; A VFP load (`vldr`). + (FpuLoad (size FpuSize) (rd WritableReg) (mem AMode)) + ;; A VFP store (`vstr`). + (FpuStore (size FpuSize) (rt Reg) (mem AMode)) + ;; Move a GPR into the low half of an S register (`vmov s, r`) for f32 bits. + (MovToFpu32 (rd WritableReg) (rt Reg)) + ;; Move two GPRs into a D register (`vmov d, rlo, rhi`) for f64 bits. + (MovToFpu64 (rd WritableReg) (rt_lo Reg) (rt_hi Reg)) + ;; `cmp/cmn/tst/teq rn, rm`. (CmpRR (op CmpOp) (rn Reg) (rm Reg)) ;; `cmp/cmn/tst/teq rn, #imm` (pre-encoded rotated imm12). @@ -474,6 +509,47 @@ (rule (dmb) (SideEffectNoResult.Inst (MInst.Barrier (BarrierOp.Dmb)))) +;;;; Floating point ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl fpu_rrr (FpuOp3 FpuSize Reg Reg) Reg) +(rule (fpu_rrr op size rn rm) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.FpuRRR op size rd rn rm)))) + rd)) + +(decl fpu_rr (FpuOp2 FpuSize Reg) Reg) +(rule (fpu_rr op size rm) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.FpuRR op size rd rm)))) + rd)) + +;; An f32 constant: build the bit pattern in a GPR, then move it into an S reg. +(decl f32_const (u64) Reg) +(rule (f32_const bits) + (let ((r Reg (gen_constant bits)) + (rd WritableReg (temp_writable_reg $F32)) + (_ Unit (emit (MInst.MovToFpu32 rd r)))) + rd)) + +;; An f64 constant: build the two halves in GPRs, then move them into a D reg. +(decl f64_const (u64) Reg) +(rule (f64_const bits) + (let ((lo Reg (gen_constant bits)) + (hi Reg (gen_constant (u64_high32 bits))) + (rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.MovToFpu64 rd lo hi)))) + rd)) + +(decl fpu_load (FpuSize AMode) Reg) +(rule (fpu_load size mem) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.FpuLoad size rd mem)))) + rd)) + +(decl fpu_store (FpuSize Reg AMode) InstOutput) +(rule (fpu_store size val mem) + (side_effect (SideEffectNoResult.Inst (MInst.FpuStore size val mem)))) + ;;;; Shifts ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Shift/rotate by a constant amount. Handles the Cranelift "modulo width" @@ -501,6 +577,10 @@ (decl pure u64_high32 (u64) u64) (extern constructor u64_high32 u64_high32) +;; Widen a u32 to u64 (zero-extended). +(decl pure u32_to_u64 (u32) u64) +(extern constructor u32_to_u64 u32_to_u64) + ;; `adds`/`adc` and `subs`/`sbc` carry chains over the halves. (decl add_i64 (ValueRegs ValueRegs) ValueRegs) (rule (add_i64 x y) diff --git a/cranelift/codegen/src/isa/arm32/inst/args.rs b/cranelift/codegen/src/isa/arm32/inst/args.rs index 95dc74786249..b20949293697 100644 --- a/cranelift/codegen/src/isa/arm32/inst/args.rs +++ b/cranelift/codegen/src/isa/arm32/inst/args.rs @@ -5,7 +5,7 @@ use crate::machinst::{OperandVisitor, Reg}; pub use crate::isa::arm32::lower::isle::generated_code::{ ALUOp, AMode, BarrierOp, BfxOp, BitOp, CmpOp, Cond, DspMul3Op, DspMul4Op, DspMulLOp, ExtAddOp, - ExtOp, LoadKind, ParAluOp, PkhOp, QAluOp, SatOp, ShiftOp, StoreKind, + ExtOp, FpuOp2, FpuOp3, FpuSize, LoadKind, ParAluOp, PkhOp, QAluOp, SatOp, ShiftOp, StoreKind, }; /// A memory address resolved to a concrete base register and either an @@ -607,6 +607,65 @@ impl StoreKind { } } +impl FpuSize { + /// The size bit (bit 8) that distinguishes F64 (set) from F32 (clear). + pub(crate) fn size_bit(self) -> u32 { + match self { + FpuSize::F32 => 0, + FpuSize::F64 => 0x100, + } + } + + pub(crate) fn suffix(self) -> &'static str { + match self { + FpuSize::F32 => "f32", + FpuSize::F64 => "f64", + } + } +} + +impl FpuOp3 { + /// The F32 base word (`vadd.f32 s0, s0, s0`); F64 adds the size bit. + pub(crate) fn base(self) -> u32 { + match self { + FpuOp3::Vadd => 0xee30_0a00, + FpuOp3::Vsub => 0xee30_0a40, + FpuOp3::Vmul => 0xee20_0a00, + FpuOp3::Vdiv => 0xee80_0a00, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + FpuOp3::Vadd => "vadd", + FpuOp3::Vsub => "vsub", + FpuOp3::Vmul => "vmul", + FpuOp3::Vdiv => "vdiv", + } + } +} + +impl FpuOp2 { + /// The F32 base word; F64 adds the size bit. + pub(crate) fn base(self) -> u32 { + match self { + FpuOp2::Vmov => 0xeeb0_0a40, + FpuOp2::Vneg => 0xeeb1_0a40, + FpuOp2::Vabs => 0xeeb0_0ac0, + FpuOp2::Vsqrt => 0xeeb1_0ac0, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + FpuOp2::Vmov => "vmov", + FpuOp2::Vneg => "vneg", + FpuOp2::Vabs => "vabs", + FpuOp2::Vsqrt => "vsqrt", + } + } +} + /// Decode a 12-bit rotated data-processing immediate back to its value, for /// pretty-printing. pub fn decode_rotated_imm(enc: u32) -> u32 { diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index db15684f4b48..92e95ec9d8b0 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -386,6 +386,46 @@ fn enc_stl(rt: u32, rn: u32) -> u32 { COND_AL | 0x0180_fc90 | (rn << 16) | rt } +/// The hardware encoding number (0-15) of a VFP register. An f32 uses the low +/// S-half of its D register, whose Vd/D fields encode identically, so this is +/// just the D-register number. +fn machreg_to_vfp(reg: Reg) -> u32 { + u32::from(reg.to_real_reg().unwrap().hw_enc() & 0xf) +} + +/// A three-register VFP data-processing instruction. +fn enc_fpu_rrr(op: FpuOp3, size: FpuSize, d: u32, n: u32, m: u32) -> u32 { + op.base() | size.size_bit() | (n << 16) | (d << 12) | m +} + +/// A two-register VFP data-processing instruction. +fn enc_fpu_rr(op: FpuOp2, size: FpuSize, d: u32, m: u32) -> u32 { + op.base() | size.size_bit() | (d << 12) | m +} + +/// `vldr`/`vstr , [rn, #±offset]` (offset must be a multiple of 4). +fn enc_vldr_vstr(load: bool, size: FpuSize, vd: u32, rn: u32, offset: i32) -> u32 { + let base = (if load { 0xed90_0a00 } else { 0xed80_0a00 }) | size.size_bit(); + assert!(offset % 4 == 0, "arm32 vldr/vstr offset must be 4-aligned"); + let (base, mag) = if offset < 0 { + (base & !0x0080_0000, ((-offset) as u32) / 4) + } else { + (base, (offset as u32) / 4) + }; + assert!(mag < 256, "arm32 vldr/vstr offset out of range"); + base | (rn << 16) | (vd << 12) | mag +} + +/// `vmov sn, rt` — move a GPR into a single-precision register. +fn enc_vmov_gpr_s(vn: u32, rt: u32) -> u32 { + COND_AL | 0x0e00_0a10 | (vn << 16) | (rt << 12) +} + +/// `vmov dm, rt_lo, rt_hi` — move two GPRs into a double register. +fn enc_vmov_gpr_d(vm: u32, rt_lo: u32, rt_hi: u32) -> u32 { + COND_AL | 0x0c40_0b10 | (rt_hi << 16) | (rt_lo << 12) | vm +} + fn put_u32(sink: &mut MachBuffer, word: u32) { for b in word.to_le_bytes() { sink.put1(b); @@ -700,6 +740,45 @@ impl MachInstEmit for Inst { put_u32(sink, enc_stl(rt, rn)); } Inst::Barrier { op } => put_u32(sink, op.encoding()), + + Inst::FpuRRR { + op, + size, + rd, + rn, + rm, + } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rn = machreg_to_vfp(*rn); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_fpu_rrr(*op, *size, rd, rn, rm)); + } + Inst::FpuRR { op, size, rd, rm } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_fpu_rr(*op, *size, rd, rm)); + } + Inst::FpuLoad { size, rd, mem } => { + let rd = machreg_to_vfp(rd.to_reg()); + let (base, offset) = resolve_fpu_amode(mem, state); + put_u32(sink, enc_vldr_vstr(true, *size, rd, base, offset)); + } + Inst::FpuStore { size, rt, mem } => { + let rt = machreg_to_vfp(*rt); + let (base, offset) = resolve_fpu_amode(mem, state); + put_u32(sink, enc_vldr_vstr(false, *size, rt, base, offset)); + } + Inst::MovToFpu32 { rd, rt } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rt = machreg_to_gpr(*rt); + put_u32(sink, enc_vmov_gpr_s(rd, rt)); + } + Inst::MovToFpu64 { rd, rt_lo, rt_hi } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rt_lo = machreg_to_gpr(*rt_lo); + let rt_hi = machreg_to_gpr(*rt_hi); + put_u32(sink, enc_vmov_gpr_d(rd, rt_lo, rt_hi)); + } Inst::CmpRR { op, rn, rm } => { let rn = machreg_to_gpr(*rn); let rm = machreg_to_gpr(*rm); @@ -773,6 +852,17 @@ impl MachInstEmit for Inst { } } +/// Resolve an `AMode` for a VFP access, which only supports an immediate +/// offset. Returns the base GPR encoding and the byte offset. +fn resolve_fpu_amode(mem: &AMode, state: &EmitState) -> (u32, i32) { + match mem.resolve(state) { + ResolvedAMode::Imm { base, offset } => (machreg_to_gpr(base), offset), + ResolvedAMode::Reg { .. } => { + unimplemented!("arm32: register-offset VFP addressing is not supported") + } + } +} + enum LoadStore { Load(LoadKind), Store(StoreKind), diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index 9cbce86cdf53..cf0f2537fcc3 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -1,7 +1,7 @@ //! This module defines arm32 (AArch32 / A32) machine instruction types. use crate::binemit::{Addend, CodeOffset, Reloc}; -use crate::ir::types::{I8, I16, I32, I64}; +use crate::ir::types::{F32, F64, I8, I16, I32, I64}; use crate::ir::{Type, types}; use crate::isa::FunctionAlignment; use crate::isa::arm32::abi::Arm32MachineDeps; @@ -147,6 +147,33 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { def_if_virtual(collector, rd); } Inst::Udf { .. } | Inst::Barrier { .. } => {} + + Inst::FpuRRR { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::FpuRR { rd, rm, .. } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::FpuLoad { rd, mem, .. } => { + mem.get_operands(collector); + def_if_virtual(collector, rd); + } + Inst::FpuStore { rt, mem, .. } => { + use_if_virtual(collector, rt); + mem.get_operands(collector); + } + Inst::MovToFpu32 { rd, rt } => { + use_if_virtual(collector, rt); + def_if_virtual(collector, rd); + } + Inst::MovToFpu64 { rd, rt_lo, rt_hi } => { + use_if_virtual(collector, rt_lo); + use_if_virtual(collector, rt_hi); + def_if_virtual(collector, rd); + } Inst::LdmStm { rn, .. } => use_if_virtual(collector, rn), Inst::LoadEx { rt, rn, .. } | Inst::LoadAcq { rt, rn } => { use_if_virtual(collector, rn); @@ -279,7 +306,7 @@ impl MachInst for Inst { fn canonical_type_for_rc(rc: RegClass) -> Type { match rc { RegClass::Int => I32, - RegClass::Float => types::F64, + RegClass::Float => F64, RegClass::Vector => types::I8X16, } } @@ -331,9 +358,20 @@ impl MachInst for Inst { } fn gen_move(to_reg: Writable, from_reg: Reg, _ty: Type) -> Inst { - Inst::MovReg { - rd: to_reg, - rm: from_reg, + if to_reg.to_reg().class() == RegClass::Float { + // A `vmov.f64` copies the whole D register, which is correct for an + // f32 held in the low half too. + Inst::FpuRR { + op: FpuOp2::Vmov, + size: FpuSize::F64, + rd: to_reg, + rm: from_reg, + } + } else { + Inst::MovReg { + rd: to_reg, + rm: from_reg, + } } } @@ -356,8 +394,11 @@ impl MachInst for Inst { I32 => Ok((&[RegClass::Int], &[I32])), // 64-bit values are held in a pair of 32-bit integer registers. I64 => Ok((&[RegClass::Int, RegClass::Int], &[I32, I32])), + // Floats live in VFP registers (an f32 in the low half of a D reg). + F32 => Ok((&[RegClass::Float], &[F32])), + F64 => Ok((&[RegClass::Float], &[F64])), _ => Err(CodegenError::Unsupported(alloc::format!( - "Unsupported type on arm32 (only i8/i16/i32/i64 are implemented so far): {ty}" + "Unsupported type on arm32 (only i8/i16/i32/i64/f32/f64 are implemented so far): {ty}" ))), } } @@ -651,6 +692,40 @@ impl Inst { } Inst::StoreRel { rt, rn } => alloc::format!("stl {}, [{}]", r(*rt), r(*rn)), Inst::Barrier { op } => op.name().to_string(), + + Inst::FpuRRR { + op, + size, + rd, + rn, + rm, + } => alloc::format!( + "{}.{} {}, {}, {}", + op.name(), + size.suffix(), + r(rd.to_reg()), + r(*rn), + r(*rm) + ), + Inst::FpuRR { op, size, rd, rm } => alloc::format!( + "{}.{} {}, {}", + op.name(), + size.suffix(), + r(rd.to_reg()), + r(*rm) + ), + Inst::FpuLoad { rd, mem, .. } => { + alloc::format!("vldr {}, {}", r(rd.to_reg()), mem.pretty_print()) + } + Inst::FpuStore { rt, mem, .. } => { + alloc::format!("vstr {}, {}", r(*rt), mem.pretty_print()) + } + Inst::MovToFpu32 { rd, rt } => { + alloc::format!("vmov {}, {}", r(rd.to_reg()), r(*rt)) + } + Inst::MovToFpu64 { rd, rt_lo, rt_hi } => { + alloc::format!("vmov {}, {}, {}", r(rd.to_reg()), r(*rt_lo), r(*rt_hi)) + } Inst::CmpRR { op, rn, rm } => { alloc::format!("{} {}, {}", op.name(), r(*rn), r(*rm)) } diff --git a/cranelift/codegen/src/isa/arm32/inst/regs.rs b/cranelift/codegen/src/isa/arm32/inst/regs.rs index a9ef5b5bf1cd..4244172032df 100644 --- a/cranelift/codegen/src/isa/arm32/inst/regs.rs +++ b/cranelift/codegen/src/isa/arm32/inst/regs.rs @@ -29,6 +29,27 @@ pub const fn preg(enc: u8) -> PReg { PReg::new(enc as usize, RegClass::Int) } +/// Construct a `Reg` referencing VFP double-register `D` (0-15). An `f32` +/// value uses the low single-register half (`S<2*enc>`) of its `D` register, so +/// each value owns a whole `D` register and the S/D register files never alias. +#[inline] +pub const fn dreg(enc: u8) -> Reg { + let p_reg = PReg::new(enc as usize, RegClass::Float); + let v_reg = VReg::new(p_reg.index(), p_reg.class()); + Reg::from_virtual_reg(v_reg) +} + +/// Construct a `PReg` referencing VFP double-register `D`. +pub const fn pdreg(enc: u8) -> PReg { + PReg::new(enc as usize, RegClass::Float) +} + +/// Get a writable reference to `D`. +#[inline] +pub fn writable_dreg(enc: u8) -> Writable { + Writable::from_reg(dreg(enc)) +} + /// Get a writable reference to GPR `enc`. #[inline] pub fn writable_xreg(enc: u8) -> Writable { @@ -93,17 +114,20 @@ pub fn pc_reg() -> Reg { xreg(15) } -/// Pretty-print a register with its AAPCS name. +/// Pretty-print a register with its AAPCS / VFP name. pub fn reg_name(reg: Reg) -> alloc::string::String { use alloc::string::ToString; match reg.to_real_reg() { - Some(real) => match real.hw_enc() { - 11 => "fp".to_string(), - 12 => "ip".to_string(), - 13 => "sp".to_string(), - 14 => "lr".to_string(), - 15 => "pc".to_string(), - enc => alloc::format!("r{enc}"), + Some(real) => match real.class() { + RegClass::Float => alloc::format!("d{}", real.hw_enc()), + _ => match real.hw_enc() { + 11 => "fp".to_string(), + 12 => "ip".to_string(), + 13 => "sp".to_string(), + 14 => "lr".to_string(), + 15 => "pc".to_string(), + enc => alloc::format!("r{enc}"), + }, }, None => alloc::format!("{reg:?}"), } @@ -112,25 +136,27 @@ pub fn reg_name(reg: Reg) -> alloc::string::String { /// Build the register environment describing which registers the allocator may /// use, in preference order. pub const fn create_reg_environment() -> MachineEnv { - // Preferred registers are the caller-saved (scratch) GPRs; the allocator - // reaches for these first since they don't need to be saved/restored. - let preferred_int = preg_set_int(&[0, 1, 2, 3]); - // Non-preferred registers are the callee-saved GPRs. - let non_preferred_int = preg_set_int(&[4, 5, 6, 7, 8, 9, 10]); + // Preferred registers are the caller-saved (scratch) registers; the + // allocator reaches for these first since they need no save/restore. + let preferred_int = preg_set(RegClass::Int, &[0, 1, 2, 3]); + let non_preferred_int = preg_set(RegClass::Int, &[4, 5, 6, 7, 8, 9, 10]); + // AAPCS: D0-D7 are caller-saved, D8-D15 callee-saved. + let preferred_float = preg_set(RegClass::Float, &[0, 1, 2, 3, 4, 5, 6, 7]); + let non_preferred_float = preg_set(RegClass::Float, &[8, 9, 10, 11, 12, 13, 14, 15]); MachineEnv { - preferred_regs_by_class: [preferred_int, empty(), empty()], - non_preferred_regs_by_class: [non_preferred_int, empty(), empty()], + preferred_regs_by_class: [preferred_int, preferred_float, empty()], + non_preferred_regs_by_class: [non_preferred_int, non_preferred_float, empty()], fixed_stack_slots: vec![], scratch_by_class: [None, None, None], } } -const fn preg_set_int(encs: &[u8]) -> regalloc2::PRegSet { +const fn preg_set(class: RegClass, encs: &[u8]) -> regalloc2::PRegSet { let mut set = regalloc2::PRegSet::empty(); let mut i = 0; while i < encs.len() { - set = set.with(preg(encs[i])); + set = set.with(PReg::new(encs[i] as usize, class)); i += 1; } set diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index bb2ab2ab46f4..6588df018124 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -130,6 +130,54 @@ (rule (lower (fence)) (side_effect (dmb))) +;;;; Floating point ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (f32const _ (u32_from_ieee32 n))) + (f32_const (u32_to_u64 n))) +(rule (lower (f64const _ (u64_from_ieee64 n))) + (f64_const n)) + +(rule (lower (fadd $F32 x y)) + (fpu_rrr (FpuOp3.Vadd) (FpuSize.F32) (put_in_reg x) (put_in_reg y))) +(rule (lower (fadd $F64 x y)) + (fpu_rrr (FpuOp3.Vadd) (FpuSize.F64) (put_in_reg x) (put_in_reg y))) +(rule (lower (fsub $F32 x y)) + (fpu_rrr (FpuOp3.Vsub) (FpuSize.F32) (put_in_reg x) (put_in_reg y))) +(rule (lower (fsub $F64 x y)) + (fpu_rrr (FpuOp3.Vsub) (FpuSize.F64) (put_in_reg x) (put_in_reg y))) +(rule (lower (fmul $F32 x y)) + (fpu_rrr (FpuOp3.Vmul) (FpuSize.F32) (put_in_reg x) (put_in_reg y))) +(rule (lower (fmul $F64 x y)) + (fpu_rrr (FpuOp3.Vmul) (FpuSize.F64) (put_in_reg x) (put_in_reg y))) +(rule (lower (fdiv $F32 x y)) + (fpu_rrr (FpuOp3.Vdiv) (FpuSize.F32) (put_in_reg x) (put_in_reg y))) +(rule (lower (fdiv $F64 x y)) + (fpu_rrr (FpuOp3.Vdiv) (FpuSize.F64) (put_in_reg x) (put_in_reg y))) + +(rule (lower (fneg $F32 x)) + (fpu_rr (FpuOp2.Vneg) (FpuSize.F32) (put_in_reg x))) +(rule (lower (fneg $F64 x)) + (fpu_rr (FpuOp2.Vneg) (FpuSize.F64) (put_in_reg x))) +(rule (lower (fabs $F32 x)) + (fpu_rr (FpuOp2.Vabs) (FpuSize.F32) (put_in_reg x))) +(rule (lower (fabs $F64 x)) + (fpu_rr (FpuOp2.Vabs) (FpuSize.F64) (put_in_reg x))) +(rule (lower (sqrt $F32 x)) + (fpu_rr (FpuOp2.Vsqrt) (FpuSize.F32) (put_in_reg x))) +(rule (lower (sqrt $F64 x)) + (fpu_rr (FpuOp2.Vsqrt) (FpuSize.F64) (put_in_reg x))) + +;; Priority 1 so the float result/value type wins over the integer load/store +;; rules (ISLE can't prove `$F32` disjoint from `fits_in_32`). +(rule 1 (lower (load $F32 flags addr offset)) + (fpu_load (FpuSize.F32) (amode addr offset))) +(rule 1 (lower (load $F64 flags addr offset)) + (fpu_load (FpuSize.F64) (amode addr offset))) +(rule 1 (lower (store flags val @ (value_type $F32) addr offset)) + (fpu_store (FpuSize.F32) (put_in_reg val) (amode addr offset))) +(rule 1 (lower (store flags val @ (value_type $F64) addr offset)) + (fpu_store (FpuSize.F64) (put_in_reg val) (amode addr offset))) + ;;;; Divides (only when hardware `sdiv`/`udiv` is available) ;;;;;;;;;;;;;;;;;;; ;; NOTE: trap-on-zero / INT_MIN overflow checks are not yet emitted. diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs index 1cc10654bf74..121e67883e02 100644 --- a/cranelift/codegen/src/isa/arm32/lower/isle.rs +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -151,6 +151,11 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { val >> 32 } + /// Zero-extend a u32 to u64. + fn u32_to_u64(&mut self, val: u32) -> u64 { + u64::from(val) + } + /// The offset of the high word of a 64-bit memory access. fn offset32_plus4(&mut self, offset: Offset32) -> i32 { i32::from(offset) + 4 diff --git a/cranelift/filetests/filetests/isa/arm32/float.clif b/cranelift/filetests/filetests/isa/arm32/float.clif new file mode 100644 index 000000000000..b8a5867084af --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/float.clif @@ -0,0 +1,129 @@ +test compile precise-output +target arm + +function %fadd32(f32, f32) -> f32 { +block0(v0: f32, v1: f32): + v2 = fadd v0, v1 + return v2 +} + +; VCode: +; block0: +; vadd.f32 d0, d0, d1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vadd.f32 s0, s0, s2 +; bx lr + +function %fmul_fsub64(f64, f64, f64) -> f64 { +block0(v0: f64, v1: f64, v2: f64): + v3 = fmul v0, v1 + v4 = fsub v3, v2 + return v4 +} + +; VCode: +; block0: +; vmul.f64 d5, d0, d1 +; vsub.f64 d0, d5, d2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmul.f64 d5, d0, d1 +; vsub.f64 d0, d5, d2 +; bx lr + +function %fdiv_sqrt(f64, f64) -> f64 { +block0(v0: f64, v1: f64): + v2 = fdiv v0, v1 + v3 = sqrt v2 + return v3 +} + +; VCode: +; block0: +; vdiv.f64 d4, d0, d1 +; vsqrt.f64 d0, d4 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vdiv.f64 d4, d0, d1 +; vsqrt.f64 d0, d4 +; bx lr + +function %fneg_abs(f32) -> f32 { +block0(v0: f32): + v1 = fneg v0 + v2 = fabs v1 + return v2 +} + +; VCode: +; block0: +; vneg.f32 d3, d0 +; vabs.f32 d0, d3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vneg.f32 s6, s0 +; vabs.f32 s0, s6 +; bx lr + +function %fconst64() -> f64 { +block0: + v0 = f64const 0x1.8p3 + return v0 +} + +; VCode: +; block0: +; mov r0, #0 +; movw r2, #0; movt r2, #16424 +; vmov d0, r0, r2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mov r0, #0 +; movw r2, #0 +; movt r2, #0x4028 +; vmov d0, r0, r2 +; bx lr + +function %load_f32(i32) -> f32 { +block0(v0: i32): + v1 = load.f32 v0+8 + return v1 +} + +; VCode: +; block0: +; vldr d0, [r0, #8] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vldr s0, [r0, #8] +; bx lr + +function %store_f64(i32, f64) { +block0(v0: i32, v1: f64): + store.f64 v1, v0 + return +} + +; VCode: +; block0: +; vstr d0, [r0, #0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vstr d0, [r0] +; bx lr + From f3e2f053e5befc3ed6b81840f40063a394b694fa Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Sat, 4 Jul 2026 09:30:04 +0300 Subject: [PATCH 11/18] arm32: lower float comparisons, conversions, and bitcasts fcmp emits vcmp followed by vmrs to get the flags into APSR, then tests the mapped condition. fpromote/fdemote and the int<->float conversions go through vcvt (only the saturating float-to-int, matching VCVT's semantics). Bitcasts between float and integer of the same width use vmov. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/inst.isle | 75 ++++++++++ cranelift/codegen/src/isa/arm32/inst/emit.rs | 78 +++++++++++ cranelift/codegen/src/isa/arm32/inst/mod.rs | 60 ++++++++ cranelift/codegen/src/isa/arm32/lower.isle | 56 ++++++++ cranelift/codegen/src/isa/arm32/lower/isle.rs | 24 ++++ .../filetests/isa/arm32/float-cvt.clif | 130 ++++++++++++++++++ 6 files changed, 423 insertions(+) create mode 100644 cranelift/filetests/filetests/isa/arm32/float-cvt.clif diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index 9fd5ac284191..7b4aa7d1f5c1 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -321,6 +321,21 @@ (MovToFpu32 (rd WritableReg) (rt Reg)) ;; Move two GPRs into a D register (`vmov d, rlo, rhi`) for f64 bits. (MovToFpu64 (rd WritableReg) (rt_lo Reg) (rt_hi Reg)) + ;; Move an S register into a GPR (`vmov r, s`). + (MovFromFpu32 (rt WritableReg) (rm Reg)) + ;; Move a D register into two GPRs (`vmov rlo, rhi, d`). + (MovFromFpu64 (rt_lo WritableReg) (rt_hi WritableReg) (rm Reg)) + + ;; Compare two float registers and move the VFP flags into the CPSR + ;; (`vcmp` + `vmrs APSR_nzcv, FPSCR`). + (VcmpMrs (size FpuSize) (rn Reg) (rm Reg)) + ;; Convert between f32 and f64 (`to_f64` selects fpromote vs fdemote). + (VcvtFF (to_f64 bool) (rd WritableReg) (rm Reg)) + ;; Convert a float to a 32-bit integer (round toward zero, saturating). + ;; `rd` receives the integer in an S register. + (VcvtToInt (signed bool) (size FpuSize) (rd WritableReg) (rm Reg)) + ;; Convert a 32-bit integer (held in the S register `rm`) to a float. + (VcvtFromInt (signed bool) (size FpuSize) (rd WritableReg) (rm Reg)) ;; `cmp/cmn/tst/teq rn, rm`. (CmpRR (op CmpOp) (rn Reg) (rm Reg)) @@ -550,6 +565,66 @@ (rule (fpu_store size val mem) (side_effect (SideEffectNoResult.Inst (MInst.FpuStore size val mem)))) +;; Map a Cranelift `FloatCC` to the ARM condition testing the VFP flags. +(decl cond_from_floatcc (FloatCC) Cond) +(extern constructor cond_from_floatcc cond_from_floatcc) + +;; `vcmp` + `vmrs` as a side effect (leaves the comparison in the CPSR). +(decl vcmp (FpuSize Reg Reg) SideEffectNoResult) +(rule (vcmp size rn rm) + (SideEffectNoResult.Inst (MInst.VcmpMrs size rn rm))) + +;; f32<->f64 conversion. +(decl vcvt_ff (bool Reg) Reg) +(rule (vcvt_ff to_f64 rm) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.VcvtFF to_f64 rd rm)))) + rd)) + +;; int (in a GPR) -> float. +(decl vcvt_from_int (bool FpuSize Reg) Reg) +(rule (vcvt_from_int signed size gpr) + (let ((s WritableReg (temp_writable_reg $F32)) + (_ Unit (emit (MInst.MovToFpu32 s gpr))) + (rd WritableReg (temp_writable_reg $F64)) + (_2 Unit (emit (MInst.VcvtFromInt signed size rd (writable_reg_to_reg s))))) + rd)) + +;; float -> i32 (in a GPR), round toward zero, saturating. +(decl vcvt_to_int (bool FpuSize Reg) Reg) +(rule (vcvt_to_int signed size f) + (let ((s WritableReg (temp_writable_reg $F32)) + (_ Unit (emit (MInst.VcvtToInt signed size s f))) + (rd WritableReg (temp_writable_reg $I32)) + (_2 Unit (emit (MInst.MovFromFpu32 rd (writable_reg_to_reg s))))) + rd)) + +;; bitcast helpers (`vmov` between the GPR and VFP files). +(decl mov_from_fpu32 (Reg) Reg) +(rule (mov_from_fpu32 rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.MovFromFpu32 rd rm)))) + rd)) + +(decl mov_to_fpu32 (Reg) Reg) +(rule (mov_to_fpu32 rt) + (let ((rd WritableReg (temp_writable_reg $F32)) + (_ Unit (emit (MInst.MovToFpu32 rd rt)))) + rd)) + +(decl mov_from_fpu64 (Reg) ValueRegs) +(rule (mov_from_fpu64 rm) + (let ((lo WritableReg (temp_writable_reg $I32)) + (hi WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.MovFromFpu64 lo hi rm)))) + (value_regs (writable_reg_to_reg lo) (writable_reg_to_reg hi)))) + +(decl mov_to_fpu64 (ValueRegs) Reg) +(rule (mov_to_fpu64 regs) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.MovToFpu64 rd (vr_lo regs) (vr_hi regs))))) + rd)) + ;;;; Shifts ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Shift/rotate by a constant amount. Handles the Cranelift "modulo width" diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index 92e95ec9d8b0..ea0d3b46821d 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -426,6 +426,42 @@ fn enc_vmov_gpr_d(vm: u32, rt_lo: u32, rt_hi: u32) -> u32 { COND_AL | 0x0c40_0b10 | (rt_hi << 16) | (rt_lo << 12) | vm } +/// `vmov rt, sn` — move a single register into a GPR. +fn enc_vmov_s_to_gpr(rt: u32, vn: u32) -> u32 { + COND_AL | 0x0e10_0a10 | (vn << 16) | (rt << 12) +} + +/// `vmov rt_lo, rt_hi, dm` — move a double register into two GPRs. +fn enc_vmov_d_to_gpr(rt_lo: u32, rt_hi: u32, vm: u32) -> u32 { + COND_AL | 0x0c50_0b10 | (rt_hi << 16) | (rt_lo << 12) | vm +} + +/// The `vmrs APSR_nzcv, FPSCR` instruction (moves the VFP flags to the CPSR). +const VMRS_APSR_NZCV: u32 = 0xeef1_fa10; + +/// `vcmp.f32/f64 sn/dn, sm/dm`. +fn enc_vcmp(size: FpuSize, n: u32, m: u32) -> u32 { + (0xeeb4_0a40 | size.size_bit()) | (n << 12) | m +} + +/// `vcvt.f64.f32`/`vcvt.f32.f64` (`to_f64` selects promote vs demote). +fn enc_vcvt_ff(to_f64: bool, d: u32, m: u32) -> u32 { + let base = if to_f64 { 0xeeb7_0ac0 } else { 0xeeb7_0bc0 }; + base | (d << 12) | m +} + +/// `vcvt.s32/u32.f32/f64` — float to integer, round toward zero. +fn enc_vcvt_to_int(signed: bool, size: FpuSize, d: u32, m: u32) -> u32 { + let sign = if signed { 0x1_0000 } else { 0 }; + (0xeebc_0ac0 | sign | size.size_bit()) | (d << 12) | m +} + +/// `vcvt.f32/f64.s32/u32` — integer (in the S reg `m`) to float. +fn enc_vcvt_from_int(signed: bool, size: FpuSize, d: u32, m: u32) -> u32 { + let sign = if signed { 0x80 } else { 0 }; + (0xeeb8_0a40 | sign | size.size_bit()) | (d << 12) | m +} + fn put_u32(sink: &mut MachBuffer, word: u32) { for b in word.to_le_bytes() { sink.put1(b); @@ -779,6 +815,48 @@ impl MachInstEmit for Inst { let rt_hi = machreg_to_gpr(*rt_hi); put_u32(sink, enc_vmov_gpr_d(rd, rt_lo, rt_hi)); } + Inst::MovFromFpu32 { rt, rm } => { + let rt = machreg_to_gpr(rt.to_reg()); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_vmov_s_to_gpr(rt, rm)); + } + Inst::MovFromFpu64 { rt_lo, rt_hi, rm } => { + let rt_lo = machreg_to_gpr(rt_lo.to_reg()); + let rt_hi = machreg_to_gpr(rt_hi.to_reg()); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_vmov_d_to_gpr(rt_lo, rt_hi, rm)); + } + Inst::VcmpMrs { size, rn, rm } => { + let rn = machreg_to_vfp(*rn); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_vcmp(*size, rn, rm)); + put_u32(sink, VMRS_APSR_NZCV); + } + Inst::VcvtFF { to_f64, rd, rm } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_vcvt_ff(*to_f64, rd, rm)); + } + Inst::VcvtToInt { + signed, + size, + rd, + rm, + } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_vcvt_to_int(*signed, *size, rd, rm)); + } + Inst::VcvtFromInt { + signed, + size, + rd, + rm, + } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_vcvt_from_int(*signed, *size, rd, rm)); + } Inst::CmpRR { op, rn, rm } => { let rn = machreg_to_gpr(*rn); let rm = machreg_to_gpr(*rm); diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index cf0f2537fcc3..769c86a36ea0 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -174,6 +174,25 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { use_if_virtual(collector, rt_hi); def_if_virtual(collector, rd); } + Inst::MovFromFpu32 { rt, rm } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rt); + } + Inst::MovFromFpu64 { rt_lo, rt_hi, rm } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rt_lo); + def_if_virtual(collector, rt_hi); + } + Inst::VcmpMrs { rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + } + Inst::VcvtFF { rd, rm, .. } + | Inst::VcvtToInt { rd, rm, .. } + | Inst::VcvtFromInt { rd, rm, .. } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } Inst::LdmStm { rn, .. } => use_if_virtual(collector, rn), Inst::LoadEx { rt, rn, .. } | Inst::LoadAcq { rt, rn } => { use_if_virtual(collector, rn); @@ -726,6 +745,47 @@ impl Inst { Inst::MovToFpu64 { rd, rt_lo, rt_hi } => { alloc::format!("vmov {}, {}, {}", r(rd.to_reg()), r(*rt_lo), r(*rt_hi)) } + Inst::MovFromFpu32 { rt, rm } => { + alloc::format!("vmov {}, {}", r(rt.to_reg()), r(*rm)) + } + Inst::MovFromFpu64 { rt_lo, rt_hi, rm } => { + alloc::format!("vmov {}, {}, {}", r(rt_lo.to_reg()), r(rt_hi.to_reg()), r(*rm)) + } + Inst::VcmpMrs { size, rn, rm } => { + alloc::format!("vcmp.{} {}, {}; vmrs", size.suffix(), r(*rn), r(*rm)) + } + Inst::VcvtFF { to_f64, rd, rm } => { + let (dst, src) = if *to_f64 { ("f64", "f32") } else { ("f32", "f64") }; + alloc::format!("vcvt.{dst}.{src} {}, {}", r(rd.to_reg()), r(*rm)) + } + Inst::VcvtToInt { + signed, + size, + rd, + rm, + } => { + let int = if *signed { "s32" } else { "u32" }; + alloc::format!( + "vcvt.{int}.{} {}, {}", + size.suffix(), + r(rd.to_reg()), + r(*rm) + ) + } + Inst::VcvtFromInt { + signed, + size, + rd, + rm, + } => { + let int = if *signed { "s32" } else { "u32" }; + alloc::format!( + "vcvt.{}.{int} {}, {}", + size.suffix(), + r(rd.to_reg()), + r(*rm) + ) + } Inst::CmpRR { op, rn, rm } => { alloc::format!("{} {}, {}", op.name(), r(*rn), r(*rm)) } diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index 6588df018124..3157e29c5475 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -178,6 +178,50 @@ (rule 1 (lower (store flags val @ (value_type $F64) addr offset)) (fpu_store (FpuSize.F64) (put_in_reg val) (amode addr offset))) +;; Float comparisons. +(rule (lower (fcmp _ cc a @ (value_type $F32) b)) + (let ((_ Unit (emit_side_effect (vcmp (FpuSize.F32) (put_in_reg a) (put_in_reg b))))) + (csetv (cond_from_floatcc cc)))) +(rule (lower (fcmp _ cc a @ (value_type $F64) b)) + (let ((_ Unit (emit_side_effect (vcmp (FpuSize.F64) (put_in_reg a) (put_in_reg b))))) + (csetv (cond_from_floatcc cc)))) + +;; f32 <-> f64. +(rule (lower (fpromote $F64 x)) + (vcvt_ff true (put_in_reg x))) +(rule (lower (fdemote $F32 x)) + (vcvt_ff false (put_in_reg x))) + +;; int -> float. +(rule (lower (fcvt_from_sint $F32 x @ (value_type $I32))) + (vcvt_from_int true (FpuSize.F32) (put_in_reg x))) +(rule (lower (fcvt_from_sint $F64 x @ (value_type $I32))) + (vcvt_from_int true (FpuSize.F64) (put_in_reg x))) +(rule (lower (fcvt_from_uint $F32 x @ (value_type $I32))) + (vcvt_from_int false (FpuSize.F32) (put_in_reg x))) +(rule (lower (fcvt_from_uint $F64 x @ (value_type $I32))) + (vcvt_from_int false (FpuSize.F64) (put_in_reg x))) + +;; float -> i32 (saturating). +(rule (lower (fcvt_to_sint_sat $I32 x @ (value_type $F32))) + (vcvt_to_int true (FpuSize.F32) (put_in_reg x))) +(rule (lower (fcvt_to_sint_sat $I32 x @ (value_type $F64))) + (vcvt_to_int true (FpuSize.F64) (put_in_reg x))) +(rule (lower (fcvt_to_uint_sat $I32 x @ (value_type $F32))) + (vcvt_to_int false (FpuSize.F32) (put_in_reg x))) +(rule (lower (fcvt_to_uint_sat $I32 x @ (value_type $F64))) + (vcvt_to_int false (FpuSize.F64) (put_in_reg x))) + +;; Bitcasts between the integer and float register files. +(rule (lower (bitcast $I32 _ x @ (value_type $F32))) + (mov_from_fpu32 (put_in_reg x))) +(rule (lower (bitcast $F32 _ x @ (value_type $I32))) + (mov_to_fpu32 (put_in_reg x))) +(rule (lower (bitcast $I64 _ x @ (value_type $F64))) + (mov_from_fpu64 (put_in_reg x))) +(rule (lower (bitcast $F64 _ x @ (value_type $I64))) + (mov_to_fpu64 (put_in_regs x))) + ;;;; Divides (only when hardware `sdiv`/`udiv` is available) ;;;;;;;;;;;;;;;;;;; ;; NOTE: trap-on-zero / INT_MIN overflow checks are not yet emitted. @@ -331,6 +375,18 @@ (emit_side_effect (cond_br (lower_icmp_i64 cc (put_in_regs a) (put_in_regs b)) taken not_taken))) +;; Fuse an `fcmp` feeding a `brif`. +(rule 1 (lower_branch (brif (fcmp _ cc a @ (value_type $F32) b) _ _) (two_targets taken not_taken)) + (emit_side_effect + (side_effect_concat + (vcmp (FpuSize.F32) (put_in_reg a) (put_in_reg b)) + (cond_br (cond_from_floatcc cc) taken not_taken)))) +(rule 1 (lower_branch (brif (fcmp _ cc a @ (value_type $F64) b) _ _) (two_targets taken not_taken)) + (emit_side_effect + (side_effect_concat + (vcmp (FpuSize.F64) (put_in_reg a) (put_in_reg b)) + (cond_br (cond_from_floatcc cc) taken not_taken)))) + ;; Generic `brif`: branch when the value is non-zero. (rule (lower_branch (brif v _ _) (two_targets taken not_taken)) (emit_side_effect diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs index 121e67883e02..e648714e320e 100644 --- a/cranelift/codegen/src/isa/arm32/lower/isle.rs +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -295,6 +295,30 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { } } + /// Map a `FloatCC` to the ARM condition testing the flags left by a `vcmp` + /// (`vmrs`). The `vcmp` sets N/Z/C/V per an IEEE ordered comparison, with an + /// unordered result giving C=1, V=1. + fn cond_from_floatcc(&mut self, cc: &FloatCC) -> Cond { + match cc { + FloatCC::Equal => Cond::Eq, + FloatCC::NotEqual => Cond::Ne, + FloatCC::LessThan => Cond::Mi, + FloatCC::LessThanOrEqual => Cond::Ls, + FloatCC::GreaterThan => Cond::Gt, + FloatCC::GreaterThanOrEqual => Cond::Ge, + FloatCC::Ordered => Cond::Vc, + FloatCC::Unordered => Cond::Vs, + FloatCC::UnorderedOrLessThan => Cond::Lt, + FloatCC::UnorderedOrLessThanOrEqual => Cond::Le, + FloatCC::UnorderedOrGreaterThan => Cond::Hi, + FloatCC::UnorderedOrGreaterThanOrEqual => Cond::Hs, + // These need a two-condition sequence and aren't handled yet. + FloatCC::OrderedNotEqual | FloatCC::UnorderedOrEqual => { + unimplemented!("arm32: fcmp condition {cc:?} not yet implemented") + } + } + } + fn cond_from_intcc(&mut self, cc: &IntCC) -> Cond { match cc { IntCC::Equal => Cond::Eq, diff --git a/cranelift/filetests/filetests/isa/arm32/float-cvt.clif b/cranelift/filetests/filetests/isa/arm32/float-cvt.clif new file mode 100644 index 000000000000..2bc07d680ef4 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/float-cvt.clif @@ -0,0 +1,130 @@ +test compile precise-output +target arm + +function %feq(f64, f64) -> i8 { +block0(v0: f64, v1: f64): + v2 = fcmp eq v0, v1 + return v2 +} + +; VCode: +; block0: +; vcmp.f64 d0, d1; vmrs +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; moveq r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vcmp.f64 d0, d1 +; vmrs apsr_nzcv, fpscr +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; moveq r0, r1 +; bx lr + +function %flt_br(f32, f32) -> i32 { +block0(v0: f32, v1: f32): + v2 = fcmp lt v0, v1 + brif v2, block1, block2 +block1: + v3 = iconst.i32 1 + return v3 +block2: + v4 = iconst.i32 0 + return v4 +} + +; VCode: +; block0: +; vcmp.f32 d0, d1; vmrs +; bmi label2; b label1 +; block1: +; mov r0, #0 +; bx lr +; block2: +; mov r0, #1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vcmp.f32 s0, s2 +; vmrs apsr_nzcv, fpscr +; bmi #0x14 +; block1: ; offset 0xc +; mov r0, #0 +; bx lr +; block2: ; offset 0x14 +; mov r0, #1 +; bx lr + +function %promote(f32) -> f64 { +block0(v0: f32): + v1 = fpromote.f64 v0 + return v1 +} + +; VCode: +; block0: +; vcvt.f64.f32 d0, d0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vcvt.f64.f32 d0, s0 +; bx lr + +function %f2i(f64) -> i32 { +block0(v0: f64): + v1 = fcvt_to_sint_sat.i32 v0 + return v1 +} + +; VCode: +; block0: +; vcvt.s32.f64 d2, d0 +; vmov r0, d2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vcvt.s32.f64 s4, d0 +; vmov r0, s4 +; bx lr + +function %i2f(i32) -> f64 { +block0(v0: i32): + v1 = fcvt_from_sint.f64 v0 + return v1 +} + +; VCode: +; block0: +; vmov d2, r0 +; vcvt.f64.s32 d0, d2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmov s4, r0 +; vcvt.f64.s32 d0, s4 +; bx lr + +function %bitcast_f2i(f32) -> i32 { +block0(v0: f32): + v1 = bitcast.i32 v0 + return v1 +} + +; VCode: +; block0: +; vmov r0, d0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmov r0, s0 +; bx lr + From c4960bbcca90bfbb3331b7bb6e33c9e1d2b0c714 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Sat, 4 Jul 2026 11:16:51 +0300 Subject: [PATCH 12/18] arm32: lower fmin/fmax, copysign, and float select fmin/fmax reuse the trick of picking vminnm/vmaxnm when the inputs are ordered and x+y (the propagated NaN) when they aren't, which needs ARMv8. copysign is done with GPR bit twiddling on the sign bit. A scalar-float select becomes a cmp plus a conditional vmov. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/inst.isle | 33 +++++ cranelift/codegen/src/isa/arm32/inst/emit.rs | 31 +++++ cranelift/codegen/src/isa/arm32/inst/mod.rs | 30 +++++ cranelift/codegen/src/isa/arm32/inst/regs.rs | 1 + cranelift/codegen/src/isa/arm32/lower.isle | 21 +++ cranelift/codegen/src/isa/arm32/lower/isle.rs | 67 ++++++++++ .../filetests/isa/arm32/float-minmax.clif | 124 ++++++++++++++++++ 7 files changed, 307 insertions(+) create mode 100644 cranelift/filetests/filetests/isa/arm32/float-minmax.clif diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index 7b4aa7d1f5c1..5a635d6a5857 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -336,6 +336,10 @@ (VcvtToInt (signed bool) (size FpuSize) (rd WritableReg) (rm Reg)) ;; Convert a 32-bit integer (held in the S register `rm`) to a float. (VcvtFromInt (signed bool) (size FpuSize) (rd WritableReg) (rm Reg)) + ;; A conditional float select, emitted as `vmov rd, rm; vmov rd, rn`. + (FpuCSel (cond Cond) (rd WritableReg) (rn Reg) (rm Reg)) + ;; IEEE `vminnm`/`vmaxnm` (number min/max; ARMv8). `max` selects vmaxnm. + (FpuMinMax (max bool) (size FpuSize) (rd WritableReg) (rn Reg) (rm Reg)) ;; `cmp/cmn/tst/teq rn, rm`. (CmpRR (op CmpOp) (rn Reg) (rm Reg)) @@ -569,6 +573,12 @@ (decl cond_from_floatcc (FloatCC) Cond) (extern constructor cond_from_floatcc cond_from_floatcc) +;; `fcopysign` via GPR bit manipulation (clear the sign of `a`, OR in `b`'s). +(decl gen_copysign_f32 (Reg Reg) Reg) +(extern constructor gen_copysign_f32 gen_copysign_f32) +(decl gen_copysign_f64 (Reg Reg) Reg) +(extern constructor gen_copysign_f64 gen_copysign_f64) + ;; `vcmp` + `vmrs` as a side effect (leaves the comparison in the CPSR). (decl vcmp (FpuSize Reg Reg) SideEffectNoResult) (rule (vcmp size rn rm) @@ -625,6 +635,29 @@ (_ Unit (emit (MInst.MovToFpu64 rd (vr_lo regs) (vr_hi regs))))) rd)) +;; Conditional float select: `if_true` when `cond` holds, else `if_false`. +(decl fpu_csel (Cond Reg Reg) Reg) +(rule (fpu_csel cond if_true if_false) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.FpuCSel cond rd if_true if_false)))) + rd)) + +;; `vminnm`/`vmaxnm` (IEEE number min/max, ignoring NaN). +(decl fpu_minmax (bool FpuSize Reg Reg) Reg) +(rule (fpu_minmax is_max size rn rm) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.FpuMinMax is_max size rd rn rm)))) + rd)) + +;; fmin/fmax with WebAssembly NaN propagation: use the IEEE number min/max +;; (which handles signed zeros) when ordered, and `x+y` (a NaN) when unordered. +(decl gen_fminmax (bool FpuSize Reg Reg) Reg) +(rule (gen_fminmax is_max size x y) + (let ((nm Reg (fpu_minmax is_max size x y)) + (nan Reg (fpu_rrr (FpuOp3.Vadd) size x y)) + (_ Unit (emit_side_effect (vcmp size x y)))) + (fpu_csel (Cond.Vs) nan nm))) + ;;;; Shifts ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Shift/rotate by a constant amount. Handles the Cranelift "modulo width" diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index ea0d3b46821d..6bc9f96e1b3c 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -462,6 +462,17 @@ fn enc_vcvt_from_int(signed: bool, size: FpuSize, d: u32, m: u32) -> u32 { (0xeeb8_0a40 | sign | size.size_bit()) | (d << 12) | m } +/// `vmov.f64 dd, dm` — a conditional double-register move. +fn enc_vmov_f64_cond(cond: Cond, d: u32, m: u32) -> u32 { + (cond.bits() << 28) | 0x0eb0_0b40 | (d << 12) | m +} + +/// `vminnm`/`vmaxnm` (IEEE number min/max; ARMv8). These are unconditional. +fn enc_fpu_minmax(max: bool, size: FpuSize, d: u32, n: u32, m: u32) -> u32 { + let op = if max { 0 } else { 0x40 }; + (0xfe80_0a00 | op | size.size_bit()) | (n << 16) | (d << 12) | m +} + fn put_u32(sink: &mut MachBuffer, word: u32) { for b in word.to_le_bytes() { sink.put1(b); @@ -857,6 +868,26 @@ impl MachInstEmit for Inst { let rm = machreg_to_vfp(*rm); put_u32(sink, enc_vcvt_from_int(*signed, *size, rd, rm)); } + Inst::FpuCSel { cond, rd, rn, rm } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rn = machreg_to_vfp(*rn); + let rm = machreg_to_vfp(*rm); + // Default to `if_false`, then conditionally overwrite. + put_u32(sink, enc_fpu_rr(FpuOp2::Vmov, FpuSize::F64, rd, rm)); + put_u32(sink, enc_vmov_f64_cond(*cond, rd, rn)); + } + Inst::FpuMinMax { + max, + size, + rd, + rn, + rm, + } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rn = machreg_to_vfp(*rn); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_fpu_minmax(*max, *size, rd, rn, rm)); + } Inst::CmpRR { op, rn, rm } => { let rn = machreg_to_gpr(*rn); let rm = machreg_to_gpr(*rm); diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index 769c86a36ea0..d4fcaa0608f5 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -193,6 +193,18 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { use_if_virtual(collector, rm); def_if_virtual(collector, rd); } + Inst::FpuMinMax { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::FpuCSel { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + if rd.to_reg().to_real_reg().is_none() { + collector.reg_early_def(rd); + } + } Inst::LdmStm { rn, .. } => use_if_virtual(collector, rn), Inst::LoadEx { rt, rn, .. } | Inst::LoadAcq { rt, rn } => { use_if_virtual(collector, rn); @@ -786,6 +798,24 @@ impl Inst { r(*rm) ) } + Inst::FpuCSel { cond, rd, rn, rm } => { + let rd = r(rd.to_reg()); + alloc::format!("vmov {rd}, {}; vmov{} {rd}, {}", r(*rm), cond.name(), r(*rn)) + } + Inst::FpuMinMax { + max, + size, + rd, + rn, + rm, + } => alloc::format!( + "{}.{} {}, {}, {}", + if *max { "vmaxnm" } else { "vminnm" }, + size.suffix(), + r(rd.to_reg()), + r(*rn), + r(*rm) + ), Inst::CmpRR { op, rn, rm } => { alloc::format!("{} {}, {}", op.name(), r(*rn), r(*rm)) } diff --git a/cranelift/codegen/src/isa/arm32/inst/regs.rs b/cranelift/codegen/src/isa/arm32/inst/regs.rs index 4244172032df..38648cd648f9 100644 --- a/cranelift/codegen/src/isa/arm32/inst/regs.rs +++ b/cranelift/codegen/src/isa/arm32/inst/regs.rs @@ -46,6 +46,7 @@ pub const fn pdreg(enc: u8) -> PReg { /// Get a writable reference to `D`. #[inline] +#[allow(dead_code, reason = "part of the register API, used as the backend grows")] pub fn writable_dreg(enc: u8) -> Writable { Writable::from_reg(dreg(enc)) } diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index 3157e29c5475..131c0bd43b93 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -212,6 +212,27 @@ (rule (lower (fcvt_to_uint_sat $I32 x @ (value_type $F64))) (vcvt_to_int false (FpuSize.F64) (put_in_reg x))) +;; fmin/fmax (WebAssembly NaN-propagating). +(rule (lower (fmin $F32 x y)) + (gen_fminmax false (FpuSize.F32) (put_in_reg x) (put_in_reg y))) +(rule (lower (fmin $F64 x y)) + (gen_fminmax false (FpuSize.F64) (put_in_reg x) (put_in_reg y))) +(rule (lower (fmax $F32 x y)) + (gen_fminmax true (FpuSize.F32) (put_in_reg x) (put_in_reg y))) +(rule (lower (fmax $F64 x y)) + (gen_fminmax true (FpuSize.F64) (put_in_reg x) (put_in_reg y))) + +;; fcopysign. +(rule (lower (fcopysign $F32 x y)) + (gen_copysign_f32 (put_in_reg x) (put_in_reg y))) +(rule (lower (fcopysign $F64 x y)) + (gen_copysign_f64 (put_in_reg x) (put_in_reg y))) + +;; Float-valued select (priority 2 so it wins over the fits_in_32 rule for f32). +(rule 2 (lower (select (ty_scalar_float _) c a b)) + (let ((_ Unit (emit_side_effect (cmp_imm (put_in_reg c) 0)))) + (fpu_csel (Cond.Ne) (put_in_reg a) (put_in_reg b)))) + ;; Bitcasts between the integer and float register files. (rule (lower (bitcast $I32 _ x @ (value_type $F32))) (mov_from_fpu32 (put_in_reg x))) diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs index e648714e320e..5f043883ebc8 100644 --- a/cranelift/codegen/src/isa/arm32/lower/isle.rs +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -58,6 +58,50 @@ impl<'a, 'b> Arm32IsleContext<'a, 'b, MInst, Arm32Backend> { self.alu_raw(ALUOp::Orr, false, a, b) } + /// Emit `op rd, rn, #imm12` (immediate already rotated-encoded). + fn alu_imm_raw(&mut self, op: ALUOp, rn: Reg, imm12: u32) -> Reg { + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::AluRRImm { op, rd, rn, imm12 }); + rd.to_reg() + } + + /// `vmov r, s` — move an S register's bits into a GPR. + fn mov_from_s_raw(&mut self, s: Reg) -> Reg { + let rt = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::MovFromFpu32 { rt, rm: s }); + rt.to_reg() + } + + /// `vmov s, r` — move a GPR's bits into an S register. + fn mov_to_s_raw(&mut self, gpr: Reg) -> Reg { + let rd = self.lower_ctx.alloc_tmp(F32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::MovToFpu32 { rd, rt: gpr }); + rd.to_reg() + } + + /// `vmov rlo, rhi, d` — move a D register's bits into two GPRs. + fn mov_from_d_raw(&mut self, d: Reg) -> (Reg, Reg) { + let lo = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + let hi = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::MovFromFpu64 { + rt_lo: lo, + rt_hi: hi, + rm: d, + }); + (lo.to_reg(), hi.to_reg()) + } + + /// `vmov d, rlo, rhi` — move two GPRs into a D register. + fn mov_to_d_raw(&mut self, lo: Reg, hi: Reg) -> Reg { + let rd = self.lower_ctx.alloc_tmp(F64).only_reg().unwrap(); + self.lower_ctx.emit(MInst::MovToFpu64 { + rd, + rt_lo: lo, + rt_hi: hi, + }); + rd.to_reg() + } + /// Emit `op{s} rd, rn, rm` into a fresh register (which is returned). fn alu_raw(&mut self, op: ALUOp, set_flags: bool, rn: Reg, rm: Reg) -> Reg { let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); @@ -319,6 +363,29 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { } } + /// `fcopysign` for f32: clear the sign bit of `a` and OR in `b`'s sign, + /// working on the raw bits in GPRs. + fn gen_copysign_f32(&mut self, a: Reg, b: Reg) -> Reg { + let sign_mask = encode_rotated_imm(0x8000_0000).unwrap(); + let a_bits = self.mov_from_s_raw(a); + let b_bits = self.mov_from_s_raw(b); + let sign = self.alu_imm_raw(ALUOp::And, b_bits, sign_mask); + let cleared = self.alu_imm_raw(ALUOp::Bic, a_bits, sign_mask); + let res = self.alu_raw(ALUOp::Orr, false, cleared, sign); + self.mov_to_s_raw(res) + } + + /// `fcopysign` for f64: the sign bit lives in the high word. + fn gen_copysign_f64(&mut self, a: Reg, b: Reg) -> Reg { + let sign_mask = encode_rotated_imm(0x8000_0000).unwrap(); + let (a_lo, a_hi) = self.mov_from_d_raw(a); + let (_b_lo, b_hi) = self.mov_from_d_raw(b); + let sign = self.alu_imm_raw(ALUOp::And, b_hi, sign_mask); + let cleared = self.alu_imm_raw(ALUOp::Bic, a_hi, sign_mask); + let res_hi = self.alu_raw(ALUOp::Orr, false, cleared, sign); + self.mov_to_d_raw(a_lo, res_hi) + } + fn cond_from_intcc(&mut self, cc: &IntCC) -> Cond { match cc { IntCC::Equal => Cond::Eq, diff --git a/cranelift/filetests/filetests/isa/arm32/float-minmax.clif b/cranelift/filetests/filetests/isa/arm32/float-minmax.clif new file mode 100644 index 000000000000..5d9092e3da7c --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/float-minmax.clif @@ -0,0 +1,124 @@ +test compile precise-output +target arm + +function %fmin64(f64, f64) -> f64 { +block0(v0: f64, v1: f64): + v2 = fmin v0, v1 + return v2 +} + +; VCode: +; block0: +; vminnm.f64 d3, d0, d1 +; vadd.f64 d5, d0, d1 +; vcmp.f64 d0, d1; vmrs +; vmov d0, d3; vmovvs d0, d5 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vminnm.f64 d3, d0, d1 +; vadd.f64 d5, d0, d1 +; vcmp.f64 d0, d1 +; vmrs apsr_nzcv, fpscr +; vmov.f64 d0, d3 +; vmovvs.f64 d0, d5 +; bx lr + +function %fmax32(f32, f32) -> f32 { +block0(v0: f32, v1: f32): + v2 = fmax v0, v1 + return v2 +} + +; VCode: +; block0: +; vmaxnm.f32 d3, d0, d1 +; vadd.f32 d5, d0, d1 +; vcmp.f32 d0, d1; vmrs +; vmov d0, d3; vmovvs d0, d5 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmaxnm.f32 s6, s0, s2 +; vadd.f32 s10, s0, s2 +; vcmp.f32 s0, s2 +; vmrs apsr_nzcv, fpscr +; vmov.f64 d0, d3 +; vmovvs.f64 d0, d5 +; bx lr + +function %copysign32(f32, f32) -> f32 { +block0(v0: f32, v1: f32): + v2 = fcopysign v0, v1 + return v2 +} + +; VCode: +; block0: +; vmov r3, d0 +; vmov r0, d1 +; and r0, r0, #2147483648 +; bic r1, r3, #2147483648 +; orr r0, r1, r0 +; vmov d0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmov r3, s0 +; vmov r0, s2 +; and r0, r0, #0x80000000 +; bic r1, r3, #0x80000000 +; orr r0, r1, r0 +; vmov s0, r0 +; bx lr + +function %copysign64(f64, f64) -> f64 { +block0(v0: f64, v1: f64): + v2 = fcopysign v0, v1 + return v2 +} + +; VCode: +; block0: +; vmov r3, r0, d0 +; vmov r2, r1, d1 +; and r1, r1, #2147483648 +; bic r0, r0, #2147483648 +; orr r0, r0, r1 +; vmov d0, r3, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmov r3, r0, d0 +; vmov r2, r1, d1 +; and r1, r1, #0x80000000 +; bic r0, r0, #0x80000000 +; orr r0, r0, r1 +; vmov d0, r3, r0 +; bx lr + +function %fsel(i32, f64, f64) -> f64 { +block0(v0: i32, v1: f64, v2: f64): + v3 = select v0, v1, v2 + return v3 +} + +; VCode: +; block0: +; vmov.f64 d6, d0 +; cmp r0, #0 +; vmov d0, d1; vmovne d0, d6 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmov.f64 d6, d0 +; cmp r0, #0 +; vmov.f64 d0, d1 +; vmovne.f64 d0, d6 +; bx lr + From b907db436e93625b7f3263258697ce479e8208c9 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Sat, 4 Jul 2026 03:42:05 +0300 Subject: [PATCH 13/18] arm32: lower br_table with an inline jump table Emits a bounds check against the default target, then an add pc, pc, index, lsl into an inline table of branches. Branch26 range covers the table so no islands are needed. br_table and the call sequences are exempt from the per-instruction worst-case-size check. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/inst.isle | 9 +++ cranelift/codegen/src/isa/arm32/inst/emit.rs | 40 +++++++++++- cranelift/codegen/src/isa/arm32/inst/mod.rs | 21 +++++- cranelift/codegen/src/isa/arm32/lower.isle | 3 + cranelift/codegen/src/isa/arm32/lower/isle.rs | 10 +++ .../filetests/isa/arm32/br_table.clif | 64 +++++++++++++++++++ 6 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/arm32/br_table.clif diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index 5a635d6a5857..40f21cdb41c6 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -193,6 +193,7 @@ (type BoxCallInfo (primitive BoxCallInfo)) (type BoxCallIndInfo (primitive BoxCallIndInfo)) +(type VecMachLabel extern (enum)) ;;;; Instruction formats ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -370,6 +371,10 @@ ;; A conditional branch: `b taken; b not_taken`. (CondBr (cond Cond) (taken MachLabel) (not_taken MachLabel)) + ;; A jump table: bounds-check `index` against the entry count and branch to + ;; `targets[index]`, else to the default (`targets[0]`). + (BrTable (index Reg) (tmp WritableReg) (targets VecMachLabel)) + ;; Pseudo-instruction capturing incoming register arguments in vregs. (Args (args VecArgPair)) ;; Pseudo-instruction moving vregs into the return registers. @@ -783,6 +788,10 @@ (rule (jump_impl label) (SideEffectNoResult.Inst (MInst.Jump label))) +;; A jump table (the first target is the default block). +(decl lower_br_table (Reg MachLabelSlice) Unit) +(extern constructor lower_br_table lower_br_table) + ;;;; Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl gen_call_info (Sig ExternalName CallArgList CallRetList OptionTryCallInfo bool) BoxCallInfo) diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index 6bc9f96e1b3c..4b1ab2ade38b 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -928,6 +928,38 @@ impl MachInstEmit for Inst { sink.add_uncond_branch(sink.cur_offset(), sink.cur_offset() + 4, *dest); put_u32(sink, enc_b(0)); } + Inst::BrTable { + index, + tmp, + targets, + } => { + let (default, entries) = targets.split_first().unwrap(); + let index = machreg_to_gpr(*index); + let n = entries.len() as u32; + + // Bounds check: branch to the default block if `index >=u n`. + if let Some(imm12) = encode_rotated_imm(n) { + put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, index, imm12)); + } else { + let tmp = machreg_to_gpr(tmp.to_reg()); + put_u32(sink, enc_movw(tmp, n & 0xffff)); + if n >> 16 != 0 { + put_u32(sink, enc_movt(tmp, n >> 16)); + } + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, index, tmp)); + } + sink.use_label_at_offset(sink.cur_offset(), *default, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Hs, 0)); + + // Jump into the inline table: `pc` reads as this instruction's + // address + 8, so the table begins one slot after the filler. + put_u32(sink, 0xe08f_f100 | index); // add pc, pc, index, lsl #2 + put_u32(sink, COND_AL | 0x0320_f000); // nop (filler) + for target in entries { + sink.use_label_at_offset(sink.cur_offset(), *target, LabelUse::Branch26); + put_u32(sink, enc_b(0)); + } + } Inst::CondBr { cond, taken, @@ -949,9 +981,15 @@ impl MachInstEmit for Inst { } } + // `BrTable` and calls emit variable-length sequences, so they are + // exempt from the per-instruction worst-case-size check. + let variable_length = matches!( + self, + Inst::BrTable { .. } | Inst::Call { .. } | Inst::CallInd { .. } + ); let end = sink.cur_offset(); debug_assert!( - (end - start) <= Inst::worst_case_size(), + variable_length || (end - start) <= Inst::worst_case_size(), "instruction {self:?} exceeded worst-case size" ); } diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index d4fcaa0608f5..84cfdd0dd268 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -55,6 +55,13 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { | Inst::Push { .. } | Inst::Pop { .. } => {} + Inst::BrTable { index, tmp, .. } => { + use_if_virtual(collector, index); + if tmp.to_reg().to_real_reg().is_none() { + collector.reg_early_def(tmp); + } + } + Inst::MovImm { rd, .. } | Inst::MovRotImm { rd, .. } | Inst::MvnRotImm { rd, .. } @@ -379,7 +386,9 @@ impl MachInst for Inst { fn is_term(&self) -> MachTerminator { match self { Inst::Rets { .. } => MachTerminator::Ret, - Inst::Jump { .. } | Inst::CondBr { .. } => MachTerminator::Branch, + Inst::Jump { .. } | Inst::CondBr { .. } | Inst::BrTable { .. } => { + MachTerminator::Branch + } _ => MachTerminator::None, } } @@ -845,6 +854,16 @@ impl Inst { Inst::Call { info } => alloc::format!("bl {}", info.dest.display(None)), Inst::CallInd { info } => alloc::format!("blx {}", r(info.dest)), Inst::Jump { dest } => alloc::format!("b {}", dest.to_string()), + Inst::BrTable { index, targets, .. } => { + let (default, entries) = targets.split_first().unwrap(); + let entries: Vec = entries.iter().map(|l| l.to_string()).collect(); + alloc::format!( + "br_table {} [{}] default {}", + r(*index), + entries.join(", "), + default.to_string() + ) + } Inst::CondBr { cond, taken, diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index 131c0bd43b93..df9fc2b85de4 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -384,6 +384,9 @@ (rule (lower_branch (jump _) (single_target label)) (emit_side_effect (jump_impl label))) +(rule (lower_branch (br_table index _) targets) + (lower_br_table (put_in_reg index) targets)) + ;; Fuse a 32-bit `icmp` feeding a `brif` into a compare plus conditional branch. (rule 1 (lower_branch (brif (icmp _ cc a @ (value_type (fits_in_32 _)) b) _ _) (two_targets taken not_taken)) (emit_side_effect diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs index 5f043883ebc8..3db14a6911a6 100644 --- a/cranelift/codegen/src/isa/arm32/lower/isle.rs +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -24,6 +24,7 @@ use regalloc2::PReg; type VecArgPair = Vec; type VecRetPair = Vec; +type VecMachLabel = Vec; type BoxCallInfo = Box>; type BoxCallIndInfo = Box>; @@ -123,6 +124,15 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { self.lower_ctx.emit(inst.clone()); } + fn lower_br_table(&mut self, index: Reg, targets: &[MachLabel]) -> Unit { + let tmp = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::BrTable { + index, + tmp, + targets: targets.to_vec(), + }); + } + fn gen_call_info( &mut self, sig: Sig, diff --git a/cranelift/filetests/filetests/isa/arm32/br_table.clif b/cranelift/filetests/filetests/isa/arm32/br_table.clif new file mode 100644 index 000000000000..f942aea13e61 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/br_table.clif @@ -0,0 +1,64 @@ +test compile precise-output +target arm + +function %jt(i32) -> i32 { +block0(v0: i32): + br_table v0, block4, [block1, block2, block3] + +block1: + v1 = iconst.i32 10 + return v1 + +block2: + v2 = iconst.i32 20 + return v2 + +block3: + v3 = iconst.i32 30 + return v3 + +block4: + v4 = iconst.i32 99 + return v4 +} + +; VCode: +; block0: +; mov r1, r0 +; br_table r1 [label3, label2, label1] default label4 +; block1: +; mov r0, #30 +; bx lr +; block2: +; mov r0, #20 +; bx lr +; block3: +; mov r0, #10 +; bx lr +; block4: +; mov r0, #99 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mov r1, r0 +; cmp r1, #3 +; bhs #0x38 +; add pc, pc, r1, lsl #2 +; nop +; b #0x30 +; b #0x28 +; b #0x20 +; block1: ; offset 0x20 +; mov r0, #0x1e +; bx lr +; block2: ; offset 0x28 +; mov r0, #0x14 +; bx lr +; block3: ; offset 0x30 +; mov r0, #0xa +; bx lr +; block4: ; offset 0x38 +; mov r0, #0x63 +; bx lr + From 52b731aac17d2abcfd8f9714dac13e5420e60ce5 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Sat, 4 Jul 2026 07:08:22 +0300 Subject: [PATCH 14/18] arm32: add 32-bit atomics atomic_rmw and atomic_cas lower to ldaex/.../stlex retry loops emitted as single pseudo-instructions, since the register allocator can't be allowed to split the exclusive sequence. The rmw pseudo covers all of the operations; cas branches out on a mismatch. atomic_load/store use lda/stl, which give seq_cst ordering on ARMv8. Only i32 for now. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/inst.isle | 38 +++++ cranelift/codegen/src/isa/arm32/inst/emit.rs | 115 ++++++++++++- cranelift/codegen/src/isa/arm32/inst/mod.rs | 58 +++++++ cranelift/codegen/src/isa/arm32/lower.isle | 11 ++ .../filetests/filetests/isa/arm32/atomic.clif | 161 ++++++++++++++++++ 5 files changed, 382 insertions(+), 1 deletion(-) create mode 100644 cranelift/filetests/filetests/isa/arm32/atomic.clif diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index 40f21cdb41c6..9c431dd02fd0 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -310,6 +310,16 @@ ;; A memory barrier / clear-exclusive. (Barrier (op BarrierOp)) + ;; An atomic read-modify-write loop over a 32-bit word: + ;; loop: ldaex rd, [addr]; tmp1, rd, operand; + ;; stlex tmp2, tmp1, [addr]; cmp tmp2, #0; bne loop + ;; `rd` receives the old value. + (AtomicRmw (op AtomicRmwOp) (rd WritableReg) (addr Reg) (operand Reg) + (tmp1 WritableReg) (tmp2 WritableReg)) + ;; An atomic compare-and-swap loop over a 32-bit word; `rd` receives the + ;; old value. + (AtomicCas (rd WritableReg) (addr Reg) (expected Reg) (new Reg) (tmp WritableReg)) + ;; A three-register VFP op (`vadd`/`vsub`/`vmul`/`vdiv`). (FpuRRR (op FpuOp3) (size FpuSize) (rd WritableReg) (rn Reg) (rm Reg)) ;; A two-register VFP op (`vmov`/`vneg`/`vabs`/`vsqrt`). @@ -533,6 +543,34 @@ (rule (dmb) (SideEffectNoResult.Inst (MInst.Barrier (BarrierOp.Dmb)))) +;;;; Atomics (32-bit) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl atomic_rmw_i32 (AtomicRmwOp Reg Reg) Reg) +(rule (atomic_rmw_i32 op addr operand) + (let ((rd WritableReg (temp_writable_reg $I32)) + (tmp1 WritableReg (temp_writable_reg $I32)) + (tmp2 WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AtomicRmw op rd addr operand tmp1 tmp2)))) + rd)) + +(decl atomic_cas_i32 (Reg Reg Reg) Reg) +(rule (atomic_cas_i32 addr expected new) + (let ((rd WritableReg (temp_writable_reg $I32)) + (tmp WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AtomicCas rd addr expected new tmp)))) + rd)) + +;; Sequentially-consistent load/store via load-acquire / store-release. +(decl atomic_load_i32 (Reg) Reg) +(rule (atomic_load_i32 addr) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.LoadAcq rd addr)))) + rd)) + +(decl atomic_store_i32 (Reg Reg) InstOutput) +(rule (atomic_store_i32 src addr) + (side_effect (SideEffectNoResult.Inst (MInst.StoreRel src addr)))) + ;;;; Floating point ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl fpu_rrr (FpuOp3 FpuSize Reg Reg) Reg) diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index 4b1ab2ade38b..65e768effb2b 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -2,6 +2,7 @@ use crate::binemit::Reloc; use crate::ir; +use crate::ir::AtomicRmwOp; use crate::isa::arm32::inst::*; use crate::settings; use cranelift_control::ControlPlane; @@ -787,6 +788,38 @@ impl MachInstEmit for Inst { put_u32(sink, enc_stl(rt, rn)); } Inst::Barrier { op } => put_u32(sink, op.encoding()), + Inst::AtomicRmw { + op, + rd, + addr, + operand, + tmp1, + tmp2, + } => emit_atomic_rmw( + sink, + state, + *op, + machreg_to_gpr(rd.to_reg()), + machreg_to_gpr(*addr), + machreg_to_gpr(*operand), + machreg_to_gpr(tmp1.to_reg()), + machreg_to_gpr(tmp2.to_reg()), + ), + Inst::AtomicCas { + rd, + addr, + expected, + new, + tmp, + } => emit_atomic_cas( + sink, + state, + machreg_to_gpr(rd.to_reg()), + machreg_to_gpr(*addr), + machreg_to_gpr(*expected), + machreg_to_gpr(*new), + machreg_to_gpr(tmp.to_reg()), + ), Inst::FpuRRR { op, @@ -985,7 +1018,11 @@ impl MachInstEmit for Inst { // exempt from the per-instruction worst-case-size check. let variable_length = matches!( self, - Inst::BrTable { .. } | Inst::Call { .. } | Inst::CallInd { .. } + Inst::BrTable { .. } + | Inst::Call { .. } + | Inst::CallInd { .. } + | Inst::AtomicRmw { .. } + | Inst::AtomicCas { .. } ); let end = sink.cur_offset(); debug_assert!( @@ -1010,6 +1047,82 @@ fn resolve_fpu_amode(mem: &AMode, state: &EmitState) -> (u32, i32) { } } +/// Emit the LDAEX/op/STLEX retry loop for an atomic read-modify-write. +fn emit_atomic_rmw( + sink: &mut MachBuffer, + state: &mut EmitState, + op: AtomicRmwOp, + rd: u32, + addr: u32, + operand: u32, + tmp1: u32, + tmp2: u32, +) { + let loop_lbl = sink.get_label(); + sink.bind_label(loop_lbl, state.ctrl_plane_mut()); + put_u32(sink, enc_load_ex(true, rd, addr)); // ldaex rd, [addr] + + // Compute the new value into tmp1. + let alu = |sink: &mut MachBuffer, op: ALUOp| { + put_u32(sink, enc_dp_reg(op.opcode(), 0, tmp1, rd, operand)); + }; + match op { + AtomicRmwOp::Add => alu(sink, ALUOp::Add), + AtomicRmwOp::Sub => alu(sink, ALUOp::Sub), + AtomicRmwOp::And => alu(sink, ALUOp::And), + AtomicRmwOp::Or => alu(sink, ALUOp::Orr), + AtomicRmwOp::Xor => alu(sink, ALUOp::Eor), + AtomicRmwOp::Nand => { + alu(sink, ALUOp::And); + put_u32(sink, enc_dp_reg(0b1111, 0, tmp1, 0, tmp1)); // mvn tmp1, tmp1 + } + AtomicRmwOp::Xchg => { + put_u32(sink, enc_dp_reg(0b1101, 0, tmp1, 0, operand)); // mov tmp1, operand + } + AtomicRmwOp::Umin | AtomicRmwOp::Umax | AtomicRmwOp::Smin | AtomicRmwOp::Smax => { + let cond = match op { + AtomicRmwOp::Umin => Cond::Lo, + AtomicRmwOp::Umax => Cond::Hi, + AtomicRmwOp::Smin => Cond::Lt, + _ => Cond::Gt, + }; + // tmp1 = operand; cmp rd, operand; tmp1 = rd if `cond`. + put_u32(sink, enc_dp_reg(0b1101, 0, tmp1, 0, operand)); + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd, operand)); + put_u32(sink, enc_mov_cond(cond, tmp1, rd)); + } + } + + put_u32(sink, enc_store_ex(true, tmp2, tmp1, addr)); // stlex tmp2, tmp1, [addr] + put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, tmp2, 0)); // cmp tmp2, #0 + sink.use_label_at_offset(sink.cur_offset(), loop_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne loop +} + +/// Emit the LDAEX/compare/STLEX retry loop for an atomic compare-and-swap. +fn emit_atomic_cas( + sink: &mut MachBuffer, + state: &mut EmitState, + rd: u32, + addr: u32, + expected: u32, + new: u32, + tmp: u32, +) { + let loop_lbl = sink.get_label(); + let done_lbl = sink.get_label(); + sink.bind_label(loop_lbl, state.ctrl_plane_mut()); + put_u32(sink, enc_load_ex(true, rd, addr)); // ldaex rd, [addr] + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd, expected)); // cmp rd, expected + sink.use_label_at_offset(sink.cur_offset(), done_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne done + put_u32(sink, enc_store_ex(true, tmp, new, addr)); // stlex tmp, new, [addr] + put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, tmp, 0)); // cmp tmp, #0 + sink.use_label_at_offset(sink.cur_offset(), loop_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne loop + sink.bind_label(done_lbl, state.ctrl_plane_mut()); +} + enum LoadStore { Load(LoadKind), Store(StoreKind), diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index 84cfdd0dd268..8beff0aa1e0e 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -155,6 +155,39 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { } Inst::Udf { .. } | Inst::Barrier { .. } => {} + Inst::AtomicRmw { + rd, + addr, + operand, + tmp1, + tmp2, + .. + } => { + use_if_virtual(collector, addr); + use_if_virtual(collector, operand); + for d in [rd, tmp1, tmp2] { + if d.to_reg().to_real_reg().is_none() { + collector.reg_early_def(d); + } + } + } + Inst::AtomicCas { + rd, + addr, + expected, + new, + tmp, + } => { + use_if_virtual(collector, addr); + use_if_virtual(collector, expected); + use_if_virtual(collector, new); + for d in [rd, tmp] { + if d.to_reg().to_real_reg().is_none() { + collector.reg_early_def(d); + } + } + } + Inst::FpuRRR { rd, rn, rm, .. } => { use_if_virtual(collector, rn); use_if_virtual(collector, rm); @@ -732,6 +765,31 @@ impl Inst { } Inst::StoreRel { rt, rn } => alloc::format!("stl {}, [{}]", r(*rt), r(*rn)), Inst::Barrier { op } => op.name().to_string(), + Inst::AtomicRmw { + op, + rd, + addr, + operand, + .. + } => alloc::format!( + "atomic_rmw.{op:?} {}, [{}], {}", + r(rd.to_reg()), + r(*addr), + r(*operand) + ), + Inst::AtomicCas { + rd, + addr, + expected, + new, + .. + } => alloc::format!( + "atomic_cas {}, [{}], {}, {}", + r(rd.to_reg()), + r(*addr), + r(*expected), + r(*new) + ), Inst::FpuRRR { op, diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index df9fc2b85de4..30db4e325781 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -130,6 +130,17 @@ (rule (lower (fence)) (side_effect (dmb))) +;;;; Atomics (32-bit) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (atomic_rmw $I32 _flags op addr x)) + (atomic_rmw_i32 op (put_in_reg addr) (put_in_reg x))) +(rule (lower (atomic_cas $I32 _flags addr e x)) + (atomic_cas_i32 (put_in_reg addr) (put_in_reg e) (put_in_reg x))) +(rule (lower (atomic_load $I32 _flags addr)) + (atomic_load_i32 (put_in_reg addr))) +(rule (lower (atomic_store _flags src @ (value_type $I32) addr)) + (atomic_store_i32 (put_in_reg src) (put_in_reg addr))) + ;;;; Floating point ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (f32const _ (u32_from_ieee32 n))) diff --git a/cranelift/filetests/filetests/isa/arm32/atomic.clif b/cranelift/filetests/filetests/isa/arm32/atomic.clif new file mode 100644 index 000000000000..600a9433e76b --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/atomic.clif @@ -0,0 +1,161 @@ +test compile precise-output +target arm + +function %rmw_add(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = atomic_rmw.i32 add v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #0] +; block0: +; mov r2, r0 +; mov r7, r1 +; atomic_rmw.Add r3, [r2], r7 +; mov r0, r3 +; ldr r7, [sp, #0] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp] +; block1: ; offset 0x10 +; mov r2, r0 +; mov r7, r1 +; ldaex r3, [r2] +; add r1, r3, r7 +; stlex r0, r1, [r2] +; cmp r0, #0 +; bne #0x18 +; mov r0, r3 +; ldr r7, [sp] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +function %rmw_umax(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = atomic_rmw.i32 umax v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #0] +; block0: +; mov r2, r0 +; mov r7, r1 +; atomic_rmw.Umax r3, [r2], r7 +; mov r0, r3 +; ldr r7, [sp, #0] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp] +; block1: ; offset 0x10 +; mov r2, r0 +; mov r7, r1 +; ldaex r3, [r2] +; mov r1, r7 +; cmp r3, r7 +; movhi r1, r3 +; stlex r0, r1, [r2] +; cmp r0, #0 +; bne #0x18 +; mov r0, r3 +; ldr r7, [sp] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +function %cas(i32, i32, i32) -> i32 { +block0(v0: i32, v1: i32, v2: i32): + v3 = atomic_cas.i32 v0, v1, v2 + return v3 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #0] +; block0: +; mov r3, r0 +; mov r7, r1 +; atomic_cas r1, [r3], r7, r2 +; mov r0, r1 +; ldr r7, [sp, #0] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp] +; block1: ; offset 0x10 +; mov r3, r0 +; mov r7, r1 +; ldaex r1, [r3] +; cmp r1, r7 +; bne #0x30 +; stlex r0, r2, [r3] +; cmp r0, #0 +; bne #0x18 +; mov r0, r1 +; ldr r7, [sp] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +function %aload(i32) -> i32 { +block0(v0: i32): + v1 = atomic_load.i32 v0 + return v1 +} + +; VCode: +; block0: +; lda r0, [r0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; lda r0, [r0] +; bx lr + +function %astore(i32, i32) { +block0(v0: i32, v1: i32): + atomic_store.i32 v1, v0 + return +} + +; VCode: +; block0: +; stl r1, [r0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; stl r1, [r0] +; bx lr + From 0d5dfedd19d40c4722d9aea1dc3426891f16017b Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Wed, 8 Jul 2026 20:02:12 +0300 Subject: [PATCH 15/18] arm32: support far calls, func_addr, and symbol_value Direct calls to a non-colocated callee had no lowering. A new LoadExtName pseudo materializes a symbol's absolute address inline (ldr rd, [pc]; b .+8; .word ) with an Abs4 relocation, and a far call loads the target that way and calls it indirectly. The same helper also lowers func_addr and symbol_value. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/inst.isle | 17 +++++ cranelift/codegen/src/isa/arm32/inst/emit.rs | 14 ++++ cranelift/codegen/src/isa/arm32/inst/mod.rs | 6 ++ cranelift/codegen/src/isa/arm32/lower.isle | 22 +++++- .../filetests/isa/arm32/far-call.clif | 73 +++++++++++++++++++ 5 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 cranelift/filetests/filetests/isa/arm32/far-call.clif diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index 9c431dd02fd0..9fc60f17d213 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -375,6 +375,13 @@ ;; An indirect call (`blx rm`). (CallInd (info BoxCallIndInfo)) + ;; Load the absolute address of an external symbol into `rd` via an + ;; inline literal: `ldr rd, [pc]; b .+8; .word ` where the + ;; `.word` carries an `Abs4` relocation resolved by the linker. + (LoadExtName (rd WritableReg) + (name BoxExternalName) + (offset i64)) + ;; An unconditional branch to a label (`b`). (Jump (dest MachLabel)) @@ -842,6 +849,16 @@ (rule (call_impl info) (SideEffectNoResult.Inst (MInst.Call info))) +;; Load the absolute address of an external symbol (plus a byte offset) into a +;; fresh register. Used for far/non-colocated calls and symbol addresses. +;; (`box_external_name` is applied implicitly by the `ExternalName -> +;; BoxExternalName` conversion in the prelude.) +(decl load_ext_name (ExternalName i64) Reg) +(rule (load_ext_name name offset) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.LoadExtName rd name offset)))) + rd)) + (decl call_ind_impl (BoxCallIndInfo) SideEffectNoResult) (rule (call_ind_impl info) (SideEffectNoResult.Inst (MInst.CallInd info))) diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index 65e768effb2b..c90191dea15f 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -955,6 +955,19 @@ impl MachInstEmit for Inst { put_u32(sink, enc_blx(rm)); emit_call_epilogue(sink, emit_info, state, info); } + Inst::LoadExtName { rd, name, offset } => { + let rd = machreg_to_gpr(rd.to_reg()); + // `ldr rd, [pc]` reads `pc` as this instruction's address + 8, + // i.e. the inline literal placed two slots ahead. + put_u32(sink, enc_ldr_str_imm(true, false, rd, 15, 0)); + // `b .+8` branches over the literal (its `pc` is already the + // instruction just past the `.word`, so the offset is 0). + put_u32(sink, enc_b(0)); + // The literal: the symbol's absolute address plus `offset`, + // supplied by the linker via an `Abs4` (32-bit absolute) reloc. + sink.add_reloc(Reloc::Abs4, name.as_ref(), *offset); + put_u32(sink, 0); + } Inst::Jump { dest } => { sink.use_label_at_offset(sink.cur_offset(), *dest, LabelUse::Branch26); @@ -1023,6 +1036,7 @@ impl MachInstEmit for Inst { | Inst::CallInd { .. } | Inst::AtomicRmw { .. } | Inst::AtomicCas { .. } + | Inst::LoadExtName { .. } ); let end = sink.cur_offset(); debug_assert!( diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index 8beff0aa1e0e..0a8b4787a249 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -331,6 +331,9 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { } collector.reg_clobbers(info.clobbers); } + Inst::LoadExtName { rd, .. } => { + collector.reg_def(rd); + } Inst::CallInd { info } => { let CallInfo { dest, uses, defs, .. @@ -911,6 +914,9 @@ impl Inst { Inst::Pop { reg_list } => alloc::format!("pop {}", reglist(*reg_list)), Inst::Call { info } => alloc::format!("bl {}", info.dest.display(None)), Inst::CallInd { info } => alloc::format!("blx {}", r(info.dest)), + Inst::LoadExtName { rd, name, offset } => { + alloc::format!("load_ext_name {}, {:?} + {}", r(rd.to_reg()), name, offset) + } Inst::Jump { dest } => alloc::format!("b {}", dest.to_string()), Inst::BrTable { index, targets, .. } => { let (default, entries) = targets.split_first().unwrap(); diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index 30db4e325781..35911cb7a9aa 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -368,7 +368,7 @@ ;;;; Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Direct call to an in-range function (`bl`). -(rule (lower (call (func_ref_data sig_ref name (RelocDistance.Near) patchable) args)) +(rule 1 (lower (call (func_ref_data sig_ref name (RelocDistance.Near) patchable) args)) (let ((output ValueRegsVec (gen_call_output sig_ref)) (abi Sig (abi_sig sig_ref)) (uses CallArgList (gen_call_args abi args)) @@ -377,6 +377,18 @@ (_ Unit (emit_side_effect (call_impl info)))) output)) +;; Direct call to a far/non-colocated function: materialize the callee's +;; absolute address, then call it indirectly (`blx`). +(rule 0 (lower (call (func_ref_data sig_ref name _dist _patchable) args)) + (let ((output ValueRegsVec (gen_call_output sig_ref)) + (abi Sig (abi_sig sig_ref)) + (target Reg (load_ext_name name 0)) + (uses CallArgList (gen_call_args abi args)) + (defs CallRetList (gen_call_rets abi output)) + (info BoxCallIndInfo (gen_call_ind_info abi target uses defs (try_call_none))) + (_ Unit (emit_side_effect (call_ind_impl info)))) + output)) + ;; Indirect call through a register (`blx`). (rule (lower (call_indirect sig_ref ptr args)) (let ((output ValueRegsVec (gen_call_output sig_ref)) @@ -388,6 +400,14 @@ (_ Unit (emit_side_effect (call_ind_impl info)))) output)) +;; Address of a function symbol. +(rule (lower (func_addr _ (func_ref_data _ name _dist _))) + (load_ext_name name 0)) + +;; Address of a data/global symbol (plus a constant byte offset). +(rule (lower (symbol_value _ (symbol_value_data name _dist offset))) + (load_ext_name name offset)) + ;;;; Branches ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl partial lower_branch (Inst MachLabelSlice) Unit) diff --git a/cranelift/filetests/filetests/isa/arm32/far-call.clif b/cranelift/filetests/filetests/isa/arm32/far-call.clif new file mode 100644 index 000000000000..5f7f7fd771a7 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/far-call.clif @@ -0,0 +1,73 @@ +test compile precise-output +target arm + +;; Direct call to a non-colocated (far) function: materialize the callee's +;; absolute address via an inline literal, then call it indirectly. +function %call_far(i32) -> i32 { + fn0 = %g(i32) -> i32 +block0(v0: i32): + v1 = call fn0(v0) + return v1 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; block0: +; load_ext_name r3, TestCase(%g) + 0 +; blx r3 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; block1: ; offset 0x8 +; ldr r3, [pc] +; b #0x14 +; andeq r0, r0, r0 ; reloc_external Abs4 %g 0 +; blx r3 +; pop {fp, lr} +; bx lr + +;; Address of a function symbol. +function %func_addr() -> i32 { + fn0 = %g(i32) -> i32 +block0: + v0 = func_addr.i32 fn0 + return v0 +} + +; VCode: +; block0: +; load_ext_name r0, TestCase(%g) + 0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldr r0, [pc] +; b #0xc +; andeq r0, r0, r0 ; reloc_external Abs4 %g 0 +; bx lr + +;; Address of a data symbol plus a constant byte offset. +function %symbol_addr() -> i32 { + gv0 = symbol %sym+16 +block0: + v0 = symbol_value.i32 gv0 + return v0 +} + +; VCode: +; block0: +; load_ext_name r0, TestCase(%sym) + 16 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldr r0, [pc] +; b #0xc +; andeq r0, r0, r0 ; reloc_external Abs4 %sym 16 +; bx lr + From c225589056530433f62a3e7bbf86067742ba5f90 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Thu, 9 Jul 2026 07:29:25 +0300 Subject: [PATCH 16/18] arm32: extend atomics to i8, i16, and i64 Threads an access size through the exclusive instructions so they encode the byte and halfword forms. Sub-word rmw needs no operand extension for the plain ops (the store truncates), but min/max and cas extend their inputs before comparing. 64-bit is trickier: ldrexd/strexd need consecutive even/odd register pairs, which regalloc2 can't express, so the paired atomics pin every operand to a fixed register the way aarch64 does. rmw uses the adds/adc/subs/sbc carry chains, with min/max selecting both halves on a Z-independent condition. Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/inst.isle | 142 ++++-- cranelift/codegen/src/isa/arm32/inst/args.rs | 47 +- cranelift/codegen/src/isa/arm32/inst/emit.rs | 339 +++++++++++-- .../codegen/src/isa/arm32/inst/emit_tests.rs | 171 ++++++- cranelift/codegen/src/isa/arm32/inst/mod.rs | 204 ++++++-- cranelift/codegen/src/isa/arm32/inst/regs.rs | 15 +- cranelift/codegen/src/isa/arm32/lower.isle | 31 +- cranelift/codegen/src/isa/arm32/lower/isle.rs | 6 +- .../filetests/filetests/isa/arm32/atomic.clif | 461 ++++++++++++++++++ 9 files changed, 1260 insertions(+), 156 deletions(-) diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index 9fc60f17d213..dfe5c48a0ed9 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -133,6 +133,16 @@ (Isb) (Clrex))) +;; Access width for exclusive / acquire-release memory operations. `DWord` +;; (64-bit) uses an even/odd register pair and is only valid on the paired +;; exclusive instructions. +(type AtomicSize + (enum + (Byte) + (Half) + (Word) + (DWord))) + ;; The precision of a VFP floating-point operation. (type FpuSize (enum @@ -297,28 +307,50 @@ ;; Load/store multiple, increment-after (`ldmia`/`stmia rn{!}, {list}`). (LdmStm (load bool) (rn Reg) (writeback bool) (reg_list u32)) - ;; Load-exclusive (`ldrex`) or load-acquire-exclusive (`ldaex`). - (LoadEx (acquire bool) (rt WritableReg) (rn Reg)) - ;; Store-exclusive (`strex`) or store-release-exclusive (`stlex`); `rd` - ;; receives the success flag. - (StoreEx (acquire bool) (rd WritableReg) (rt Reg) (rn Reg)) - ;; Load-acquire (`lda rt, [rn]`). - (LoadAcq (rt WritableReg) (rn Reg)) - ;; Store-release (`stl rt, [rn]`). - (StoreRel (rt Reg) (rn Reg)) + ;; Load-exclusive (`ldrex{b,h}`) or load-acquire-exclusive (`ldaex{b,h}`). + (LoadEx (acquire bool) (size AtomicSize) (rt WritableReg) (rn Reg)) + ;; Store-exclusive (`strex{b,h}`) or store-release-exclusive + ;; (`stlex{b,h}`); `rd` receives the success flag. + (StoreEx (acquire bool) (size AtomicSize) (rd WritableReg) (rt Reg) (rn Reg)) + ;; Load-acquire (`lda{b,h} rt, [rn]`). + (LoadAcq (size AtomicSize) (rt WritableReg) (rn Reg)) + ;; Store-release (`stl{b,h} rt, [rn]`). + (StoreRel (size AtomicSize) (rt Reg) (rn Reg)) ;; A memory barrier / clear-exclusive. (Barrier (op BarrierOp)) - ;; An atomic read-modify-write loop over a 32-bit word: - ;; loop: ldaex rd, [addr]; tmp1, rd, operand; - ;; stlex tmp2, tmp1, [addr]; cmp tmp2, #0; bne loop - ;; `rd` receives the old value. - (AtomicRmw (op AtomicRmwOp) (rd WritableReg) (addr Reg) (operand Reg) - (tmp1 WritableReg) (tmp2 WritableReg)) - ;; An atomic compare-and-swap loop over a 32-bit word; `rd` receives the - ;; old value. - (AtomicCas (rd WritableReg) (addr Reg) (expected Reg) (new Reg) (tmp WritableReg)) + ;; An atomic read-modify-write loop over a `size`-wide location: + ;; loop: ldaex{,b,h} rd, [addr]; tmp1, rd, operand; + ;; stlex{,b,h} tmp2, tmp1, [addr]; cmp tmp2, #0; bne loop + ;; `rd` receives the (zero-extended) old value. + (AtomicRmw (op AtomicRmwOp) (size AtomicSize) (rd WritableReg) (addr Reg) + (operand Reg) (tmp1 WritableReg) (tmp2 WritableReg)) + ;; An atomic compare-and-swap loop over a `size`-wide location; `rd` + ;; receives the (zero-extended) old value. + (AtomicCas (size AtomicSize) (rd WritableReg) (addr Reg) (expected Reg) + (new Reg) (tmp WritableReg)) + + ;; A 64-bit atomic read-modify-write loop using the paired exclusive + ;; instructions (`ldaexd`/`stlexd`). The old value is returned in + ;; (rd_lo, rd_hi). All registers are pinned to a fixed even/odd layout in + ;; the operand collector because A32 exclusive-pair instructions require + ;; consecutive even/odd register pairs, which regalloc2 cannot model. + (AtomicRmw64 (op AtomicRmwOp) (rd_lo WritableReg) (rd_hi WritableReg) + (addr Reg) (operand_lo Reg) (operand_hi Reg) + (tmp_new_lo WritableReg) (tmp_new_hi WritableReg) + (tmp_res WritableReg)) + ;; A 64-bit atomic compare-and-swap loop; the old value is in (rd_lo, rd_hi). + (AtomicCas64 (rd_lo WritableReg) (rd_hi WritableReg) (addr Reg) + (expected_lo Reg) (expected_hi Reg) (new_lo Reg) (new_hi Reg) + (tmp_res WritableReg)) + ;; A 64-bit sequentially-consistent atomic load (`ldaexd` + `clrex`). + (AtomicLoad64 (rd_lo WritableReg) (rd_hi WritableReg) (addr Reg)) + ;; A 64-bit sequentially-consistent atomic store via an `ldaexd`/`stlexd` + ;; retry loop; the loaded value is discarded into (scratch_lo, scratch_hi). + (AtomicStore64 (addr Reg) (src_lo Reg) (src_hi Reg) + (scratch_lo WritableReg) (scratch_hi WritableReg) + (tmp_res WritableReg)) ;; A three-register VFP op (`vadd`/`vsub`/`vmul`/`vdiv`). (FpuRRR (op FpuOp3) (size FpuSize) (rd WritableReg) (rn Reg) (rm Reg)) @@ -550,33 +582,77 @@ (rule (dmb) (SideEffectNoResult.Inst (MInst.Barrier (BarrierOp.Dmb)))) -;;;; Atomics (32-bit) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; Atomics (8/16/32-bit) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(decl atomic_rmw_i32 (AtomicRmwOp Reg Reg) Reg) -(rule (atomic_rmw_i32 op addr operand) +;; Map a scalar integer type to the exclusive-access width. +(decl atomic_size (Type) AtomicSize) +(rule (atomic_size $I8) (AtomicSize.Byte)) +(rule (atomic_size $I16) (AtomicSize.Half)) +(rule (atomic_size $I32) (AtomicSize.Word)) + +(decl atomic_rmw_size (Type AtomicRmwOp Reg Reg) Reg) +(rule (atomic_rmw_size ty op addr operand) (let ((rd WritableReg (temp_writable_reg $I32)) (tmp1 WritableReg (temp_writable_reg $I32)) (tmp2 WritableReg (temp_writable_reg $I32)) - (_ Unit (emit (MInst.AtomicRmw op rd addr operand tmp1 tmp2)))) + (_ Unit (emit (MInst.AtomicRmw op (atomic_size ty) rd addr operand tmp1 tmp2)))) rd)) -(decl atomic_cas_i32 (Reg Reg Reg) Reg) -(rule (atomic_cas_i32 addr expected new) +(decl atomic_cas_size (Type Reg Reg Reg) Reg) +(rule (atomic_cas_size ty addr expected new) (let ((rd WritableReg (temp_writable_reg $I32)) (tmp WritableReg (temp_writable_reg $I32)) - (_ Unit (emit (MInst.AtomicCas rd addr expected new tmp)))) + (_ Unit (emit (MInst.AtomicCas (atomic_size ty) rd addr expected new tmp)))) rd)) ;; Sequentially-consistent load/store via load-acquire / store-release. -(decl atomic_load_i32 (Reg) Reg) -(rule (atomic_load_i32 addr) +(decl atomic_load_size (Type Reg) Reg) +(rule (atomic_load_size ty addr) (let ((rd WritableReg (temp_writable_reg $I32)) - (_ Unit (emit (MInst.LoadAcq rd addr)))) - rd)) - -(decl atomic_store_i32 (Reg Reg) InstOutput) -(rule (atomic_store_i32 src addr) - (side_effect (SideEffectNoResult.Inst (MInst.StoreRel src addr)))) + (_ Unit (emit (MInst.LoadAcq (atomic_size ty) rd addr)))) + rd)) + +(decl atomic_store_size (Type Reg Reg) InstOutput) +(rule (atomic_store_size ty src addr) + (side_effect (SideEffectNoResult.Inst (MInst.StoreRel (atomic_size ty) src addr)))) + +;;;; Atomics (64-bit) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl atomic_rmw_i64 (AtomicRmwOp Reg ValueRegs) ValueRegs) +(rule (atomic_rmw_i64 op addr operand) + (let ((rd_lo WritableReg (temp_writable_reg $I32)) + (rd_hi WritableReg (temp_writable_reg $I32)) + (nl WritableReg (temp_writable_reg $I32)) + (nh WritableReg (temp_writable_reg $I32)) + (res WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AtomicRmw64 op rd_lo rd_hi addr + (vr_lo operand) (vr_hi operand) nl nh res)))) + (value_regs (writable_reg_to_reg rd_lo) (writable_reg_to_reg rd_hi)))) + +(decl atomic_cas_i64 (Reg ValueRegs ValueRegs) ValueRegs) +(rule (atomic_cas_i64 addr expected new) + (let ((rd_lo WritableReg (temp_writable_reg $I32)) + (rd_hi WritableReg (temp_writable_reg $I32)) + (res WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AtomicCas64 rd_lo rd_hi addr + (vr_lo expected) (vr_hi expected) + (vr_lo new) (vr_hi new) res)))) + (value_regs (writable_reg_to_reg rd_lo) (writable_reg_to_reg rd_hi)))) + +(decl atomic_load_i64 (Reg) ValueRegs) +(rule (atomic_load_i64 addr) + (let ((rd_lo WritableReg (temp_writable_reg $I32)) + (rd_hi WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AtomicLoad64 rd_lo rd_hi addr)))) + (value_regs (writable_reg_to_reg rd_lo) (writable_reg_to_reg rd_hi)))) + +(decl atomic_store_i64 (ValueRegs Reg) InstOutput) +(rule (atomic_store_i64 src addr) + (let ((sl WritableReg (temp_writable_reg $I32)) + (sh WritableReg (temp_writable_reg $I32)) + (res WritableReg (temp_writable_reg $I32))) + (side_effect (SideEffectNoResult.Inst + (MInst.AtomicStore64 addr (vr_lo src) (vr_hi src) sl sh res))))) ;;;; Floating point ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/arm32/inst/args.rs b/cranelift/codegen/src/isa/arm32/inst/args.rs index b20949293697..eba41a211350 100644 --- a/cranelift/codegen/src/isa/arm32/inst/args.rs +++ b/cranelift/codegen/src/isa/arm32/inst/args.rs @@ -4,10 +4,53 @@ use crate::isa::arm32::inst::*; use crate::machinst::{OperandVisitor, Reg}; pub use crate::isa::arm32::lower::isle::generated_code::{ - ALUOp, AMode, BarrierOp, BfxOp, BitOp, CmpOp, Cond, DspMul3Op, DspMul4Op, DspMulLOp, ExtAddOp, - ExtOp, FpuOp2, FpuOp3, FpuSize, LoadKind, ParAluOp, PkhOp, QAluOp, SatOp, ShiftOp, StoreKind, + ALUOp, AMode, AtomicSize, BarrierOp, BfxOp, BitOp, CmpOp, Cond, DspMul3Op, DspMul4Op, + DspMulLOp, ExtAddOp, ExtOp, FpuOp2, FpuOp3, FpuSize, LoadKind, ParAluOp, PkhOp, QAluOp, SatOp, + ShiftOp, StoreKind, }; +impl AtomicSize { + /// The instruction-mnemonic suffix for this access width (`ldrex`, + /// `ldrexb`, `ldrexh`, `ldrexd`). + pub(crate) fn suffix(self) -> &'static str { + match self { + AtomicSize::Byte => "b", + AtomicSize::Half => "h", + AtomicSize::Word => "", + AtomicSize::DWord => "d", + } + } + + /// The size field (bits 22:21) of an A32 exclusive/acquire-release + /// instruction encoding. + pub(crate) fn enc_bits(self) -> u32 { + let sz = match self { + AtomicSize::Word => 0b00, + AtomicSize::DWord => 0b01, + AtomicSize::Byte => 0b10, + AtomicSize::Half => 0b11, + }; + sz << 21 + } + + /// True for a sub-word (byte or halfword) access. + pub(crate) fn is_subword(self) -> bool { + matches!(self, AtomicSize::Byte | AtomicSize::Half) + } + + /// The sign/zero-extend op that widens a sub-word value of this size to a + /// full 32-bit register. + pub(crate) fn extend_op(self, signed: bool) -> ExtOp { + match (self, signed) { + (AtomicSize::Byte, false) => ExtOp::Uxtb, + (AtomicSize::Byte, true) => ExtOp::Sxtb, + (AtomicSize::Half, false) => ExtOp::Uxth, + (AtomicSize::Half, true) => ExtOp::Sxth, + _ => unreachable!("extend_op is only valid for sub-word sizes"), + } + } +} + /// A memory address resolved to a concrete base register and either an /// immediate or register offset, ready for encoding. pub(crate) enum ResolvedAMode { diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index c90191dea15f..cbe272bda71d 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -173,7 +173,8 @@ fn enc_sp_adjust(amount: i32) -> u32 { } else { (ALUOp::Add, amount as u32) }; - let imm12 = encode_rotated_imm(mag).expect("arm32 sp adjust not encodable as a single immediate"); + let imm12 = + encode_rotated_imm(mag).expect("arm32 sp adjust not encodable as a single immediate"); enc_dp_imm(op.opcode(), 0, 13, 13, imm12) } @@ -365,26 +366,26 @@ fn enc_ldm_stm(load: bool, rn: u32, writeback: bool, reg_list: u32) -> u32 { COND_AL | base | wb | (rn << 16) | (reg_list & 0xffff) } -/// `ldrex`/`ldaex rt, [rn]`. -fn enc_load_ex(acquire: bool, rt: u32, rn: u32) -> u32 { +/// `ldrex{b,h,d}`/`ldaex{b,h,d} rt, [rn]` (the size field selects the width). +fn enc_load_ex(acquire: bool, size: AtomicSize, rt: u32, rn: u32) -> u32 { let base = if acquire { 0x0190_0e9f } else { 0x0190_0f9f }; - COND_AL | base | (rn << 16) | (rt << 12) + COND_AL | base | size.enc_bits() | (rn << 16) | (rt << 12) } -/// `strex`/`stlex rd, rt, [rn]`. -fn enc_store_ex(acquire: bool, rd: u32, rt: u32, rn: u32) -> u32 { +/// `strex{b,h,d}`/`stlex{b,h,d} rd, rt, [rn]`. +fn enc_store_ex(acquire: bool, size: AtomicSize, rd: u32, rt: u32, rn: u32) -> u32 { let base = if acquire { 0x0180_0e90 } else { 0x0180_0f90 }; - COND_AL | base | (rn << 16) | (rd << 12) | rt + COND_AL | base | size.enc_bits() | (rn << 16) | (rd << 12) | rt } -/// `lda rt, [rn]`. -fn enc_lda(rt: u32, rn: u32) -> u32 { - COND_AL | 0x0190_0c9f | (rn << 16) | (rt << 12) +/// `lda{b,h} rt, [rn]`. +fn enc_lda(size: AtomicSize, rt: u32, rn: u32) -> u32 { + COND_AL | 0x0190_0c9f | size.enc_bits() | (rn << 16) | (rt << 12) } -/// `stl rt, [rn]`. -fn enc_stl(rt: u32, rn: u32) -> u32 { - COND_AL | 0x0180_fc90 | (rn << 16) | rt +/// `stl{b,h} rt, [rn]`. +fn enc_stl(size: AtomicSize, rt: u32, rn: u32) -> u32 { + COND_AL | 0x0180_fc90 | size.enc_bits() | (rn << 16) | rt } /// The hardware encoding number (0-15) of a VFP register. An f32 uses the low @@ -540,12 +541,7 @@ impl MachInstEmit for Inst { let rm = machreg_to_gpr(*rm); put_u32(sink, enc_dp_reg(op.opcode(), 1, rd, rn, rm)); } - Inst::ShiftImm { - op, - rd, - rm, - amount, - } => { + Inst::ShiftImm { op, rd, rm, amount } => { let rd = machreg_to_gpr(rd.to_reg()); let rm = machreg_to_gpr(*rm); put_u32(sink, enc_shift_imm(*op, rd, rm, u32::from(*amount))); @@ -761,13 +757,19 @@ impl MachInstEmit for Inst { let rn = machreg_to_gpr(*rn); put_u32(sink, enc_ldm_stm(*load, rn, *writeback, *reg_list)); } - Inst::LoadEx { acquire, rt, rn } => { + Inst::LoadEx { + acquire, + size, + rt, + rn, + } => { let rt = machreg_to_gpr(rt.to_reg()); let rn = machreg_to_gpr(*rn); - put_u32(sink, enc_load_ex(*acquire, rt, rn)); + put_u32(sink, enc_load_ex(*acquire, *size, rt, rn)); } Inst::StoreEx { acquire, + size, rd, rt, rn, @@ -775,21 +777,22 @@ impl MachInstEmit for Inst { let rd = machreg_to_gpr(rd.to_reg()); let rt = machreg_to_gpr(*rt); let rn = machreg_to_gpr(*rn); - put_u32(sink, enc_store_ex(*acquire, rd, rt, rn)); + put_u32(sink, enc_store_ex(*acquire, *size, rd, rt, rn)); } - Inst::LoadAcq { rt, rn } => { + Inst::LoadAcq { size, rt, rn } => { let rt = machreg_to_gpr(rt.to_reg()); let rn = machreg_to_gpr(*rn); - put_u32(sink, enc_lda(rt, rn)); + put_u32(sink, enc_lda(*size, rt, rn)); } - Inst::StoreRel { rt, rn } => { + Inst::StoreRel { size, rt, rn } => { let rt = machreg_to_gpr(*rt); let rn = machreg_to_gpr(*rn); - put_u32(sink, enc_stl(rt, rn)); + put_u32(sink, enc_stl(*size, rt, rn)); } Inst::Barrier { op } => put_u32(sink, op.encoding()), Inst::AtomicRmw { op, + size, rd, addr, operand, @@ -799,6 +802,7 @@ impl MachInstEmit for Inst { sink, state, *op, + *size, machreg_to_gpr(rd.to_reg()), machreg_to_gpr(*addr), machreg_to_gpr(*operand), @@ -806,6 +810,7 @@ impl MachInstEmit for Inst { machreg_to_gpr(tmp2.to_reg()), ), Inst::AtomicCas { + size, rd, addr, expected, @@ -814,12 +819,82 @@ impl MachInstEmit for Inst { } => emit_atomic_cas( sink, state, + *size, machreg_to_gpr(rd.to_reg()), machreg_to_gpr(*addr), machreg_to_gpr(*expected), machreg_to_gpr(*new), machreg_to_gpr(tmp.to_reg()), ), + Inst::AtomicRmw64 { + op, + rd_lo, + rd_hi, + addr, + operand_lo, + operand_hi, + tmp_new_lo, + tmp_new_hi, + tmp_res, + } => emit_atomic_rmw64( + sink, + state, + *op, + machreg_to_gpr(rd_lo.to_reg()), + machreg_to_gpr(rd_hi.to_reg()), + machreg_to_gpr(*addr), + machreg_to_gpr(*operand_lo), + machreg_to_gpr(*operand_hi), + machreg_to_gpr(tmp_new_lo.to_reg()), + machreg_to_gpr(tmp_new_hi.to_reg()), + machreg_to_gpr(tmp_res.to_reg()), + ), + Inst::AtomicCas64 { + rd_lo, + rd_hi, + addr, + expected_lo, + expected_hi, + new_lo, + new_hi, + tmp_res, + } => emit_atomic_cas64( + sink, + state, + machreg_to_gpr(rd_lo.to_reg()), + machreg_to_gpr(rd_hi.to_reg()), + machreg_to_gpr(*addr), + machreg_to_gpr(*expected_lo), + machreg_to_gpr(*expected_hi), + machreg_to_gpr(*new_lo), + machreg_to_gpr(*new_hi), + machreg_to_gpr(tmp_res.to_reg()), + ), + Inst::AtomicLoad64 { rd_lo, rd_hi, addr } => { + let rd_lo = machreg_to_gpr(rd_lo.to_reg()); + debug_assert_eq!(machreg_to_gpr(rd_hi.to_reg()), rd_lo + 1); + let addr = machreg_to_gpr(*addr); + // ldaexd rd_lo, rd_hi, [addr]; clrex + put_u32(sink, enc_load_ex(true, AtomicSize::DWord, rd_lo, addr)); + put_u32(sink, BarrierOp::Clrex.encoding()); + } + Inst::AtomicStore64 { + addr, + src_lo, + src_hi, + scratch_lo, + scratch_hi, + tmp_res, + } => emit_atomic_store64( + sink, + state, + machreg_to_gpr(*addr), + machreg_to_gpr(*src_lo), + machreg_to_gpr(*src_hi), + machreg_to_gpr(scratch_lo.to_reg()), + machreg_to_gpr(scratch_hi.to_reg()), + machreg_to_gpr(tmp_res.to_reg()), + ), Inst::FpuRRR { op, @@ -1036,6 +1111,9 @@ impl MachInstEmit for Inst { | Inst::CallInd { .. } | Inst::AtomicRmw { .. } | Inst::AtomicCas { .. } + | Inst::AtomicRmw64 { .. } + | Inst::AtomicCas64 { .. } + | Inst::AtomicStore64 { .. } | Inst::LoadExtName { .. } ); let end = sink.cur_offset(); @@ -1062,10 +1140,16 @@ fn resolve_fpu_amode(mem: &AMode, state: &EmitState) -> (u32, i32) { } /// Emit the LDAEX/op/STLEX retry loop for an atomic read-modify-write. +/// +/// The load zero-extends a sub-word value into `rd`; the store truncates. For +/// add/sub/and/or/xor/nand/xchg the low byte/halfword of the arithmetic is +/// correct regardless of the operand's upper bits, so no extension is needed. +/// The min/max comparisons do extend both inputs to the access width. fn emit_atomic_rmw( sink: &mut MachBuffer, state: &mut EmitState, op: AtomicRmwOp, + size: AtomicSize, rd: u32, addr: u32, operand: u32, @@ -1074,7 +1158,7 @@ fn emit_atomic_rmw( ) { let loop_lbl = sink.get_label(); sink.bind_label(loop_lbl, state.ctrl_plane_mut()); - put_u32(sink, enc_load_ex(true, rd, addr)); // ldaex rd, [addr] + put_u32(sink, enc_load_ex(true, size, rd, addr)); // ldaex{,b,h} rd, [addr] // Compute the new value into tmp1. let alu = |sink: &mut MachBuffer, op: ALUOp| { @@ -1094,20 +1178,32 @@ fn emit_atomic_rmw( put_u32(sink, enc_dp_reg(0b1101, 0, tmp1, 0, operand)); // mov tmp1, operand } AtomicRmwOp::Umin | AtomicRmwOp::Umax | AtomicRmwOp::Smin | AtomicRmwOp::Smax => { + let signed = matches!(op, AtomicRmwOp::Smin | AtomicRmwOp::Smax); let cond = match op { AtomicRmwOp::Umin => Cond::Lo, AtomicRmwOp::Umax => Cond::Hi, AtomicRmwOp::Smin => Cond::Lt, _ => Cond::Gt, }; - // tmp1 = operand; cmp rd, operand; tmp1 = rd if `cond`. - put_u32(sink, enc_dp_reg(0b1101, 0, tmp1, 0, operand)); - put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd, operand)); - put_u32(sink, enc_mov_cond(cond, tmp1, rd)); + if size.is_subword() { + // Extend both inputs to 32 bits so the comparison is correct, + // using tmp1/tmp2 as scratch before selecting the result. + let ext = size.extend_op(signed).template(); + put_u32(sink, enc_op_rd_rm(ext, tmp1, rd)); // ext tmp1, rd + put_u32(sink, enc_op_rd_rm(ext, tmp2, operand)); // ext tmp2, operand + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, tmp1, tmp2)); + put_u32(sink, enc_dp_reg(0b1101, 0, tmp1, 0, operand)); // mov tmp1, operand + put_u32(sink, enc_mov_cond(cond, tmp1, rd)); // tmp1 = rd if cond + } else { + // tmp1 = operand; cmp rd, operand; tmp1 = rd if `cond`. + put_u32(sink, enc_dp_reg(0b1101, 0, tmp1, 0, operand)); + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd, operand)); + put_u32(sink, enc_mov_cond(cond, tmp1, rd)); + } } } - put_u32(sink, enc_store_ex(true, tmp2, tmp1, addr)); // stlex tmp2, tmp1, [addr] + put_u32(sink, enc_store_ex(true, size, tmp2, tmp1, addr)); // stlex{,b,h} tmp2, tmp1, [addr] put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, tmp2, 0)); // cmp tmp2, #0 sink.use_label_at_offset(sink.cur_offset(), loop_lbl, LabelUse::Branch26); put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne loop @@ -1117,6 +1213,7 @@ fn emit_atomic_rmw( fn emit_atomic_cas( sink: &mut MachBuffer, state: &mut EmitState, + size: AtomicSize, rd: u32, addr: u32, expected: u32, @@ -1126,17 +1223,175 @@ fn emit_atomic_cas( let loop_lbl = sink.get_label(); let done_lbl = sink.get_label(); sink.bind_label(loop_lbl, state.ctrl_plane_mut()); - put_u32(sink, enc_load_ex(true, rd, addr)); // ldaex rd, [addr] - put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd, expected)); // cmp rd, expected + put_u32(sink, enc_load_ex(true, size, rd, addr)); // ldaex{,b,h} rd, [addr] + // `rd` is zero-extended; compare it against a zero-extended `expected` so + // only the accessed byte/halfword participates. + if size.is_subword() { + let ext = size.extend_op(false).template(); + put_u32(sink, enc_op_rd_rm(ext, tmp, expected)); // uxt tmp, expected + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd, tmp)); // cmp rd, tmp + } else { + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd, expected)); // cmp rd, expected + } sink.use_label_at_offset(sink.cur_offset(), done_lbl, LabelUse::Branch26); put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne done - put_u32(sink, enc_store_ex(true, tmp, new, addr)); // stlex tmp, new, [addr] + put_u32(sink, enc_store_ex(true, size, tmp, new, addr)); // stlex{,b,h} tmp, new, [addr] put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, tmp, 0)); // cmp tmp, #0 sink.use_label_at_offset(sink.cur_offset(), loop_lbl, LabelUse::Branch26); put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne loop sink.bind_label(done_lbl, state.ctrl_plane_mut()); } +/// Emit the LDAEXD/op/STLEXD retry loop for a 64-bit atomic read-modify-write. +/// The old value is loaded into (rd_lo, rd_hi) and the new value computed into +/// (nl, nh) before the paired store-exclusive. +fn emit_atomic_rmw64( + sink: &mut MachBuffer, + state: &mut EmitState, + op: AtomicRmwOp, + rd_lo: u32, + rd_hi: u32, + addr: u32, + op_lo: u32, + op_hi: u32, + nl: u32, + nh: u32, + res: u32, +) { + debug_assert_eq!(rd_hi, rd_lo + 1, "ldaexd needs a consecutive pair"); + debug_assert_eq!(nh, nl + 1, "stlexd needs a consecutive pair"); + const MOV: u32 = 0b1101; + const MVN: u32 = 0b1111; + let loop_lbl = sink.get_label(); + sink.bind_label(loop_lbl, state.ctrl_plane_mut()); + put_u32(sink, enc_load_ex(true, AtomicSize::DWord, rd_lo, addr)); // ldaexd rd_lo,rd_hi,[addr] + + // Compute the new 64-bit value into (nl, nh). + let dp = |sink: &mut MachBuffer, opcode: u32, s: u32, rd: u32, rn: u32, rm: u32| { + put_u32(sink, enc_dp_reg(opcode, s, rd, rn, rm)); + }; + match op { + AtomicRmwOp::Add => { + dp(sink, ALUOp::Add.opcode(), 1, nl, rd_lo, op_lo); // adds nl, rd_lo, op_lo + dp(sink, ALUOp::Adc.opcode(), 0, nh, rd_hi, op_hi); // adc nh, rd_hi, op_hi + } + AtomicRmwOp::Sub => { + dp(sink, ALUOp::Sub.opcode(), 1, nl, rd_lo, op_lo); // subs nl, rd_lo, op_lo + dp(sink, ALUOp::Sbc.opcode(), 0, nh, rd_hi, op_hi); // sbc nh, rd_hi, op_hi + } + AtomicRmwOp::And | AtomicRmwOp::Nand => { + dp(sink, ALUOp::And.opcode(), 0, nl, rd_lo, op_lo); + dp(sink, ALUOp::And.opcode(), 0, nh, rd_hi, op_hi); + if let AtomicRmwOp::Nand = op { + dp(sink, MVN, 0, nl, 0, nl); // mvn nl, nl + dp(sink, MVN, 0, nh, 0, nh); // mvn nh, nh + } + } + AtomicRmwOp::Or => { + dp(sink, ALUOp::Orr.opcode(), 0, nl, rd_lo, op_lo); + dp(sink, ALUOp::Orr.opcode(), 0, nh, rd_hi, op_hi); + } + AtomicRmwOp::Xor => { + dp(sink, ALUOp::Eor.opcode(), 0, nl, rd_lo, op_lo); + dp(sink, ALUOp::Eor.opcode(), 0, nh, rd_hi, op_hi); + } + AtomicRmwOp::Xchg => { + dp(sink, MOV, 0, nl, 0, op_lo); // mov nl, op_lo + dp(sink, MOV, 0, nh, 0, op_hi); // mov nh, op_hi + } + AtomicRmwOp::Umin | AtomicRmwOp::Umax | AtomicRmwOp::Smin | AtomicRmwOp::Smax => { + // Set flags for `old - operand` (64-bit) via subs/sbcs, using + // (nl, nh) as scratch. The chosen condition is Z-independent (the + // sbcs Z flag reflects only the high word), so equality never + // affects the result — which is correct, since min/max of equal + // values is that value either way. + let cond = match op { + AtomicRmwOp::Umin => Cond::Lo, // old Cond::Hs, // old >=u operand + AtomicRmwOp::Smin => Cond::Lt, // old Cond::Ge, // old >=s operand + }; + dp(sink, ALUOp::Sub.opcode(), 1, nl, rd_lo, op_lo); // subs nl, rd_lo, op_lo + dp(sink, ALUOp::Sbc.opcode(), 1, nh, rd_hi, op_hi); // sbcs nh, rd_hi, op_hi + dp(sink, MOV, 0, nl, 0, op_lo); // nl = operand (default) + dp(sink, MOV, 0, nh, 0, op_hi); + put_u32(sink, enc_mov_cond(cond, nl, rd_lo)); // nl = rd_lo if cond + put_u32(sink, enc_mov_cond(cond, nh, rd_hi)); // nh = rd_hi if cond + } + } + + put_u32(sink, enc_store_ex(true, AtomicSize::DWord, res, nl, addr)); // stlexd res, nl,nh,[addr] + put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, res, 0)); // cmp res, #0 + sink.use_label_at_offset(sink.cur_offset(), loop_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne loop +} + +/// Emit the LDAEXD/compare/STLEXD retry loop for a 64-bit compare-and-swap. +fn emit_atomic_cas64( + sink: &mut MachBuffer, + state: &mut EmitState, + rd_lo: u32, + rd_hi: u32, + addr: u32, + exp_lo: u32, + exp_hi: u32, + new_lo: u32, + new_hi: u32, + res: u32, +) { + debug_assert_eq!(rd_hi, rd_lo + 1, "ldaexd needs a consecutive pair"); + debug_assert_eq!(new_hi, new_lo + 1, "stlexd needs a consecutive pair"); + let loop_lbl = sink.get_label(); + let done_lbl = sink.get_label(); + sink.bind_label(loop_lbl, state.ctrl_plane_mut()); + put_u32(sink, enc_load_ex(true, AtomicSize::DWord, rd_lo, addr)); // ldaexd rd_lo,rd_hi,[addr] + // Both halves must match `expected`. + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd_lo, exp_lo)); // cmp rd_lo, exp_lo + sink.use_label_at_offset(sink.cur_offset(), done_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne done + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd_hi, exp_hi)); // cmp rd_hi, exp_hi + sink.use_label_at_offset(sink.cur_offset(), done_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne done + put_u32( + sink, + enc_store_ex(true, AtomicSize::DWord, res, new_lo, addr), + ); // stlexd res,new_lo,new_hi + put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, res, 0)); // cmp res, #0 + sink.use_label_at_offset(sink.cur_offset(), loop_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne loop + sink.bind_label(done_lbl, state.ctrl_plane_mut()); +} + +/// Emit the LDAEXD/STLEXD retry loop for a 64-bit atomic store. The load is +/// required to arm the exclusive monitor; its result is discarded. +fn emit_atomic_store64( + sink: &mut MachBuffer, + state: &mut EmitState, + addr: u32, + src_lo: u32, + src_hi: u32, + scratch_lo: u32, + scratch_hi: u32, + res: u32, +) { + debug_assert_eq!( + scratch_hi, + scratch_lo + 1, + "ldaexd needs a consecutive pair" + ); + debug_assert_eq!(src_hi, src_lo + 1, "stlexd needs a consecutive pair"); + let loop_lbl = sink.get_label(); + sink.bind_label(loop_lbl, state.ctrl_plane_mut()); + put_u32(sink, enc_load_ex(true, AtomicSize::DWord, scratch_lo, addr)); // ldaexd scratch,[addr] + put_u32( + sink, + enc_store_ex(true, AtomicSize::DWord, res, src_lo, addr), + ); // stlexd res, src,[addr] + put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, res, 0)); // cmp res, #0 + sink.use_label_at_offset(sink.cur_offset(), loop_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne loop +} + enum LoadStore { Load(LoadKind), Store(StoreKind), @@ -1155,16 +1410,12 @@ fn emit_load_store( let word = match op { LoadStore::Load(LoadKind::Word) => enc_ldr_str_imm(true, false, rt, base, offset), LoadStore::Load(LoadKind::UByte) => enc_ldr_str_imm(true, true, rt, base, offset), - LoadStore::Load(LoadKind::SByte) => { - enc_ldrh_strh_imm(true, 1, 0, rt, base, offset) - } - LoadStore::Load(LoadKind::UHalf) => { - enc_ldrh_strh_imm(true, 0, 1, rt, base, offset) - } - LoadStore::Load(LoadKind::SHalf) => { - enc_ldrh_strh_imm(true, 1, 1, rt, base, offset) + LoadStore::Load(LoadKind::SByte) => enc_ldrh_strh_imm(true, 1, 0, rt, base, offset), + LoadStore::Load(LoadKind::UHalf) => enc_ldrh_strh_imm(true, 0, 1, rt, base, offset), + LoadStore::Load(LoadKind::SHalf) => enc_ldrh_strh_imm(true, 1, 1, rt, base, offset), + LoadStore::Store(StoreKind::Word) => { + enc_ldr_str_imm(false, false, rt, base, offset) } - LoadStore::Store(StoreKind::Word) => enc_ldr_str_imm(false, false, rt, base, offset), LoadStore::Store(StoreKind::Byte) => enc_ldr_str_imm(false, true, rt, base, offset), LoadStore::Store(StoreKind::Half) => { enc_ldrh_strh_imm(false, 0, 1, rt, base, offset) diff --git a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs index 633d4fe04a9d..7a5aaf489c04 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs @@ -547,15 +547,42 @@ fn parallel_add_sub() { // All 36 members must encode to distinct, well-formed words that share the // parallel add/sub encoding skeleton. let all = [ - ParAluOp::Sadd8, ParAluOp::Sadd16, ParAluOp::Ssub8, ParAluOp::Ssub16, - ParAluOp::Sasx, ParAluOp::Ssax, ParAluOp::Qadd8, ParAluOp::Qadd16, - ParAluOp::Qsub8, ParAluOp::Qsub16, ParAluOp::Qasx, ParAluOp::Qsax, - ParAluOp::Shadd8, ParAluOp::Shadd16, ParAluOp::Shsub8, ParAluOp::Shsub16, - ParAluOp::Shasx, ParAluOp::Shsax, ParAluOp::Uadd8, ParAluOp::Uadd16, - ParAluOp::Usub8, ParAluOp::Usub16, ParAluOp::Uasx, ParAluOp::Usax, - ParAluOp::Uqadd8, ParAluOp::Uqadd16, ParAluOp::Uqsub8, ParAluOp::Uqsub16, - ParAluOp::Uqasx, ParAluOp::Uqsax, ParAluOp::Uhadd8, ParAluOp::Uhadd16, - ParAluOp::Uhsub8, ParAluOp::Uhsub16, ParAluOp::Uhasx, ParAluOp::Uhsax, + ParAluOp::Sadd8, + ParAluOp::Sadd16, + ParAluOp::Ssub8, + ParAluOp::Ssub16, + ParAluOp::Sasx, + ParAluOp::Ssax, + ParAluOp::Qadd8, + ParAluOp::Qadd16, + ParAluOp::Qsub8, + ParAluOp::Qsub16, + ParAluOp::Qasx, + ParAluOp::Qsax, + ParAluOp::Shadd8, + ParAluOp::Shadd16, + ParAluOp::Shsub8, + ParAluOp::Shsub16, + ParAluOp::Shasx, + ParAluOp::Shsax, + ParAluOp::Uadd8, + ParAluOp::Uadd16, + ParAluOp::Usub8, + ParAluOp::Usub16, + ParAluOp::Uasx, + ParAluOp::Usax, + ParAluOp::Uqadd8, + ParAluOp::Uqadd16, + ParAluOp::Uqsub8, + ParAluOp::Uqsub16, + ParAluOp::Uqasx, + ParAluOp::Uqsax, + ParAluOp::Uhadd8, + ParAluOp::Uhadd16, + ParAluOp::Uhsub8, + ParAluOp::Uhsub16, + ParAluOp::Uhasx, + ParAluOp::Uhsax, ]; let mut seen = alloc::collections::BTreeSet::new(); for op in all { @@ -654,6 +681,7 @@ fn memory_multiple_and_exclusive() { assert_eq!( u32_le(Inst::LoadEx { acquire: false, + size: AtomicSize::Word, rt: writable_xreg(1), rn: xreg(0), }), @@ -662,14 +690,34 @@ fn memory_multiple_and_exclusive() { assert_eq!( u32_le(Inst::LoadEx { acquire: true, + size: AtomicSize::Word, rt: writable_xreg(1), rn: xreg(0), }), 0xe190_1e9f // ldaex r1, [r0] ); + assert_eq!( + u32_le(Inst::LoadEx { + acquire: true, + size: AtomicSize::Byte, + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe1d0_1e9f // ldaexb r1, [r0] + ); + assert_eq!( + u32_le(Inst::LoadEx { + acquire: true, + size: AtomicSize::Half, + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe1f0_1e9f // ldaexh r1, [r0] + ); assert_eq!( u32_le(Inst::StoreEx { acquire: false, + size: AtomicSize::Word, rd: writable_xreg(0), rt: xreg(1), rn: xreg(2), @@ -679,26 +727,81 @@ fn memory_multiple_and_exclusive() { assert_eq!( u32_le(Inst::StoreEx { acquire: true, + size: AtomicSize::Word, rd: writable_xreg(0), rt: xreg(1), rn: xreg(2), }), 0xe182_0e91 // stlex r0, r1, [r2] ); + assert_eq!( + u32_le(Inst::StoreEx { + acquire: true, + size: AtomicSize::Byte, + rd: writable_xreg(0), + rt: xreg(1), + rn: xreg(2), + }), + 0xe1c2_0e91 // stlexb r0, r1, [r2] + ); + assert_eq!( + u32_le(Inst::StoreEx { + acquire: true, + size: AtomicSize::Half, + rd: writable_xreg(0), + rt: xreg(1), + rn: xreg(2), + }), + 0xe1e2_0e91 // stlexh r0, r1, [r2] + ); assert_eq!( u32_le(Inst::LoadAcq { + size: AtomicSize::Word, rt: writable_xreg(1), rn: xreg(0), }), 0xe190_1c9f // lda r1, [r0] ); + assert_eq!( + u32_le(Inst::LoadAcq { + size: AtomicSize::Byte, + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe1d0_1c9f // ldab r1, [r0] + ); + assert_eq!( + u32_le(Inst::LoadAcq { + size: AtomicSize::Half, + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe1f0_1c9f // ldah r1, [r0] + ); assert_eq!( u32_le(Inst::StoreRel { + size: AtomicSize::Word, rt: xreg(1), rn: xreg(0), }), 0xe180_fc91 // stl r1, [r0] ); + assert_eq!( + u32_le(Inst::StoreRel { + size: AtomicSize::Byte, + rt: xreg(1), + rn: xreg(0), + }), + 0xe1c0_fc91 // stlb r1, [r0] + ); + assert_eq!( + u32_le(Inst::StoreRel { + size: AtomicSize::Half, + rt: xreg(1), + rn: xreg(0), + }), + 0xe1e0_fc91 // stlh r1, [r0] + ); } #[test] @@ -739,7 +842,10 @@ fn memory() { assert_eq!( u32_le(Inst::Load { rt: writable_xreg(0), - mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, kind: LoadKind::Word, }), 0xe591_0004 // ldr r0, [r1, #4] @@ -747,7 +853,10 @@ fn memory() { assert_eq!( u32_le(Inst::Store { rt: xreg(0), - mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, kind: StoreKind::Word, }), 0xe581_0004 // str r0, [r1, #4] @@ -755,7 +864,10 @@ fn memory() { assert_eq!( u32_le(Inst::Load { rt: writable_xreg(0), - mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, kind: LoadKind::UByte, }), 0xe5d1_0004 // ldrb r0, [r1, #4] @@ -763,7 +875,10 @@ fn memory() { assert_eq!( u32_le(Inst::Store { rt: xreg(0), - mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, kind: StoreKind::Byte, }), 0xe5c1_0004 // strb r0, [r1, #4] @@ -771,7 +886,10 @@ fn memory() { assert_eq!( u32_le(Inst::Load { rt: writable_xreg(0), - mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, kind: LoadKind::UHalf, }), 0xe1d1_00b4 // ldrh r0, [r1, #4] @@ -779,7 +897,10 @@ fn memory() { assert_eq!( u32_le(Inst::Store { rt: xreg(0), - mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, kind: StoreKind::Half, }), 0xe1c1_00b4 // strh r0, [r1, #4] @@ -787,7 +908,10 @@ fn memory() { assert_eq!( u32_le(Inst::Load { rt: writable_xreg(0), - mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, kind: LoadKind::SByte, }), 0xe1d1_00d4 // ldrsb r0, [r1, #4] @@ -795,7 +919,10 @@ fn memory() { assert_eq!( u32_le(Inst::Load { rt: writable_xreg(0), - mem: AMode::RegOffset { rn: xreg(1), offset: 4 }, + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, kind: LoadKind::SHalf, }), 0xe1d1_00f4 // ldrsh r0, [r1, #4] @@ -804,7 +931,10 @@ fn memory() { assert_eq!( u32_le(Inst::Load { rt: writable_xreg(0), - mem: AMode::RegReg { rn: xreg(1), rm: xreg(2) }, + mem: AMode::RegReg { + rn: xreg(1), + rm: xreg(2) + }, kind: LoadKind::Word, }), 0xe791_0002 // ldr r0, [r1, r2] @@ -812,7 +942,10 @@ fn memory() { assert_eq!( u32_le(Inst::Store { rt: xreg(0), - mem: AMode::RegReg { rn: xreg(1), rm: xreg(2) }, + mem: AMode::RegReg { + rn: xreg(1), + rm: xreg(2) + }, kind: StoreKind::Word, }), 0xe781_0002 // str r0, [r1, r2] diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index 0a8b4787a249..41b0dd4b5638 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -177,6 +177,7 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { expected, new, tmp, + .. } => { use_if_virtual(collector, addr); use_if_virtual(collector, expected); @@ -188,6 +189,72 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { } } + // The 64-bit atomics pin every operand to a fixed even/odd register + // layout: r0/r1 (old value), r2/r3 (new/store value), r4 (addr), + // r5/r6 (operand/expected), r7 (store-exclusive result). A32's paired + // exclusive instructions require consecutive even/odd pairs, which + // regalloc2 cannot express, so the pairs must be fixed physical + // registers. + Inst::AtomicRmw64 { + rd_lo, + rd_hi, + addr, + operand_lo, + operand_hi, + tmp_new_lo, + tmp_new_hi, + tmp_res, + .. + } => { + collector.reg_fixed_use(addr, xreg(4)); + collector.reg_fixed_use(operand_lo, xreg(5)); + collector.reg_fixed_use(operand_hi, xreg(6)); + collector.reg_fixed_def(rd_lo, xreg(0)); + collector.reg_fixed_def(rd_hi, xreg(1)); + collector.reg_fixed_def(tmp_new_lo, xreg(2)); + collector.reg_fixed_def(tmp_new_hi, xreg(3)); + collector.reg_fixed_def(tmp_res, xreg(7)); + } + Inst::AtomicCas64 { + rd_lo, + rd_hi, + addr, + expected_lo, + expected_hi, + new_lo, + new_hi, + tmp_res, + } => { + collector.reg_fixed_use(addr, xreg(4)); + collector.reg_fixed_use(expected_lo, xreg(5)); + collector.reg_fixed_use(expected_hi, xreg(6)); + collector.reg_fixed_use(new_lo, xreg(2)); + collector.reg_fixed_use(new_hi, xreg(3)); + collector.reg_fixed_def(rd_lo, xreg(0)); + collector.reg_fixed_def(rd_hi, xreg(1)); + collector.reg_fixed_def(tmp_res, xreg(7)); + } + Inst::AtomicLoad64 { rd_lo, rd_hi, addr } => { + collector.reg_fixed_use(addr, xreg(4)); + collector.reg_fixed_def(rd_lo, xreg(0)); + collector.reg_fixed_def(rd_hi, xreg(1)); + } + Inst::AtomicStore64 { + addr, + src_lo, + src_hi, + scratch_lo, + scratch_hi, + tmp_res, + } => { + collector.reg_fixed_use(addr, xreg(4)); + collector.reg_fixed_use(src_lo, xreg(2)); + collector.reg_fixed_use(src_hi, xreg(3)); + collector.reg_fixed_def(scratch_lo, xreg(0)); + collector.reg_fixed_def(scratch_hi, xreg(1)); + collector.reg_fixed_def(tmp_res, xreg(7)); + } + Inst::FpuRRR { rd, rn, rm, .. } => { use_if_virtual(collector, rn); use_if_virtual(collector, rm); @@ -246,7 +313,7 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { } } Inst::LdmStm { rn, .. } => use_if_virtual(collector, rn), - Inst::LoadEx { rt, rn, .. } | Inst::LoadAcq { rt, rn } => { + Inst::LoadEx { rt, rn, .. } | Inst::LoadAcq { rt, rn, .. } => { use_if_virtual(collector, rn); def_if_virtual(collector, rt); } @@ -255,7 +322,7 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { use_if_virtual(collector, rn); def_if_virtual(collector, rd); } - Inst::StoreRel { rt, rn } => { + Inst::StoreRel { rt, rn, .. } => { use_if_virtual(collector, rt); use_if_virtual(collector, rn); } @@ -559,43 +626,20 @@ impl Inst { Inst::AluRRRFlags { op, rd, rn, rm } => { alloc::format!("{}s {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) } - Inst::ShiftImm { - op, - rd, - rm, - amount, - } => { + Inst::ShiftImm { op, rd, rm, amount } => { alloc::format!("{} {}, {}, #{}", op.name(), r(rd.to_reg()), r(*rm), amount) } Inst::ShiftReg { op, rd, rm, rs } => { - alloc::format!( - "{} {}, {}, {}", - op.name(), - r(rd.to_reg()), - r(*rm), - r(*rs) - ) + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rm), r(*rs)) } Inst::Mul { rd, rn, rm } => { alloc::format!("mul {}, {}, {}", r(rd.to_reg()), r(*rn), r(*rm)) } Inst::Mla { rd, rn, rm, ra } => { - alloc::format!( - "mla {}, {}, {}, {}", - r(rd.to_reg()), - r(*rn), - r(*rm), - r(*ra) - ) + alloc::format!("mla {}, {}, {}, {}", r(rd.to_reg()), r(*rn), r(*rm), r(*ra)) } Inst::Mls { rd, rn, rm, ra } => { - alloc::format!( - "mls {}, {}, {}, {}", - r(rd.to_reg()), - r(*rn), - r(*rm), - r(*ra) - ) + alloc::format!("mls {}, {}, {}, {}", r(rd.to_reg()), r(*rn), r(*rm), r(*ra)) } Inst::Umull { rd_lo, @@ -745,54 +789,120 @@ impl Inst { names.join(", ") ) } - Inst::LoadEx { acquire, rt, rn } => alloc::format!( - "{} {}, [{}]", + Inst::LoadEx { + acquire, + size, + rt, + rn, + } => alloc::format!( + "{}{} {}, [{}]", if *acquire { "ldaex" } else { "ldrex" }, + size.suffix(), r(rt.to_reg()), r(*rn) ), Inst::StoreEx { acquire, + size, rd, rt, rn, } => alloc::format!( - "{} {}, {}, [{}]", + "{}{} {}, {}, [{}]", if *acquire { "stlex" } else { "strex" }, + size.suffix(), r(rd.to_reg()), r(*rt), r(*rn) ), - Inst::LoadAcq { rt, rn } => { - alloc::format!("lda {}, [{}]", r(rt.to_reg()), r(*rn)) + Inst::LoadAcq { size, rt, rn } => { + alloc::format!("lda{} {}, [{}]", size.suffix(), r(rt.to_reg()), r(*rn)) + } + Inst::StoreRel { size, rt, rn } => { + alloc::format!("stl{} {}, [{}]", size.suffix(), r(*rt), r(*rn)) } - Inst::StoreRel { rt, rn } => alloc::format!("stl {}, [{}]", r(*rt), r(*rn)), Inst::Barrier { op } => op.name().to_string(), Inst::AtomicRmw { op, + size, rd, addr, operand, .. } => alloc::format!( - "atomic_rmw.{op:?} {}, [{}], {}", + "atomic_rmw.{op:?}{} {}, [{}], {}", + size.suffix(), r(rd.to_reg()), r(*addr), r(*operand) ), Inst::AtomicCas { + size, rd, addr, expected, new, .. } => alloc::format!( - "atomic_cas {}, [{}], {}, {}", + "atomic_cas{} {}, [{}], {}, {}", + size.suffix(), r(rd.to_reg()), r(*addr), r(*expected), r(*new) ), + Inst::AtomicRmw64 { + op, + rd_lo, + rd_hi, + addr, + operand_lo, + operand_hi, + .. + } => alloc::format!( + "atomic_rmw.{op:?}.i64 {}, {}, [{}], {}, {}", + r(rd_lo.to_reg()), + r(rd_hi.to_reg()), + r(*addr), + r(*operand_lo), + r(*operand_hi) + ), + Inst::AtomicCas64 { + rd_lo, + rd_hi, + addr, + expected_lo, + expected_hi, + new_lo, + new_hi, + .. + } => alloc::format!( + "atomic_cas.i64 {}, {}, [{}], {}, {}, {}, {}", + r(rd_lo.to_reg()), + r(rd_hi.to_reg()), + r(*addr), + r(*expected_lo), + r(*expected_hi), + r(*new_lo), + r(*new_hi) + ), + Inst::AtomicLoad64 { rd_lo, rd_hi, addr } => alloc::format!( + "atomic_load.i64 {}, {}, [{}]", + r(rd_lo.to_reg()), + r(rd_hi.to_reg()), + r(*addr) + ), + Inst::AtomicStore64 { + addr, + src_lo, + src_hi, + .. + } => alloc::format!( + "atomic_store.i64 {}, {}, [{}]", + r(*src_lo), + r(*src_hi), + r(*addr) + ), Inst::FpuRRR { op, @@ -831,13 +941,22 @@ impl Inst { alloc::format!("vmov {}, {}", r(rt.to_reg()), r(*rm)) } Inst::MovFromFpu64 { rt_lo, rt_hi, rm } => { - alloc::format!("vmov {}, {}, {}", r(rt_lo.to_reg()), r(rt_hi.to_reg()), r(*rm)) + alloc::format!( + "vmov {}, {}, {}", + r(rt_lo.to_reg()), + r(rt_hi.to_reg()), + r(*rm) + ) } Inst::VcmpMrs { size, rn, rm } => { alloc::format!("vcmp.{} {}, {}; vmrs", size.suffix(), r(*rn), r(*rm)) } Inst::VcvtFF { to_f64, rd, rm } => { - let (dst, src) = if *to_f64 { ("f64", "f32") } else { ("f32", "f64") }; + let (dst, src) = if *to_f64 { + ("f64", "f32") + } else { + ("f32", "f64") + }; alloc::format!("vcvt.{dst}.{src} {}, {}", r(rd.to_reg()), r(*rm)) } Inst::VcvtToInt { @@ -870,7 +989,12 @@ impl Inst { } Inst::FpuCSel { cond, rd, rn, rm } => { let rd = r(rd.to_reg()); - alloc::format!("vmov {rd}, {}; vmov{} {rd}, {}", r(*rm), cond.name(), r(*rn)) + alloc::format!( + "vmov {rd}, {}; vmov{} {rd}, {}", + r(*rm), + cond.name(), + r(*rn) + ) } Inst::FpuMinMax { max, diff --git a/cranelift/codegen/src/isa/arm32/inst/regs.rs b/cranelift/codegen/src/isa/arm32/inst/regs.rs index 38648cd648f9..ea7f0ce7e812 100644 --- a/cranelift/codegen/src/isa/arm32/inst/regs.rs +++ b/cranelift/codegen/src/isa/arm32/inst/regs.rs @@ -46,7 +46,10 @@ pub const fn pdreg(enc: u8) -> PReg { /// Get a writable reference to `D`. #[inline] -#[allow(dead_code, reason = "part of the register API, used as the backend grows")] +#[allow( + dead_code, + reason = "part of the register API, used as the backend grows" +)] pub fn writable_dreg(enc: u8) -> Writable { Writable::from_reg(dreg(enc)) } @@ -93,14 +96,20 @@ pub fn writable_stack_reg() -> Writable { /// Link register (r14). #[inline] -#[allow(dead_code, reason = "part of the register API, used as the backend grows")] +#[allow( + dead_code, + reason = "part of the register API, used as the backend grows" +)] pub fn link_reg() -> Reg { xreg(14) } /// Writable link register. #[inline] -#[allow(dead_code, reason = "part of the register API, used as the backend grows")] +#[allow( + dead_code, + reason = "part of the register API, used as the backend grows" +)] pub fn writable_link_reg() -> Writable { Writable::from_reg(link_reg()) } diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index 35911cb7a9aa..916c86ce3823 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -130,16 +130,27 @@ (rule (lower (fence)) (side_effect (dmb))) -;;;; Atomics (32-bit) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -(rule (lower (atomic_rmw $I32 _flags op addr x)) - (atomic_rmw_i32 op (put_in_reg addr) (put_in_reg x))) -(rule (lower (atomic_cas $I32 _flags addr e x)) - (atomic_cas_i32 (put_in_reg addr) (put_in_reg e) (put_in_reg x))) -(rule (lower (atomic_load $I32 _flags addr)) - (atomic_load_i32 (put_in_reg addr))) -(rule (lower (atomic_store _flags src @ (value_type $I32) addr)) - (atomic_store_i32 (put_in_reg src) (put_in_reg addr))) +;;;; Atomics (8/16/32-bit) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (atomic_rmw (fits_in_32 ty) _flags op addr x)) + (atomic_rmw_size ty op (put_in_reg addr) (put_in_reg x))) +(rule (lower (atomic_cas (fits_in_32 ty) _flags addr e x)) + (atomic_cas_size ty (put_in_reg addr) (put_in_reg e) (put_in_reg x))) +(rule (lower (atomic_load (fits_in_32 ty) _flags addr)) + (atomic_load_size ty (put_in_reg addr))) +(rule (lower (atomic_store _flags src @ (value_type (fits_in_32 ty)) addr)) + (atomic_store_size ty (put_in_reg src) (put_in_reg addr))) + +;; 64-bit atomics use the paired exclusive instructions (priority 4 so `$I64` +;; wins over the `fits_in_32` rules, which ISLE can't prove disjoint). +(rule 4 (lower (atomic_rmw $I64 _flags op addr x)) + (atomic_rmw_i64 op (put_in_reg addr) (put_in_regs x))) +(rule 4 (lower (atomic_cas $I64 _flags addr e x)) + (atomic_cas_i64 (put_in_reg addr) (put_in_regs e) (put_in_regs x))) +(rule 4 (lower (atomic_load $I64 _flags addr)) + (atomic_load_i64 (put_in_reg addr))) +(rule 4 (lower (atomic_store _flags src @ (value_type $I64) addr)) + (atomic_store_i64 (put_in_regs src) (put_in_reg addr))) ;;;; Floating point ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs index 3db14a6911a6..03bb0d209ef2 100644 --- a/cranelift/codegen/src/isa/arm32/lower/isle.rs +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -114,7 +114,6 @@ impl<'a, 'b> Arm32IsleContext<'a, 'b, MInst, Arm32Backend> { self.lower_ctx.emit(inst); rd.to_reg() } - } impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { @@ -181,10 +180,7 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { } else if let Some(imm12) = encode_rotated_imm(!val) { MInst::MvnRotImm { rd, imm12 } } else if val >> 16 == 0 { - MInst::Movw { - rd, - imm16: val, - } + MInst::Movw { rd, imm16: val } } else { MInst::MovImm { rd, diff --git a/cranelift/filetests/filetests/isa/arm32/atomic.clif b/cranelift/filetests/filetests/isa/arm32/atomic.clif index 600a9433e76b..4e49df5d752a 100644 --- a/cranelift/filetests/filetests/isa/arm32/atomic.clif +++ b/cranelift/filetests/filetests/isa/arm32/atomic.clif @@ -159,3 +159,464 @@ block0(v0: i32, v1: i32): ; stl r1, [r0] ; bx lr +;; Sub-word: a simple op needs no extension (the store truncates). +function %rmw_add_i8(i32, i8) -> i8 { +block0(v0: i32, v1: i8): + v2 = atomic_rmw.i8 add v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #0] +; block0: +; mov r2, r0 +; mov r7, r1 +; atomic_rmw.Addb r3, [r2], r7 +; mov r0, r3 +; ldr r7, [sp, #0] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp] +; block1: ; offset 0x10 +; mov r2, r0 +; mov r7, r1 +; ldaexb r3, [r2] +; add r1, r3, r7 +; stlexb r0, r1, [r2] +; cmp r0, #0 +; bne #0x18 +; mov r0, r3 +; ldr r7, [sp] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +;; Sub-word signed min: both inputs are sign-extended before the compare. +function %rmw_smax_i8(i32, i8) -> i8 { +block0(v0: i32, v1: i8): + v2 = atomic_rmw.i8 smax v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #0] +; block0: +; mov r2, r0 +; mov r7, r1 +; atomic_rmw.Smaxb r3, [r2], r7 +; mov r0, r3 +; ldr r7, [sp, #0] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp] +; block1: ; offset 0x10 +; mov r2, r0 +; mov r7, r1 +; ldaexb r3, [r2] +; sxtb r1, r3 +; sxtb r0, r7 +; cmp r1, r0 +; mov r1, r7 +; movgt r1, r3 +; stlexb r0, r1, [r2] +; cmp r0, #0 +; bne #0x18 +; mov r0, r3 +; ldr r7, [sp] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +;; Sub-word unsigned min: both inputs are zero-extended before the compare. +function %rmw_umin_i16(i32, i16) -> i16 { +block0(v0: i32, v1: i16): + v2 = atomic_rmw.i16 umin v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #0] +; block0: +; mov r2, r0 +; mov r7, r1 +; atomic_rmw.Uminh r3, [r2], r7 +; mov r0, r3 +; ldr r7, [sp, #0] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp] +; block1: ; offset 0x10 +; mov r2, r0 +; mov r7, r1 +; ldaexh r3, [r2] +; uxth r1, r3 +; uxth r0, r7 +; cmp r1, r0 +; mov r1, r7 +; movlo r1, r3 +; stlexh r0, r1, [r2] +; cmp r0, #0 +; bne #0x18 +; mov r0, r3 +; ldr r7, [sp] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +;; Sub-word compare-and-swap: `expected` is zero-extended for the compare. +function %cas_i16(i32, i16, i16) -> i16 { +block0(v0: i32, v1: i16, v2: i16): + v3 = atomic_cas.i16 v0, v1, v2 + return v3 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #0] +; block0: +; mov r3, r0 +; mov r7, r1 +; atomic_cash r1, [r3], r7, r2 +; mov r0, r1 +; ldr r7, [sp, #0] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp] +; block1: ; offset 0x10 +; mov r3, r0 +; mov r7, r1 +; ldaexh r1, [r3] +; uxth r0, r7 +; cmp r1, r0 +; bne #0x34 +; stlexh r0, r2, [r3] +; cmp r0, #0 +; bne #0x18 +; mov r0, r1 +; ldr r7, [sp] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +function %aload_i8(i32) -> i8 { +block0(v0: i32): + v1 = atomic_load.i8 v0 + return v1 +} + +; VCode: +; block0: +; ldab r0, [r0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldab r0, [r0] +; bx lr + +function %astore_i16(i32, i16) { +block0(v0: i32, v1: i16): + atomic_store.i16 v1, v0 + return +} + +; VCode: +; block0: +; stlh r1, [r0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; stlh r1, [r0] +; bx lr + +;; 64-bit: paired exclusive loop with a carry chain. All registers are pinned +;; to a fixed even/odd layout (r0/r1 old, r2/r3 new, r4 addr, r5/r6 operand, +;; r7 result) because A32 `ldrexd`/`strexd` require consecutive pairs. +function %rmw_add_i64(i32, i64) -> i64 { +block0(v0: i32, v1: i64): + v2 = atomic_rmw.i64 add v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #16 +; str r4, [sp, #0] +; str r5, [sp, #4] +; str r6, [sp, #8] +; str r7, [sp, #12] +; block0: +; mov r4, r0 +; mov r5, r2 +; mov r6, r3 +; atomic_rmw.Add.i64 r0, r1, [r4], r5, r6 +; ldr r4, [sp, #0] +; ldr r5, [sp, #4] +; ldr r6, [sp, #8] +; ldr r7, [sp, #12] +; add sp, sp, #16 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #0x10 +; str r4, [sp] +; str r5, [sp, #4] +; str r6, [sp, #8] +; str r7, [sp, #0xc] +; block1: ; offset 0x1c +; mov r4, r0 +; mov r5, r2 +; mov r6, r3 +; ldaexd r0, r1, [r4] +; adds r2, r0, r5 +; adc r3, r1, r6 +; stlexd r7, r2, r3, [r4] +; cmp r7, #0 +; bne #0x28 +; ldr r4, [sp] +; ldr r5, [sp, #4] +; ldr r6, [sp, #8] +; ldr r7, [sp, #0xc] +; add sp, sp, #0x10 +; pop {fp, lr} +; bx lr + +;; 64-bit signed max: a subs/sbcs compare then a conditional select of both +;; halves. +function %rmw_smax_i64(i32, i64) -> i64 { +block0(v0: i32, v1: i64): + v2 = atomic_rmw.i64 smax v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #16 +; str r4, [sp, #0] +; str r5, [sp, #4] +; str r6, [sp, #8] +; str r7, [sp, #12] +; block0: +; mov r4, r0 +; mov r5, r2 +; mov r6, r3 +; atomic_rmw.Smax.i64 r0, r1, [r4], r5, r6 +; ldr r4, [sp, #0] +; ldr r5, [sp, #4] +; ldr r6, [sp, #8] +; ldr r7, [sp, #12] +; add sp, sp, #16 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #0x10 +; str r4, [sp] +; str r5, [sp, #4] +; str r6, [sp, #8] +; str r7, [sp, #0xc] +; block1: ; offset 0x1c +; mov r4, r0 +; mov r5, r2 +; mov r6, r3 +; ldaexd r0, r1, [r4] +; subs r2, r0, r5 +; sbcs r3, r1, r6 +; mov r2, r5 +; mov r3, r6 +; movge r2, r0 +; movge r3, r1 +; stlexd r7, r2, r3, [r4] +; cmp r7, #0 +; bne #0x28 +; ldr r4, [sp] +; ldr r5, [sp, #4] +; ldr r6, [sp, #8] +; ldr r7, [sp, #0xc] +; add sp, sp, #0x10 +; pop {fp, lr} +; bx lr + +function %cas_i64(i32, i64, i64) -> i64 { +block0(v0: i32, v1: i64, v2: i64): + v3 = atomic_cas.i64 v0, v1, v2 + return v3 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #16 +; str r4, [sp, #0] +; str r5, [sp, #4] +; str r6, [sp, #8] +; str r7, [sp, #12] +; block0: +; mov r5, r2 +; mov r6, r3 +; ldr r2, [incoming_arg, #8] +; ldr r3, [incoming_arg, #4] +; mov r4, r0 +; atomic_cas.i64 r0, r1, [r4], r5, r6, r2, r3 +; ldr r4, [sp, #0] +; ldr r5, [sp, #4] +; ldr r6, [sp, #8] +; ldr r7, [sp, #12] +; add sp, sp, #16 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #0x10 +; str r4, [sp] +; str r5, [sp, #4] +; str r6, [sp, #8] +; str r7, [sp, #0xc] +; block1: ; offset 0x1c +; mov r5, r2 +; mov r6, r3 +; ldr r2, [sp, #0x18] +; ldr r3, [sp, #0x1c] +; mov r4, r0 +; ldaexd r0, r1, [r4] +; cmp r0, r5 +; bne #0x50 +; cmp r1, r6 +; bne #0x50 +; stlexd r7, r2, r3, [r4] +; cmp r7, #0 +; bne #0x30 +; ldr r4, [sp] +; ldr r5, [sp, #4] +; ldr r6, [sp, #8] +; ldr r7, [sp, #0xc] +; add sp, sp, #0x10 +; pop {fp, lr} +; bx lr + +function %aload_i64(i32) -> i64 { +block0(v0: i32): + v1 = atomic_load.i64 v0 + return v1 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r4, [sp, #0] +; block0: +; mov r4, r0 +; atomic_load.i64 r0, r1, [r4] +; ldr r4, [sp, #0] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r4, [sp] +; block1: ; offset 0x10 +; mov r4, r0 +; ldaexd r0, r1, [r4] +; clrex +; ldr r4, [sp] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +function %astore_i64(i32, i64) { +block0(v0: i32, v1: i64): + atomic_store.i64 v1, v0 + return +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r4, [sp, #0] +; str r7, [sp, #4] +; block0: +; mov r4, r0 +; atomic_store.i64 r2, r3, [r4] +; ldr r4, [sp, #0] +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r4, [sp] +; str r7, [sp, #4] +; block1: ; offset 0x14 +; mov r4, r0 +; ldaexd r0, r1, [r4] +; stlexd r7, r2, r3, [r4] +; cmp r7, #0 +; bne #0x18 +; ldr r4, [sp] +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + From 63f37db0098f16dc0a2f3843732abdc1eff56b87 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Thu, 9 Jul 2026 20:51:18 +0300 Subject: [PATCH 17/18] fix StoreEx flag name (release, not acquire) Signed-off-by: Obei Sideg --- cranelift/codegen/src/isa/arm32/inst.isle | 2 +- cranelift/codegen/src/isa/arm32/inst/emit.rs | 8 ++++---- cranelift/codegen/src/isa/arm32/inst/emit_tests.rs | 8 ++++---- cranelift/codegen/src/isa/arm32/inst/mod.rs | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index dfe5c48a0ed9..e60e8e8f6139 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -311,7 +311,7 @@ (LoadEx (acquire bool) (size AtomicSize) (rt WritableReg) (rn Reg)) ;; Store-exclusive (`strex{b,h}`) or store-release-exclusive ;; (`stlex{b,h}`); `rd` receives the success flag. - (StoreEx (acquire bool) (size AtomicSize) (rd WritableReg) (rt Reg) (rn Reg)) + (StoreEx (release bool) (size AtomicSize) (rd WritableReg) (rt Reg) (rn Reg)) ;; Load-acquire (`lda{b,h} rt, [rn]`). (LoadAcq (size AtomicSize) (rt WritableReg) (rn Reg)) ;; Store-release (`stl{b,h} rt, [rn]`). diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index cbe272bda71d..b84391fc8546 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -373,8 +373,8 @@ fn enc_load_ex(acquire: bool, size: AtomicSize, rt: u32, rn: u32) -> u32 { } /// `strex{b,h,d}`/`stlex{b,h,d} rd, rt, [rn]`. -fn enc_store_ex(acquire: bool, size: AtomicSize, rd: u32, rt: u32, rn: u32) -> u32 { - let base = if acquire { 0x0180_0e90 } else { 0x0180_0f90 }; +fn enc_store_ex(release: bool, size: AtomicSize, rd: u32, rt: u32, rn: u32) -> u32 { + let base = if release { 0x0180_0e90 } else { 0x0180_0f90 }; COND_AL | base | size.enc_bits() | (rn << 16) | (rd << 12) | rt } @@ -768,7 +768,7 @@ impl MachInstEmit for Inst { put_u32(sink, enc_load_ex(*acquire, *size, rt, rn)); } Inst::StoreEx { - acquire, + release, size, rd, rt, @@ -777,7 +777,7 @@ impl MachInstEmit for Inst { let rd = machreg_to_gpr(rd.to_reg()); let rt = machreg_to_gpr(*rt); let rn = machreg_to_gpr(*rn); - put_u32(sink, enc_store_ex(*acquire, *size, rd, rt, rn)); + put_u32(sink, enc_store_ex(*release, *size, rd, rt, rn)); } Inst::LoadAcq { size, rt, rn } => { let rt = machreg_to_gpr(rt.to_reg()); diff --git a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs index 7a5aaf489c04..bec102adaac2 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs @@ -716,7 +716,7 @@ fn memory_multiple_and_exclusive() { ); assert_eq!( u32_le(Inst::StoreEx { - acquire: false, + release: false, size: AtomicSize::Word, rd: writable_xreg(0), rt: xreg(1), @@ -726,7 +726,7 @@ fn memory_multiple_and_exclusive() { ); assert_eq!( u32_le(Inst::StoreEx { - acquire: true, + release: true, size: AtomicSize::Word, rd: writable_xreg(0), rt: xreg(1), @@ -736,7 +736,7 @@ fn memory_multiple_and_exclusive() { ); assert_eq!( u32_le(Inst::StoreEx { - acquire: true, + release: true, size: AtomicSize::Byte, rd: writable_xreg(0), rt: xreg(1), @@ -746,7 +746,7 @@ fn memory_multiple_and_exclusive() { ); assert_eq!( u32_le(Inst::StoreEx { - acquire: true, + release: true, size: AtomicSize::Half, rd: writable_xreg(0), rt: xreg(1), diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index 41b0dd4b5638..952494eb0471 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -802,14 +802,14 @@ impl Inst { r(*rn) ), Inst::StoreEx { - acquire, + release, size, rd, rt, rn, } => alloc::format!( "{}{} {}, {}, [{}]", - if *acquire { "stlex" } else { "strex" }, + if *release { "stlex" } else { "strex" }, size.suffix(), r(rd.to_reg()), r(*rt), From 4147b883cf603c63ad1e7cd45adf9add904da0d6 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Thu, 9 Jul 2026 21:53:06 +0300 Subject: [PATCH 18/18] arm32: fix a bunch of bugs Signed-off-by: Obei Sideg --- cranelift/codegen/meta/src/isa/mod.rs | 6 +- cranelift/codegen/src/isa/arm32/abi.rs | 60 ++++-- cranelift/codegen/src/isa/arm32/inst.isle | 40 ++-- cranelift/codegen/src/isa/arm32/inst/emit.rs | 9 + cranelift/codegen/src/isa/arm32/inst/mod.rs | 14 +- cranelift/codegen/src/isa/arm32/lower.isle | 74 ++++++-- cranelift/codegen/src/isa/arm32/lower/isle.rs | 88 ++++++++- cranelift/codegen/src/isa/mod.rs | 6 +- cranelift/codegen/src/machinst/lower.rs | 1 + .../filetests/filetests/isa/arm32/atomic.clif | 176 +++++++++--------- .../filetests/filetests/isa/arm32/div.clif | 19 ++ .../filetests/filetests/isa/arm32/float.clif | 58 ++++++ .../filetests/filetests/isa/arm32/i64.clif | 16 +- .../filetests/filetests/isa/arm32/icmp.clif | 82 ++++++++ .../filetests/filetests/isa/arm32/memory.clif | 48 +++++ .../filetests/filetests/isa/arm32/select.clif | 41 ++++ 16 files changed, 593 insertions(+), 145 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/arm32/icmp.clif diff --git a/cranelift/codegen/meta/src/isa/mod.rs b/cranelift/codegen/meta/src/isa/mod.rs index 350f3a082160..08974c98966d 100644 --- a/cranelift/codegen/meta/src/isa/mod.rs +++ b/cranelift/codegen/meta/src/isa/mod.rs @@ -34,7 +34,11 @@ impl Isa { pub fn from_arch(arch: &str) -> Option { match arch { "aarch64" => Some(Isa::Arm64), - x if x.starts_with("arm") || x.starts_with("thumb") => Some(Isa::Arm32), + // 32-bit ARM / Thumb, but not the 64-bit `arm64*` spellings + // (e.g. `arm64ec`), which are AArch64. + x if (x.starts_with("arm") || x.starts_with("thumb")) && !x.starts_with("arm64") => { + Some(Isa::Arm32) + } "s390x" => Some(Isa::S390x), x if ["x86_64", "i386", "i586", "i686"].contains(&x) => Some(Isa::X86), "riscv64" | "riscv64gc" | "riscv64imac" => Some(Isa::Riscv64), diff --git a/cranelift/codegen/src/isa/arm32/abi.rs b/cranelift/codegen/src/isa/arm32/abi.rs index 2aa7248a8c24..c5e72d8cb38a 100644 --- a/cranelift/codegen/src/isa/arm32/abi.rs +++ b/cranelift/codegen/src/isa/arm32/abi.rs @@ -231,18 +231,36 @@ impl ABIMachineSpec for Arm32MachineDeps { } fn gen_load_base_offset(into_reg: Writable, base: Reg, offset: i32, _ty: Type) -> Inst { - Inst::Load { - rt: into_reg, - mem: AMode::RegOffset { rn: base, offset }, - kind: LoadKind::Word, + let mem = AMode::RegOffset { rn: base, offset }; + if into_reg.to_reg().class() == RegClass::Float { + Inst::FpuLoad { + size: FpuSize::F64, + rd: into_reg, + mem, + } + } else { + Inst::Load { + rt: into_reg, + mem, + kind: LoadKind::Word, + } } } fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, _ty: Type) -> Inst { - Inst::Store { - rt: from_reg, - mem: AMode::RegOffset { rn: base, offset }, - kind: StoreKind::Word, + let mem = AMode::RegOffset { rn: base, offset }; + if from_reg.class() == RegClass::Float { + Inst::FpuStore { + size: FpuSize::F64, + rt: from_reg, + mem, + } + } else { + Inst::Store { + rt: from_reg, + mem, + kind: StoreKind::Word, + } } } @@ -322,11 +340,17 @@ impl ABIMachineSpec for Arm32MachineDeps { + frame_layout.outgoing_args_size; if stack_size > 0 { insts.extend(Self::gen_sp_reg_adjust(-(stack_size as i32))); - let mut cur_offset = 0i32; + // Callee-saved registers are saved at the *top* of the frame, above + // the fixed stack-slot and outgoing-argument regions (which resolve + // from sp upward). Placing them from sp=0 would alias those regions + // and corrupt saved registers. + let mut cur_offset = 0u32; for reg in &frame_layout.clobbered_callee_saves { let r = Reg::from(reg.to_reg()); + let bytes = if r.class() == RegClass::Float { 8 } else { 4 }; + cur_offset = align_to(cur_offset, bytes); let mem = AMode::SPOffset { - offset: i64::from(cur_offset), + offset: i64::from(stack_size - cur_offset - bytes), }; if r.class() == RegClass::Float { insts.push(Inst::FpuStore { @@ -334,15 +358,14 @@ impl ABIMachineSpec for Arm32MachineDeps { rt: r, mem, }); - cur_offset += 8; } else { insts.push(Inst::Store { rt: r, mem, kind: StoreKind::Word, }); - cur_offset += 4; } + cur_offset += bytes; } } insts @@ -357,26 +380,29 @@ impl ABIMachineSpec for Arm32MachineDeps { let stack_size = frame_layout.clobber_size + frame_layout.fixed_frame_storage_size + frame_layout.outgoing_args_size; - let mut cur_offset = 0i32; + // Mirror the top-of-frame placement used by `gen_clobber_save`. + let mut cur_offset = 0u32; for reg in &frame_layout.clobbered_callee_saves { + let is_float = reg.to_reg().class() == RegClass::Float; + let bytes = if is_float { 8 } else { 4 }; + cur_offset = align_to(cur_offset, bytes); let mem = AMode::SPOffset { - offset: i64::from(cur_offset), + offset: i64::from(stack_size - cur_offset - bytes), }; - if reg.to_reg().class() == RegClass::Float { + if is_float { insts.push(Inst::FpuLoad { size: FpuSize::F64, rd: reg.map(Reg::from), mem, }); - cur_offset += 8; } else { insts.push(Inst::Load { rt: reg.map(Reg::from), mem, kind: LoadKind::Word, }); - cur_offset += 4; } + cur_offset += bytes; } if stack_size > 0 { insts.extend(Self::gen_sp_reg_adjust(stack_size as i32)); diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle index e60e8e8f6139..6fa1c7cdbe5b 100644 --- a/cranelift/codegen/src/isa/arm32/inst.isle +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -304,6 +304,10 @@ ;; A permanently-undefined instruction used to encode a trap. (Udf (code TrapCode)) + ;; Conditionally trap on the current flags: emitted as + ;; `b 1f; udf ; 1:`. + (TrapIf (cond Cond) (code TrapCode)) + ;; Load/store multiple, increment-after (`ldmia`/`stmia rn{!}, {list}`). (LdmStm (load bool) (rn Reg) (writeback bool) (reg_list u32)) @@ -534,17 +538,12 @@ (decl pure use_idiv () bool) (extern constructor use_idiv use_idiv) -(decl sdiv_reg (Reg Reg) Reg) -(rule (sdiv_reg rn rm) - (let ((rd WritableReg (temp_writable_reg $I32)) - (_ Unit (emit (MInst.SDiv rd rn rm)))) - rd)) - -(decl udiv_reg (Reg Reg) Reg) -(rule (udiv_reg rn rm) - (let ((rd WritableReg (temp_writable_reg $I32)) - (_ Unit (emit (MInst.UDiv rd rn rm)))) - rd)) +;; Signed/unsigned divide with the Cranelift-mandated trap checks (divide by +;; zero, and `INT_MIN / -1` overflow for `sdiv`) emitted before the divide. +(decl gen_sdiv (Reg Reg) Reg) +(extern constructor gen_sdiv gen_sdiv) +(decl gen_udiv (Reg Reg) Reg) +(extern constructor gen_udiv gen_udiv) ;;;; Bit operations and extends ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -699,6 +698,19 @@ (decl cond_from_floatcc (FloatCC) Cond) (extern constructor cond_from_floatcc cond_from_floatcc) +;; The two float conditions that a single ARM condition can't express, built as +;; a 0/1 boolean from the flags left by a `vcmp`. `one` (ordered and not-equal) +;; keeps the not-equal result only when ordered (`vc`); `ueq` (unordered or +;; equal) forces the equal result to 1 when unordered (`vs`). +(decl gen_fcmp_one (FpuSize Reg Reg) Reg) +(rule (gen_fcmp_one size a b) + (let ((_ Unit (emit_side_effect (vcmp size a b)))) + (csel (Cond.Vc) (csetv (Cond.Ne)) (gen_constant 0)))) +(decl gen_fcmp_ueq (FpuSize Reg Reg) Reg) +(rule (gen_fcmp_ueq size a b) + (let ((_ Unit (emit_side_effect (vcmp size a b)))) + (csel (Cond.Vs) (gen_constant 1) (csetv (Cond.Eq))))) + ;; `fcopysign` via GPR bit manipulation (clear the sign of `a`, OR in `b`'s). (decl gen_copysign_f32 (Reg Reg) Reg) (extern constructor gen_copysign_f32 gen_copysign_f32) @@ -943,6 +955,12 @@ (decl cond_from_intcc (IntCC) Cond) (extern constructor cond_from_intcc cond_from_intcc) +;; Emit a `cmp` of two (possibly narrow) integer values as a side effect, +;; widening i8/i16 operands to 32 bits per the comparison's signedness, and +;; return the ARM condition that then tests the result. +(decl emit_icmp_cmp (IntCC Value Value) Cond) +(extern constructor emit_icmp_cmp emit_icmp_cmp) + ;; Emit the compare sequence for a 64-bit `icmp` and return the ARM condition ;; that then tests it (the compare itself is emitted as a side effect). (decl lower_icmp_i64 (IntCC ValueRegs ValueRegs) Cond) diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs index b84391fc8546..5cd8466c9501 100644 --- a/cranelift/codegen/src/isa/arm32/inst/emit.rs +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -748,6 +748,15 @@ impl MachInstEmit for Inst { sink.add_trap(*code); put_u32(sink, enc_udf()); } + Inst::TrapIf { cond, code } => { + // Branch over the trap when the condition does not hold. + let skip = sink.get_label(); + sink.use_label_at_offset(sink.cur_offset(), skip, LabelUse::Branch26); + put_u32(sink, enc_bcond(cond.invert(), 0)); + sink.add_trap(*code); + put_u32(sink, enc_udf()); + sink.bind_label(skip, state.ctrl_plane_mut()); + } Inst::LdmStm { load, rn, diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index 952494eb0471..a0b6d82dc72a 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -153,7 +153,7 @@ fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { use_if_virtual(collector, rm); def_if_virtual(collector, rd); } - Inst::Udf { .. } | Inst::Barrier { .. } => {} + Inst::Udf { .. } | Inst::TrapIf { .. } | Inst::Barrier { .. } => {} Inst::AtomicRmw { rd, @@ -551,9 +551,12 @@ impl MachInst for Inst { } fn worst_case_size() -> CodeOffset { - // The largest instruction is `MovImm`, which expands to a `movw` plus a - // `movt`: 8 bytes. - 8 + // Must be an upper bound on the number of bytes any single (non + // jump-table) instruction emits, since the buffer uses it as the + // per-instruction island lookahead. The largest such sequence is the + // 64-bit atomic min/max RMW loop, which expands to 10 words (40 bytes). + // Inline jump tables manage their own islands and are exempt. + 44 } fn worst_case_island_growth() -> CodeOffset { @@ -771,6 +774,9 @@ impl Inst { alloc::format!("rrx {}, {}", r(rd.to_reg()), r(*rm)) } Inst::Udf { code } => alloc::format!("udf ; {code}"), + Inst::TrapIf { cond, code } => { + alloc::format!("b{} 1f ; udf ; 1: ; {code}", cond.invert().name()) + } Inst::LdmStm { load, rn, diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle index 916c86ce3823..f28c07ddb6bb 100644 --- a/cranelift/codegen/src/isa/arm32/lower.isle +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -208,6 +208,17 @@ (let ((_ Unit (emit_side_effect (vcmp (FpuSize.F64) (put_in_reg a) (put_in_reg b))))) (csetv (cond_from_floatcc cc)))) +;; `one` (ordered and not-equal) and `ueq` (unordered or equal) can't be +;; expressed as a single ARM condition, so they get a two-condition sequence. +(rule 1 (lower (fcmp _ (FloatCC.OrderedNotEqual) a @ (value_type $F32) b)) + (gen_fcmp_one (FpuSize.F32) (put_in_reg a) (put_in_reg b))) +(rule 1 (lower (fcmp _ (FloatCC.OrderedNotEqual) a @ (value_type $F64) b)) + (gen_fcmp_one (FpuSize.F64) (put_in_reg a) (put_in_reg b))) +(rule 1 (lower (fcmp _ (FloatCC.UnorderedOrEqual) a @ (value_type $F32) b)) + (gen_fcmp_ueq (FpuSize.F32) (put_in_reg a) (put_in_reg b))) +(rule 1 (lower (fcmp _ (FloatCC.UnorderedOrEqual) a @ (value_type $F64) b)) + (gen_fcmp_ueq (FpuSize.F64) (put_in_reg a) (put_in_reg b))) + ;; f32 <-> f64. (rule (lower (fpromote $F64 x)) (vcvt_ff true (put_in_reg x))) @@ -266,14 +277,15 @@ (mov_to_fpu64 (put_in_regs x))) ;;;; Divides (only when hardware `sdiv`/`udiv` is available) ;;;;;;;;;;;;;;;;;;; -;; NOTE: trap-on-zero / INT_MIN overflow checks are not yet emitted. +;; `gen_sdiv`/`gen_udiv` emit the trap-on-zero check (and, for `sdiv`, the +;; `INT_MIN / -1` overflow check) before the divide. (rule (lower (sdiv $I32 x y)) (if-let true (use_idiv)) - (sdiv_reg (put_in_reg x) (put_in_reg y))) + (gen_sdiv (put_in_reg x) (put_in_reg y))) (rule (lower (udiv $I32 x y)) (if-let true (use_idiv)) - (udiv_reg (put_in_reg x) (put_in_reg y))) + (gen_udiv (put_in_reg x) (put_in_reg y))) ;;;; Counting / byte-swap / bit-reverse ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -304,10 +316,11 @@ ;;;; `icmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; 32-bit comparison producing a 0/1 boolean. +;; 32-bit comparison producing a 0/1 boolean. `emit_icmp_cmp` widens narrow +;; (i8/i16) operands to 32 bits per the comparison's signedness before the +;; `cmp`, since Cranelift leaves the high bits of narrow values undefined. (rule (lower (icmp _ cc a @ (value_type (fits_in_32 _)) b)) - (let ((_ Unit (emit_side_effect (cmp_rr (put_in_reg a) (put_in_reg b))))) - (csetv (cond_from_intcc cc)))) + (csetv (emit_icmp_cmp cc a b))) ;; 64-bit comparison producing a 0/1 boolean. (rule 4 (lower (icmp _ cc a @ (value_type $I64) b)) @@ -317,14 +330,23 @@ ;; Fuse an `icmp` condition into the compare that precedes the select. (rule 1 (lower (select (fits_in_32 _) (icmp _ cc x @ (value_type (fits_in_32 _)) y) a b)) - (let ((_ Unit (emit_side_effect (cmp_rr (put_in_reg x) (put_in_reg y))))) - (csel (cond_from_intcc cc) (put_in_reg a) (put_in_reg b)))) + (let ((cond Cond (emit_icmp_cmp cc x y))) + (csel cond (put_in_reg a) (put_in_reg b)))) ;; Generic select: branch on whether the condition value is non-zero. (rule (lower (select (fits_in_32 _) c a b)) (let ((_ Unit (emit_side_effect (cmp_imm (put_in_reg c) 0)))) (csel (Cond.Ne) (put_in_reg a) (put_in_reg b)))) +;; 64-bit select: pick each half of the pair. The operands are materialized +;; before the compare so nothing clobbers the flags in between. +(rule 4 (lower (select $I64 c a b)) + (let ((av ValueRegs (put_in_regs a)) + (bv ValueRegs (put_in_regs b)) + (_ Unit (emit_side_effect (cmp_imm (put_in_reg c) 0)))) + (value_regs (csel (Cond.Ne) (vr_lo av) (vr_lo bv)) + (csel (Cond.Ne) (vr_hi av) (vr_hi bv))))) + ;;;; Shifts: `ishl` / `ushr` / `sshr` / `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule 1 (lower (ishl $I32 x (iconst _ (u64_from_imm64 n)))) @@ -349,6 +371,12 @@ ;;;; Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; A plain `load` reads exactly `sizeof(ty)` bytes; narrow types zero-extend +;; into the 32-bit result register. +(rule 1 (lower (load $I8 flags addr offset)) + (arm_load (amode addr offset) (LoadKind.UByte))) +(rule 1 (lower (load $I16 flags addr offset)) + (arm_load (amode addr offset) (LoadKind.UHalf))) (rule (lower (load (fits_in_32 _) flags addr offset)) (arm_load (amode addr offset) (LoadKind.Word))) @@ -363,6 +391,11 @@ ;;;; Stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; A plain `store` writes exactly `sizeof(ty)` bytes. +(rule 1 (lower (store flags val @ (value_type $I8) addr offset)) + (arm_store (put_in_reg val) (amode addr offset) (StoreKind.Byte))) +(rule 1 (lower (store flags val @ (value_type $I16) addr offset)) + (arm_store (put_in_reg val) (amode addr offset) (StoreKind.Half))) (rule (lower (store flags val @ (value_type (fits_in_32 _)) addr offset)) (arm_store (put_in_reg val) (amode addr offset) (StoreKind.Word))) @@ -431,16 +464,33 @@ ;; Fuse a 32-bit `icmp` feeding a `brif` into a compare plus conditional branch. (rule 1 (lower_branch (brif (icmp _ cc a @ (value_type (fits_in_32 _)) b) _ _) (two_targets taken not_taken)) - (emit_side_effect - (side_effect_concat - (cmp_rr (put_in_reg a) (put_in_reg b)) - (cond_br (cond_from_intcc cc) taken not_taken)))) + (emit_side_effect (cond_br (emit_icmp_cmp cc a b) taken not_taken))) ;; Fuse a 64-bit `icmp` feeding a `brif`. (rule 2 (lower_branch (brif (icmp _ cc a @ (value_type $I64) b) _ _) (two_targets taken not_taken)) (emit_side_effect (cond_br (lower_icmp_i64 cc (put_in_regs a) (put_in_regs b)) taken not_taken))) +;; `one`/`ueq` feeding a `brif`: materialize the two-condition boolean, then +;; branch on whether it is non-zero (priority 2 beats the generic fused rule, +;; whose single-condition mapping can't represent these). +(rule 2 (lower_branch (brif (fcmp _ (FloatCC.OrderedNotEqual) a @ (value_type $F32) b) _ _) (two_targets taken not_taken)) + (emit_side_effect (side_effect_concat + (cmp_imm (gen_fcmp_one (FpuSize.F32) (put_in_reg a) (put_in_reg b)) 0) + (cond_br (Cond.Ne) taken not_taken)))) +(rule 2 (lower_branch (brif (fcmp _ (FloatCC.OrderedNotEqual) a @ (value_type $F64) b) _ _) (two_targets taken not_taken)) + (emit_side_effect (side_effect_concat + (cmp_imm (gen_fcmp_one (FpuSize.F64) (put_in_reg a) (put_in_reg b)) 0) + (cond_br (Cond.Ne) taken not_taken)))) +(rule 2 (lower_branch (brif (fcmp _ (FloatCC.UnorderedOrEqual) a @ (value_type $F32) b) _ _) (two_targets taken not_taken)) + (emit_side_effect (side_effect_concat + (cmp_imm (gen_fcmp_ueq (FpuSize.F32) (put_in_reg a) (put_in_reg b)) 0) + (cond_br (Cond.Ne) taken not_taken)))) +(rule 2 (lower_branch (brif (fcmp _ (FloatCC.UnorderedOrEqual) a @ (value_type $F64) b) _ _) (two_targets taken not_taken)) + (emit_side_effect (side_effect_concat + (cmp_imm (gen_fcmp_ueq (FpuSize.F64) (put_in_reg a) (put_in_reg b)) 0) + (cond_br (Cond.Ne) taken not_taken)))) + ;; Fuse an `fcmp` feeding a `brif`. (rule 1 (lower_branch (brif (fcmp _ cc a @ (value_type $F32) b) _ _) (two_targets taken not_taken)) (emit_side_effect diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs index 03bb0d209ef2..409fd3243e27 100644 --- a/cranelift/codegen/src/isa/arm32/lower/isle.rs +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -12,7 +12,7 @@ use crate::ir::{ BlockCall, ExternalName, Inst, InstructionData, MemFlags, Opcode, TrapCode, Value, ValueList, }; use crate::isa::arm32::Arm32Backend; -use crate::isa::arm32::inst::{ALUOp, Cond, ShiftOp, encode_rotated_imm}; +use crate::isa::arm32::inst::{ALUOp, CmpOp, Cond, ExtOp, ShiftOp, encode_rotated_imm}; use crate::machinst::isle::*; use crate::machinst::{ ArgPair, CallArgList, CallInfo, CallRetList, InstOutput, Lower, MachInst, MachLabel, RetPair, @@ -114,6 +114,35 @@ impl<'a, 'b> Arm32IsleContext<'a, 'b, MInst, Arm32Backend> { self.lower_ctx.emit(inst); rd.to_reg() } + + /// Widen a narrow (i8/i16) comparison operand to a full 32-bit register, + /// sign- or zero-extending per `signed`. i32 (and anything wider) is + /// returned unchanged. + fn extend_for_cmp(&mut self, ty: Type, signed: bool, r: Reg) -> Reg { + let op = match (ty, signed) { + (I8, true) => ExtOp::Sxtb, + (I8, false) => ExtOp::Uxtb, + (I16, true) => ExtOp::Sxth, + (I16, false) => ExtOp::Uxth, + _ => return r, + }; + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::ExtRR { op, rd, rm: r }); + rd.to_reg() + } + + /// Emit `cmp divisor, #0; trapif eq, int_divz`. + fn emit_divz_check(&mut self, divisor: Reg) { + self.lower_ctx.emit(MInst::CmpRImm { + op: CmpOp::Cmp, + rn: divisor, + imm12: 0, + }); + self.lower_ctx.emit(MInst::TrapIf { + cond: Cond::Eq, + code: TrapCode::INTEGER_DIVISION_BY_ZERO, + }); + } } impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { @@ -362,9 +391,11 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { FloatCC::UnorderedOrLessThanOrEqual => Cond::Le, FloatCC::UnorderedOrGreaterThan => Cond::Hi, FloatCC::UnorderedOrGreaterThanOrEqual => Cond::Hs, - // These need a two-condition sequence and aren't handled yet. + // These need a two-condition sequence and are handled by dedicated + // lowering rules (`gen_fcmp_one` / `gen_fcmp_ueq`) before reaching + // this single-condition mapping. FloatCC::OrderedNotEqual | FloatCC::UnorderedOrEqual => { - unimplemented!("arm32: fcmp condition {cc:?} not yet implemented") + unreachable!("arm32: fcmp {cc:?} is lowered via a two-condition sequence") } } } @@ -392,6 +423,57 @@ impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { self.mov_to_d_raw(a_lo, res_hi) } + /// Emit the compare for an `icmp` (widening narrow operands per the + /// comparison's signedness) and return the ARM condition that tests it. + fn emit_icmp_cmp(&mut self, cc: &IntCC, a: Value, b: Value) -> Cond { + let ty = self.value_type(a); + let signed = matches!( + cc, + IntCC::SignedLessThan + | IntCC::SignedGreaterThanOrEqual + | IntCC::SignedGreaterThan + | IntCC::SignedLessThanOrEqual + ); + let ra = self.put_in_reg(a); + let ra = self.extend_for_cmp(ty, signed, ra); + let rb = self.put_in_reg(b); + let rb = self.extend_for_cmp(ty, signed, rb); + self.lower_ctx.emit(MInst::CmpRR { + op: CmpOp::Cmp, + rn: ra, + rm: rb, + }); + self.cond_from_intcc(cc) + } + + /// `udiv` with a divide-by-zero trap check. + fn gen_udiv(&mut self, x: Reg, y: Reg) -> Reg { + self.emit_divz_check(y); + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::UDiv { rd, rn: x, rm: y }); + rd.to_reg() + } + + /// `sdiv` with divide-by-zero and `INT_MIN / -1` overflow trap checks. + fn gen_sdiv(&mut self, x: Reg, y: Reg) -> Reg { + self.emit_divz_check(y); + // Overflow occurs only for `INT_MIN / -1`, i.e. when both + // `x ^ INT_MIN == 0` and `y + 1 == 0`; OR the two together and trap when + // the result is zero. + let int_min = encode_rotated_imm(0x8000_0000).unwrap(); + let t1 = self.alu_imm_raw(ALUOp::Eor, x, int_min); + let one = encode_rotated_imm(1).unwrap(); + let t2 = self.alu_imm_raw(ALUOp::Add, y, one); + let _ = self.alu_raw(ALUOp::Orr, true, t1, t2); // orrs (sets Z on overflow) + self.lower_ctx.emit(MInst::TrapIf { + cond: Cond::Eq, + code: TrapCode::INTEGER_OVERFLOW, + }); + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::SDiv { rd, rn: x, rm: y }); + rd.to_reg() + } + fn cond_from_intcc(&mut self, cc: &IntCC) -> Cond { match cc { IntCC::Equal => Cond::Eq, diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index 6fbf8101cb0c..1649c7b3927b 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -114,7 +114,11 @@ pub fn lookup(triple: Triple) -> Result { isa_builder!(x64, (feature = "x86"), triple) } Architecture::Aarch64 { .. } => isa_builder!(aarch64, (feature = "arm64"), triple), - Architecture::Arm(..) => isa_builder!(arm32, (feature = "arm32"), triple), + // The arm32 backend only emits A32 encodings, so Thumb-only targets are + // not supported. + Architecture::Arm(arm) if !arm.is_thumb() => { + isa_builder!(arm32, (feature = "arm32"), triple) + } Architecture::S390x { .. } => isa_builder!(s390x, (feature = "s390x"), triple), Architecture::Riscv64 { .. } => isa_builder!(riscv64, (feature = "riscv64"), triple), Architecture::Pulley32 | Architecture::Pulley32be => { diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index 0ca00643148d..ae7d76b257b8 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -1625,6 +1625,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { /// Increment the reference count for the Value, ensuring that it gets lowered. #[cfg(any( feature = "x86", + feature = "arm32", feature = "arm64", feature = "riscv64", feature = "s390x", diff --git a/cranelift/filetests/filetests/isa/arm32/atomic.clif b/cranelift/filetests/filetests/isa/arm32/atomic.clif index 4e49df5d752a..4694514d9c69 100644 --- a/cranelift/filetests/filetests/isa/arm32/atomic.clif +++ b/cranelift/filetests/filetests/isa/arm32/atomic.clif @@ -11,13 +11,13 @@ block0(v0: i32, v1: i32): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp, #0] +; str r7, [sp, #4] ; block0: ; mov r2, r0 ; mov r7, r1 ; atomic_rmw.Add r3, [r2], r7 ; mov r0, r3 -; ldr r7, [sp, #0] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -27,7 +27,7 @@ block0(v0: i32, v1: i32): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp] +; str r7, [sp, #4] ; block1: ; offset 0x10 ; mov r2, r0 ; mov r7, r1 @@ -37,7 +37,7 @@ block0(v0: i32, v1: i32): ; cmp r0, #0 ; bne #0x18 ; mov r0, r3 -; ldr r7, [sp] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -52,13 +52,13 @@ block0(v0: i32, v1: i32): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp, #0] +; str r7, [sp, #4] ; block0: ; mov r2, r0 ; mov r7, r1 ; atomic_rmw.Umax r3, [r2], r7 ; mov r0, r3 -; ldr r7, [sp, #0] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -68,7 +68,7 @@ block0(v0: i32, v1: i32): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp] +; str r7, [sp, #4] ; block1: ; offset 0x10 ; mov r2, r0 ; mov r7, r1 @@ -80,7 +80,7 @@ block0(v0: i32, v1: i32): ; cmp r0, #0 ; bne #0x18 ; mov r0, r3 -; ldr r7, [sp] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -95,13 +95,13 @@ block0(v0: i32, v1: i32, v2: i32): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp, #0] +; str r7, [sp, #4] ; block0: ; mov r3, r0 ; mov r7, r1 ; atomic_cas r1, [r3], r7, r2 ; mov r0, r1 -; ldr r7, [sp, #0] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -111,7 +111,7 @@ block0(v0: i32, v1: i32, v2: i32): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp] +; str r7, [sp, #4] ; block1: ; offset 0x10 ; mov r3, r0 ; mov r7, r1 @@ -122,7 +122,7 @@ block0(v0: i32, v1: i32, v2: i32): ; cmp r0, #0 ; bne #0x18 ; mov r0, r1 -; ldr r7, [sp] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -170,13 +170,13 @@ block0(v0: i32, v1: i8): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp, #0] +; str r7, [sp, #4] ; block0: ; mov r2, r0 ; mov r7, r1 ; atomic_rmw.Addb r3, [r2], r7 ; mov r0, r3 -; ldr r7, [sp, #0] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -186,7 +186,7 @@ block0(v0: i32, v1: i8): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp] +; str r7, [sp, #4] ; block1: ; offset 0x10 ; mov r2, r0 ; mov r7, r1 @@ -196,7 +196,7 @@ block0(v0: i32, v1: i8): ; cmp r0, #0 ; bne #0x18 ; mov r0, r3 -; ldr r7, [sp] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -212,13 +212,13 @@ block0(v0: i32, v1: i8): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp, #0] +; str r7, [sp, #4] ; block0: ; mov r2, r0 ; mov r7, r1 ; atomic_rmw.Smaxb r3, [r2], r7 ; mov r0, r3 -; ldr r7, [sp, #0] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -228,7 +228,7 @@ block0(v0: i32, v1: i8): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp] +; str r7, [sp, #4] ; block1: ; offset 0x10 ; mov r2, r0 ; mov r7, r1 @@ -242,7 +242,7 @@ block0(v0: i32, v1: i8): ; cmp r0, #0 ; bne #0x18 ; mov r0, r3 -; ldr r7, [sp] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -258,13 +258,13 @@ block0(v0: i32, v1: i16): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp, #0] +; str r7, [sp, #4] ; block0: ; mov r2, r0 ; mov r7, r1 ; atomic_rmw.Uminh r3, [r2], r7 ; mov r0, r3 -; ldr r7, [sp, #0] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -274,7 +274,7 @@ block0(v0: i32, v1: i16): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp] +; str r7, [sp, #4] ; block1: ; offset 0x10 ; mov r2, r0 ; mov r7, r1 @@ -288,7 +288,7 @@ block0(v0: i32, v1: i16): ; cmp r0, #0 ; bne #0x18 ; mov r0, r3 -; ldr r7, [sp] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -304,13 +304,13 @@ block0(v0: i32, v1: i16, v2: i16): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp, #0] +; str r7, [sp, #4] ; block0: ; mov r3, r0 ; mov r7, r1 ; atomic_cash r1, [r3], r7, r2 ; mov r0, r1 -; ldr r7, [sp, #0] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -320,7 +320,7 @@ block0(v0: i32, v1: i16, v2: i16): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r7, [sp] +; str r7, [sp, #4] ; block1: ; offset 0x10 ; mov r3, r0 ; mov r7, r1 @@ -332,7 +332,7 @@ block0(v0: i32, v1: i16, v2: i16): ; cmp r0, #0 ; bne #0x18 ; mov r0, r1 -; ldr r7, [sp] +; ldr r7, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -382,19 +382,19 @@ block0(v0: i32, v1: i64): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #16 -; str r4, [sp, #0] -; str r5, [sp, #4] -; str r6, [sp, #8] -; str r7, [sp, #12] +; str r4, [sp, #12] +; str r5, [sp, #8] +; str r6, [sp, #4] +; str r7, [sp, #0] ; block0: ; mov r4, r0 ; mov r5, r2 ; mov r6, r3 ; atomic_rmw.Add.i64 r0, r1, [r4], r5, r6 -; ldr r4, [sp, #0] -; ldr r5, [sp, #4] -; ldr r6, [sp, #8] -; ldr r7, [sp, #12] +; ldr r4, [sp, #12] +; ldr r5, [sp, #8] +; ldr r6, [sp, #4] +; ldr r7, [sp, #0] ; add sp, sp, #16 ; pop {fp, lr} ; bx lr @@ -404,10 +404,10 @@ block0(v0: i32, v1: i64): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #0x10 -; str r4, [sp] -; str r5, [sp, #4] -; str r6, [sp, #8] -; str r7, [sp, #0xc] +; str r4, [sp, #0xc] +; str r5, [sp, #8] +; str r6, [sp, #4] +; str r7, [sp] ; block1: ; offset 0x1c ; mov r4, r0 ; mov r5, r2 @@ -418,10 +418,10 @@ block0(v0: i32, v1: i64): ; stlexd r7, r2, r3, [r4] ; cmp r7, #0 ; bne #0x28 -; ldr r4, [sp] -; ldr r5, [sp, #4] -; ldr r6, [sp, #8] -; ldr r7, [sp, #0xc] +; ldr r4, [sp, #0xc] +; ldr r5, [sp, #8] +; ldr r6, [sp, #4] +; ldr r7, [sp] ; add sp, sp, #0x10 ; pop {fp, lr} ; bx lr @@ -438,19 +438,19 @@ block0(v0: i32, v1: i64): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #16 -; str r4, [sp, #0] -; str r5, [sp, #4] -; str r6, [sp, #8] -; str r7, [sp, #12] +; str r4, [sp, #12] +; str r5, [sp, #8] +; str r6, [sp, #4] +; str r7, [sp, #0] ; block0: ; mov r4, r0 ; mov r5, r2 ; mov r6, r3 ; atomic_rmw.Smax.i64 r0, r1, [r4], r5, r6 -; ldr r4, [sp, #0] -; ldr r5, [sp, #4] -; ldr r6, [sp, #8] -; ldr r7, [sp, #12] +; ldr r4, [sp, #12] +; ldr r5, [sp, #8] +; ldr r6, [sp, #4] +; ldr r7, [sp, #0] ; add sp, sp, #16 ; pop {fp, lr} ; bx lr @@ -460,10 +460,10 @@ block0(v0: i32, v1: i64): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #0x10 -; str r4, [sp] -; str r5, [sp, #4] -; str r6, [sp, #8] -; str r7, [sp, #0xc] +; str r4, [sp, #0xc] +; str r5, [sp, #8] +; str r6, [sp, #4] +; str r7, [sp] ; block1: ; offset 0x1c ; mov r4, r0 ; mov r5, r2 @@ -478,10 +478,10 @@ block0(v0: i32, v1: i64): ; stlexd r7, r2, r3, [r4] ; cmp r7, #0 ; bne #0x28 -; ldr r4, [sp] -; ldr r5, [sp, #4] -; ldr r6, [sp, #8] -; ldr r7, [sp, #0xc] +; ldr r4, [sp, #0xc] +; ldr r5, [sp, #8] +; ldr r6, [sp, #4] +; ldr r7, [sp] ; add sp, sp, #0x10 ; pop {fp, lr} ; bx lr @@ -496,10 +496,10 @@ block0(v0: i32, v1: i64, v2: i64): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #16 -; str r4, [sp, #0] -; str r5, [sp, #4] -; str r6, [sp, #8] -; str r7, [sp, #12] +; str r4, [sp, #12] +; str r5, [sp, #8] +; str r6, [sp, #4] +; str r7, [sp, #0] ; block0: ; mov r5, r2 ; mov r6, r3 @@ -507,10 +507,10 @@ block0(v0: i32, v1: i64, v2: i64): ; ldr r3, [incoming_arg, #4] ; mov r4, r0 ; atomic_cas.i64 r0, r1, [r4], r5, r6, r2, r3 -; ldr r4, [sp, #0] -; ldr r5, [sp, #4] -; ldr r6, [sp, #8] -; ldr r7, [sp, #12] +; ldr r4, [sp, #12] +; ldr r5, [sp, #8] +; ldr r6, [sp, #4] +; ldr r7, [sp, #0] ; add sp, sp, #16 ; pop {fp, lr} ; bx lr @@ -520,10 +520,10 @@ block0(v0: i32, v1: i64, v2: i64): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #0x10 -; str r4, [sp] -; str r5, [sp, #4] -; str r6, [sp, #8] -; str r7, [sp, #0xc] +; str r4, [sp, #0xc] +; str r5, [sp, #8] +; str r6, [sp, #4] +; str r7, [sp] ; block1: ; offset 0x1c ; mov r5, r2 ; mov r6, r3 @@ -538,10 +538,10 @@ block0(v0: i32, v1: i64, v2: i64): ; stlexd r7, r2, r3, [r4] ; cmp r7, #0 ; bne #0x30 -; ldr r4, [sp] -; ldr r5, [sp, #4] -; ldr r6, [sp, #8] -; ldr r7, [sp, #0xc] +; ldr r4, [sp, #0xc] +; ldr r5, [sp, #8] +; ldr r6, [sp, #4] +; ldr r7, [sp] ; add sp, sp, #0x10 ; pop {fp, lr} ; bx lr @@ -556,11 +556,11 @@ block0(v0: i32): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r4, [sp, #0] +; str r4, [sp, #4] ; block0: ; mov r4, r0 ; atomic_load.i64 r0, r1, [r4] -; ldr r4, [sp, #0] +; ldr r4, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -570,12 +570,12 @@ block0(v0: i32): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r4, [sp] +; str r4, [sp, #4] ; block1: ; offset 0x10 ; mov r4, r0 ; ldaexd r0, r1, [r4] ; clrex -; ldr r4, [sp] +; ldr r4, [sp, #4] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -590,13 +590,13 @@ block0(v0: i32, v1: i64): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r4, [sp, #0] -; str r7, [sp, #4] +; str r4, [sp, #4] +; str r7, [sp, #0] ; block0: ; mov r4, r0 ; atomic_store.i64 r2, r3, [r4] -; ldr r4, [sp, #0] -; ldr r7, [sp, #4] +; ldr r4, [sp, #4] +; ldr r7, [sp, #0] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -606,16 +606,16 @@ block0(v0: i32, v1: i64): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r4, [sp] -; str r7, [sp, #4] +; str r4, [sp, #4] +; str r7, [sp] ; block1: ; offset 0x14 ; mov r4, r0 ; ldaexd r0, r1, [r4] ; stlexd r7, r2, r3, [r4] ; cmp r7, #0 ; bne #0x18 -; ldr r4, [sp] -; ldr r7, [sp, #4] +; ldr r4, [sp, #4] +; ldr r7, [sp] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr diff --git a/cranelift/filetests/filetests/isa/arm32/div.clif b/cranelift/filetests/filetests/isa/arm32/div.clif index b35cda4b9d61..d8bb6eeaf8a7 100644 --- a/cranelift/filetests/filetests/isa/arm32/div.clif +++ b/cranelift/filetests/filetests/isa/arm32/div.clif @@ -9,11 +9,25 @@ block0(v0: i32, v1: i32): ; VCode: ; block0: +; cmp r1, #0 +; bne 1f ; udf ; 1: ; int_divz +; eor r2, r0, #2147483648 +; add r3, r1, #1 +; orrs r2, r2, r3 +; bne 1f ; udf ; 1: ; int_ovf ; sdiv r0, r0, r1 ; bx lr ; ; Disassembled: ; block0: ; offset 0x0 +; cmp r1, #0 +; bne #0xc +; udf #0 ; trap: int_divz +; eor r2, r0, #0x80000000 +; add r3, r1, #1 +; orrs r2, r2, r3 +; bne #0x20 +; udf #0 ; trap: int_ovf ; sdiv r0, r0, r1 ; bx lr @@ -25,11 +39,16 @@ block0(v0: i32, v1: i32): ; VCode: ; block0: +; cmp r1, #0 +; bne 1f ; udf ; 1: ; int_divz ; udiv r0, r0, r1 ; bx lr ; ; Disassembled: ; block0: ; offset 0x0 +; cmp r1, #0 +; bne #0xc +; udf #0 ; trap: int_divz ; udiv r0, r0, r1 ; bx lr diff --git a/cranelift/filetests/filetests/isa/arm32/float.clif b/cranelift/filetests/filetests/isa/arm32/float.clif index b8a5867084af..e13f460e1fe0 100644 --- a/cranelift/filetests/filetests/isa/arm32/float.clif +++ b/cranelift/filetests/filetests/isa/arm32/float.clif @@ -127,3 +127,61 @@ block0(v0: i32, v1: f64): ; vstr d0, [r0] ; bx lr +function %fcmp_one_f32(f32, f32) -> i8 { +block0(v0: f32, v1: f32): + v2 = fcmp one v0, v1 + return v2 +} + +; VCode: +; block0: +; vcmp.f32 d0, d1; vmrs +; mov r1, #1 +; mov r0, #0 +; mov r2, r0; movne r2, r1 +; mov r1, #0 +; mov r0, r1; movvc r0, r2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vcmp.f32 s0, s2 +; vmrs apsr_nzcv, fpscr +; mov r1, #1 +; mov r0, #0 +; mov r2, r0 +; movne r2, r1 +; mov r1, #0 +; mov r0, r1 +; movvc r0, r2 +; bx lr + +function %fcmp_ueq_f64(f64, f64) -> i8 { +block0(v0: f64, v1: f64): + v2 = fcmp ueq v0, v1 + return v2 +} + +; VCode: +; block0: +; vcmp.f64 d0, d1; vmrs +; mov r2, #1 +; mov r1, #1 +; mov r0, #0 +; mov r3, r0; moveq r3, r1 +; mov r0, r3; movvs r0, r2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vcmp.f64 d0, d1 +; vmrs apsr_nzcv, fpscr +; mov r2, #1 +; mov r1, #1 +; mov r0, #0 +; mov r3, r0 +; moveq r3, r1 +; mov r0, r3 +; movvs r0, r2 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/i64.clif b/cranelift/filetests/filetests/isa/arm32/i64.clif index b29d30f4c499..fd15d000af30 100644 --- a/cranelift/filetests/filetests/isa/arm32/i64.clif +++ b/cranelift/filetests/filetests/isa/arm32/i64.clif @@ -120,15 +120,15 @@ block0(v0: i64, v1: i64): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r5, [sp, #0] -; str r6, [sp, #4] +; str r5, [sp, #4] +; str r6, [sp, #0] ; block0: ; umull r5, r6, r0, r2 ; mla r0, r0, r3, r6 ; mla r1, r1, r2, r0 ; mov r0, r5 -; ldr r5, [sp, #0] -; ldr r6, [sp, #4] +; ldr r5, [sp, #4] +; ldr r6, [sp, #0] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr @@ -138,15 +138,15 @@ block0(v0: i64, v1: i64): ; push {fp, lr} ; mov fp, sp ; sub sp, sp, #8 -; str r5, [sp] -; str r6, [sp, #4] +; str r5, [sp, #4] +; str r6, [sp] ; block1: ; offset 0x14 ; umull r5, r6, r0, r2 ; mla r0, r0, r3, r6 ; mla r1, r1, r2, r0 ; mov r0, r5 -; ldr r5, [sp] -; ldr r6, [sp, #4] +; ldr r5, [sp, #4] +; ldr r6, [sp] ; add sp, sp, #8 ; pop {fp, lr} ; bx lr diff --git a/cranelift/filetests/filetests/isa/arm32/icmp.clif b/cranelift/filetests/filetests/isa/arm32/icmp.clif new file mode 100644 index 000000000000..723fb66fe47c --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/icmp.clif @@ -0,0 +1,82 @@ +test compile precise-output +target arm + +; Narrow (i8/i16) comparisons must widen their operands per the comparison's +; signedness, since Cranelift leaves the high bits of narrow values undefined. +function %icmp_slt_i8(i8, i8) -> i8 { +block0(v0: i8, v1: i8): + v2 = icmp slt v0, v1 + return v2 +} + +; VCode: +; block0: +; sxtb r3, r0 +; sxtb r0, r1 +; cmp r3, r0 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; movlt r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; sxtb r3, r0 +; sxtb r0, r1 +; cmp r3, r0 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; movlt r0, r1 +; bx lr + +function %icmp_ult_i16(i16, i16) -> i8 { +block0(v0: i16, v1: i16): + v2 = icmp ult v0, v1 + return v2 +} + +; VCode: +; block0: +; uxth r3, r0 +; uxth r0, r1 +; cmp r3, r0 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; movlo r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; uxth r3, r0 +; uxth r0, r1 +; cmp r3, r0 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; movlo r0, r1 +; bx lr + +function %icmp_eq_i32(i32, i32) -> i8 { +block0(v0: i32, v1: i32): + v2 = icmp eq v0, v1 + return v2 +} + +; VCode: +; block0: +; cmp r0, r1 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; moveq r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; cmp r0, r1 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; moveq r0, r1 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/memory.clif b/cranelift/filetests/filetests/isa/arm32/memory.clif index b3be1e5b1b44..4b8de0e625d1 100644 --- a/cranelift/filetests/filetests/isa/arm32/memory.clif +++ b/cranelift/filetests/filetests/isa/arm32/memory.clif @@ -81,3 +81,51 @@ block0(v0: i32): ; ldrsh r0, [r0, #2] ; bx lr +function %load_i8(i32) -> i8 { +block0(v0: i32): + v1 = load.i8 v0 + return v1 +} + +; VCode: +; block0: +; ldrb r0, [r0, #0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldrb r0, [r0] +; bx lr + +function %store_i8(i8, i32) { +block0(v0: i8, v1: i32): + store.i8 v0, v1 + return +} + +; VCode: +; block0: +; strb r0, [r1, #0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; strb r0, [r1] +; bx lr + +function %load_i16(i32) -> i16 { +block0(v0: i32): + v1 = load.i16 v0 + return v1 +} + +; VCode: +; block0: +; ldrh r0, [r0, #0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldrh r0, [r0] +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/select.clif b/cranelift/filetests/filetests/isa/arm32/select.clif index 6045c800a447..3296ae8639d3 100644 --- a/cranelift/filetests/filetests/isa/arm32/select.clif +++ b/cranelift/filetests/filetests/isa/arm32/select.clif @@ -40,3 +40,44 @@ block0(v0: i32, v1: i32, v2: i32): ; movne r0, r1 ; bx lr +function %select_i64(i32, i64, i64) -> i64 { +block0(v0: i32, v1: i64, v2: i64): + v3 = select v0, v1, v2 + return v3 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r6, [sp, #4] +; block0: +; ldr r1, [incoming_arg, #8] +; ldr r6, [incoming_arg, #4] +; cmp r0, #0 +; mov r0, r1; movne r0, r2 +; mov r1, r6; movne r1, r3 +; ldr r6, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r6, [sp, #4] +; block1: ; offset 0x10 +; ldr r1, [sp, #0x10] +; ldr r6, [sp, #0x14] +; cmp r0, #0 +; mov r0, r1 +; movne r0, r2 +; mov r1, r6 +; movne r1, r3 +; ldr r6, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +