diff --git a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressNeon_aarch64.inc b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressNeon_aarch64.inc index 0a25415..808aec2 100644 --- a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressNeon_aarch64.inc +++ b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressNeon_aarch64.inc @@ -15,6 +15,8 @@ // v21-v23 = rotate / diagonalize temps (b1 gather uses v23) // u64 ror-by-32 uses rev64.4s (vrev64q_u32), not rev64.16b. + {$I ..\Include\Simd\Common\SimdNonVolatileSave_aarch64.inc} + .long 0xad400400 // ldp q0, q1, [x0] .long 0xad410c02 // ldp q2, q3, [x0, #32] .long 0x4ea01c10 // orr v16.16b, v0.16b, v0.16b @@ -1392,4 +1394,6 @@ .long 0x6e231e63 // eor v3.16b, v19.16b, v3.16b .long 0xad000400 // stp q0, q1, [x0] .long 0xad010c02 // stp q2, q3, [x0, #32] + + {$I ..\Include\Simd\Common\SimdNonVolatileRestore_aarch64.inc} ret diff --git a/HashLib/src/Include/Simd/Blake3/Blake3Hash4Neon_aarch64.inc b/HashLib/src/Include/Simd/Blake3/Blake3Hash4Neon_aarch64.inc index a4d62e3..36d8eff 100644 --- a/HashLib/src/Include/Simd/Blake3/Blake3Hash4Neon_aarch64.inc +++ b/HashLib/src/Include/Simd/Blake3/Blake3Hash4Neon_aarch64.inc @@ -18,6 +18,8 @@ // [sp, #320..568] saved CV, counter vectors, IV, block len, AOut ptr, flags // Message transpose uses trn1/trn2 + zip1/zip2 (equivalent to SSE punpck / neon transpose_vecs_128). + {$I ..\Include\Simd\Common\SimdNonVolatileSave_aarch64.inc} + stp x19, x20, [sp, #-16]! sub sp, sp, #720 .long 0xbd400020 // ldr s0, [x1, #0] @@ -1708,4 +1710,6 @@ .long 0x3d801d23 // str q3, [x9, #112] add sp, sp, #720 ldp x19, x20, [sp], #16 + + {$I ..\Include\Simd\Common\SimdNonVolatileRestore_aarch64.inc} ret diff --git a/HashLib/src/Include/Simd/Common/SimdNonVolatileRestore_aarch64.inc b/HashLib/src/Include/Simd/Common/SimdNonVolatileRestore_aarch64.inc new file mode 100644 index 0000000..b7f37aa --- /dev/null +++ b/HashLib/src/Include/Simd/Common/SimdNonVolatileRestore_aarch64.inc @@ -0,0 +1,9 @@ +// Shared nonvolatile-SIMD restore (AArch64). Mirror of SimdNonVolatileSave_aarch64.inc: +// restores d8..d15 and frees the 64-byte save frame. Include it at the kernel's +// restore point (after the last use of v8..v15, before returning), in any +// procedure that included SimdNonVolatileSave_aarch64.inc. Restoring d8..d15 +// zeroes the upper halves of v8..v15, which is fine since those are caller-saved. + .long 0x6d412fea // ldp d10, d11, [sp, #16] + .long 0x6d4237ec // ldp d12, d13, [sp, #32] + .long 0x6d433fee // ldp d14, d15, [sp, #48] + .long 0x6cc427e8 // ldp d8, d9, [sp], #64 diff --git a/HashLib/src/Include/Simd/Common/SimdNonVolatileSave_aarch64.inc b/HashLib/src/Include/Simd/Common/SimdNonVolatileSave_aarch64.inc new file mode 100644 index 0000000..8ea5657 --- /dev/null +++ b/HashLib/src/Include/Simd/Common/SimdNonVolatileSave_aarch64.inc @@ -0,0 +1,19 @@ +// Shared nonvolatile-SIMD save (AArch64). Opt-in by inclusion: a kernel that +// clobbers any of v8..v15 includes this at its save point (before it first +// writes those registers), and the matching SimdNonVolatileRestore_aarch64.inc +// at its restore point (after the last use, before returning). Kernels that +// touch only v0..v7 / v16..v31 do NOT include it. +// +// AAPCS64 makes the low 64 bits of v8..v15 (d8..d15) callee-saved; a kernel +// that clobbers them must preserve them for the caller. Only the low 64 bits +// are callee-saved (the upper halves of v8..v15 are caller-saved), so restoring +// d8..d15 - which zeroes the upper halves - is sufficient. All eight are saved +// with one fixed 64-byte frame (a multiple of 16, keeping sp 16-byte aligned). +// +// Unlike x86-64 there is no ABI split: the d8..d15 rule is uniform across +// Linux/macOS/Windows, so this always emits (no platform guard). +// Save and restore MUST be paired within the same procedure. + .long 0x6dbc27e8 // stp d8, d9, [sp, #-64]! + .long 0x6d012fea // stp d10, d11, [sp, #16] + .long 0x6d0237ec // stp d12, d13, [sp, #32] + .long 0x6d033fee // stp d14, d15, [sp, #48] diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExtAbsorb_aarch64.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExtAbsorb_aarch64.inc index bf57716..0d181fd 100644 --- a/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExtAbsorb_aarch64.inc +++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExtAbsorb_aarch64.inc @@ -9,10 +9,7 @@ // v25-v30 = permutation scratch // v31 = loaded input lane - .long 0x6dbc27e8 // stp d8, d9, [sp, #-64]! - .long 0x6d012fea // stp d10, d11, [sp, #16] - .long 0x6d0237ec // stp d12, d13, [sp, #32] - .long 0x6d033fee // stp d14, d15, [sp, #48] + {$I ..\Include\Simd\Common\SimdNonVolatileSave_aarch64.inc} .long 0x6d400400 // ldp d0, d1, [x0] .long 0x6d410c02 // ldp d2, d3, [x0, #16] .long 0x6d421404 // ldp d4, d5, [x0, #32] @@ -203,8 +200,5 @@ .long 0x6d0a5414 // stp d20, d21, [x0, #160] .long 0x6d0b5c16 // stp d22, d23, [x0, #176] .long 0xfd006018 // str d24, [x0, #192] - .long 0x6d412fea // ldp d10, d11, [sp, #16] - .long 0x6d4237ec // ldp d12, d13, [sp, #32] - .long 0x6d433fee // ldp d14, d15, [sp, #48] - .long 0x6cc427e8 // ldp d8, d9, [sp], #64 + {$I ..\Include\Simd\Common\SimdNonVolatileRestore_aarch64.inc} ret diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExt_aarch64.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExt_aarch64.inc index dd56a74..e619255 100644 --- a/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExt_aarch64.inc +++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExt_aarch64.inc @@ -11,10 +11,7 @@ // v26 = iota round constant (ld1r) // v30-v31 = Rho/Pi scratch - .long 0x6dbc27e8 // stp d8, d9, [sp, #-64]! - .long 0x6d012fea // stp d10, d11, [sp, #16] - .long 0x6d0237ec // stp d12, d13, [sp, #32] - .long 0x6d033fee // stp d14, d15, [sp, #48] + {$I ..\Include\Simd\Common\SimdNonVolatileSave_aarch64.inc} .long 0x6d400400 // ldp d0, d1, [x0] .long 0x6d410c02 // ldp d2, d3, [x0, #16] .long 0x6d421404 // ldp d4, d5, [x0, #32] @@ -112,8 +109,5 @@ .long 0x6d0a5414 // stp d20, d21, [x0, #160] .long 0x6d0b5c16 // stp d22, d23, [x0, #176] .long 0xfd006018 // str d24, [x0, #192] - .long 0x6d412fea // ldp d10, d11, [sp, #16] - .long 0x6d4237ec // ldp d12, d13, [sp, #32] - .long 0x6d433fee // ldp d14, d15, [sp, #48] - .long 0x6cc427e8 // ldp d8, d9, [sp], #64 + {$I ..\Include\Simd\Common\SimdNonVolatileRestore_aarch64.inc} ret