Skip to content

Rework x86 SHA SIMD kernels: AVX2 2-block + SIMD-schedule SSE2 #95

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Xor-el merged 6 commits into from
Jul 7, 2026
Merged

Rework x86 SHA SIMD kernels: AVX2 2-block + SIMD-schedule SSE2 #95

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
afe138d
Rework SHA-512 x86 SIMD kernels: AVX2 2-block + SIMD-schedule SSE2
Xor-el Jul 6, 2026
652f3e6
clean up unused sha512 bswap constant
Xor-el Jul 6, 2026
a1b01f6
Optimize SHA-256 x86 SIMD kernels (AVX2 2-block + SIMD-schedule SSE2)
Xor-el Jul 7, 2026
740be3d
Optimize SHA-1 x86 SIMD kernels (AVX2 2-block + SIMD-schedule SSE2)
Xor-el Jul 7, 2026
2e59d1d
SHA-NI enhancements
Xor-el Jul 7, 2026
2e20707
fix header comments in aarch64 CryptoExt inc files
Xor-el Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 46 additions & 41 deletions HashLib/src/Crypto/HlpSHA1Dispatch.pas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,34 @@ interface
);

{$IFDEF HASHLIB_X86_SIMD}
// BSWAP32 shuffle mask for pshufb (x86 SIMD only): reverses bytes within each
// dword. Not a SHA-1 constant; passed separately to the SIMD kernels. ARM
// byte-swaps with REV32 and needs no mask table.
// BSWAP32 shuffle mask for pshufb (x86 SIMD only): byte-swaps and reverses
// dword order in one shuffle (sha1rnds4 reads its four words in reverse). Not a
// SHA-1 constant; used only by the SHA-NI kernel. ARM byte-swaps with REV32 and
// needs no mask table.
BSWAP32_MASK: array [0 .. 3] of UInt32 = (
$00010203, $04050607, $08090A0B, $0C0D0E0F
$0C0D0E0F, $08090A0B, $04050607, $00010203
);

// Doubled SHA-1 round constants plus the AVX2 byte-swap masks, shared by the
// AVX2 and SSE2 SIMD-schedule SHA-1 kernels. Each round constant fills a 128-bit
// lane (its four dwords) and is stored twice so one table feeds both the 256-bit
// AVX2 read and the 128-bit SSE2 reads (both read at a 32-byte stride, skipping
// the duplicate halves). Only the AVX2 kernel uses the appended masks: the
// byte-swap mask (BSWAP32 pattern, twice) then a whole-vector reverse mask; the
// SSE2 kernel computes its byte-swap and needs no mask. Read unaligned, so no
// special alignment is required.
K_SHA1_Doubled: array [0 .. 43] of UInt32 = (
$5A827999, $5A827999, $5A827999, $5A827999,
$5A827999, $5A827999, $5A827999, $5A827999,
$6ED9EBA1, $6ED9EBA1, $6ED9EBA1, $6ED9EBA1,
$6ED9EBA1, $6ED9EBA1, $6ED9EBA1, $6ED9EBA1,
$8F1BBCDC, $8F1BBCDC, $8F1BBCDC, $8F1BBCDC,
$8F1BBCDC, $8F1BBCDC, $8F1BBCDC, $8F1BBCDC,
$CA62C1D6, $CA62C1D6, $CA62C1D6, $CA62C1D6,
$CA62C1D6, $CA62C1D6, $CA62C1D6, $CA62C1D6,
$00010203, $04050607, $08090A0B, $0C0D0E0F,
$00010203, $04050607, $08090A0B, $0C0D0E0F,
$0C0D0E0F, $08090A0B, $04050607, $00010203
);
{$ENDIF HASHLIB_X86_SIMD}

Expand Down Expand Up @@ -112,27 +135,22 @@ procedure SHA1_Compress_Scalar(AState, AData: Pointer; ANumBlocks: UInt32);
// =============================================================================
// SIMD implementations
//
// i386: SSE2, SSSE3
// x86_64: ShaNi, AVX2, SSSE3, SSE2
// i386: SSE2
// x86_64: ShaNi, AVX2, SSE2
// aarch64: SHA1 Crypto Extensions
// =============================================================================

{$IFDEF HASHLIB_I386_ASM}

procedure SHA1_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32);
{$I ..\Include\Simd\Common\SimdProc3Begin_i386.inc}
procedure SHA1_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32;
AConstants: Pointer);
{$I ..\Include\Simd\Common\SimdProc4Begin_i386.inc}
{$I ..\Include\Simd\SHA1\SHA1CompressSse2_i386.inc}
end;

procedure SHA1_Compress_Ssse3(AState, AData: Pointer; ANumBlocks: UInt32;
AConstants, AMask: Pointer);
{$I ..\Include\Simd\Common\SimdProc5Begin_i386.inc}
{$I ..\Include\Simd\SHA1\SHA1CompressSsse3_i386.inc}
end;

procedure SHA1_Compress_Ssse3_Wrap(AState, AData: Pointer; ANumBlocks: UInt32);
procedure SHA1_Compress_Sse2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32);
begin
SHA1_Compress_Ssse3(AState, AData, ANumBlocks, @K_SHA1, @BSWAP32_MASK);
SHA1_Compress_Sse2(AState, AData, ANumBlocks, @K_SHA1_Doubled);
end;

{$ENDIF HASHLIB_I386_ASM}
Expand All @@ -150,31 +168,26 @@ procedure SHA1_Compress_ShaNi_Wrap(AState, AData: Pointer; ANumBlocks: UInt32);
SHA1_Compress_ShaNi(AState, AData, ANumBlocks, @K_SHA1, @BSWAP32_MASK);
end;

procedure SHA1_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32);
{$I ..\Include\Simd\Common\SimdProc3Begin_x86_64.inc}
procedure SHA1_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32;
AConstants: Pointer);
{$I ..\Include\Simd\Common\SimdProc4Begin_x86_64.inc}
{$I ..\Include\Simd\SHA1\SHA1CompressSse2_x86_64.inc}
end;

procedure SHA1_Compress_Ssse3(AState, AData: Pointer; ANumBlocks: UInt32;
AConstants, AMask: Pointer);
{$I ..\Include\Simd\Common\SimdProc5Begin_x86_64.inc}
{$I ..\Include\Simd\SHA1\SHA1CompressSsse3_x86_64.inc}
end;

procedure SHA1_Compress_Ssse3_Wrap(AState, AData: Pointer; ANumBlocks: UInt32);
procedure SHA1_Compress_Sse2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32);
begin
SHA1_Compress_Ssse3(AState, AData, ANumBlocks, @K_SHA1, @BSWAP32_MASK);
SHA1_Compress_Sse2(AState, AData, ANumBlocks, @K_SHA1_Doubled);
end;

procedure SHA1_Compress_Avx2(AState, AData: Pointer; ANumBlocks: UInt32;
AConstants, AMask: Pointer);
{$I ..\Include\Simd\Common\SimdProc5Begin_x86_64.inc}
AConstants: Pointer);
{$I ..\Include\Simd\Common\SimdProc4Begin_x86_64.inc}
{$I ..\Include\Simd\SHA1\SHA1CompressAvx2_x86_64.inc}
end;

procedure SHA1_Compress_Avx2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32);
begin
SHA1_Compress_Avx2(AState, AData, ANumBlocks, @K_SHA1, @BSWAP32_MASK);
SHA1_Compress_Avx2(AState, AData, ANumBlocks, @K_SHA1_Doubled);
end;

{$ENDIF HASHLIB_X86_64_ASM}
Expand Down Expand Up @@ -202,14 +215,10 @@ procedure InitDispatch();
begin
SHA1_Compress := @SHA1_Compress_Scalar;
{$IFDEF HASHLIB_I386_ASM}
case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSSE3, TX86SimdLevel.SSE2]) of
TX86SimdLevel.SSSE3:
begin
SHA1_Compress := @SHA1_Compress_Ssse3_Wrap;
end;
case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of
TX86SimdLevel.SSE2:
begin
SHA1_Compress := @SHA1_Compress_Sse2;
SHA1_Compress := @SHA1_Compress_Sse2_Wrap;
end;
end;
{$ENDIF}
Expand All @@ -219,18 +228,14 @@ procedure InitDispatch();
SHA1_Compress := @SHA1_Compress_ShaNi_Wrap;
Exit;
end;
case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, TX86SimdLevel.SSE2]) of
case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of
TX86SimdLevel.AVX2:
begin
SHA1_Compress := @SHA1_Compress_Avx2_Wrap;
end;
TX86SimdLevel.SSSE3:
begin
SHA1_Compress := @SHA1_Compress_Ssse3_Wrap;
end;
TX86SimdLevel.SSE2:
begin
SHA1_Compress := @SHA1_Compress_Sse2;
SHA1_Compress := @SHA1_Compress_Sse2_Wrap;
end;
end;
{$ENDIF}
Expand Down
103 changes: 59 additions & 44 deletions HashLib/src/Crypto/HlpSHA2_256Dispatch.pas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,60 @@ interface

{$IFDEF HASHLIB_X86_SIMD}
// BSWAP32 shuffle mask for pshufb (x86 SIMD only): reverses bytes within each
// dword. Not a SHA-256 constant; passed separately to the SIMD kernels. ARM
// byte-swaps with REV32 and needs no mask table.
// dword. Not a SHA-256 constant; used only by the SHA-NI kernel. ARM byte-swaps
// with REV32 and needs no mask table.
BSWAP32_MASK: array [0 .. 3] of UInt32 = (
$00010203, $04050607, $08090A0B, $0C0D0E0F
);

// Doubled K256 round constants plus the three AVX2 message-schedule masks,
// shared by the AVX2 and SSE2 SIMD-schedule SHA-256 kernels. Each 128-bit K256
// quadruple is stored twice so one table feeds both the 256-bit AVX2 lanes and
// the 128-bit SSE2 reads (both read at a 32-byte stride, skipping the duplicate
// halves). Only the AVX2 kernel uses the appended masks: the byte-swap mask
// (BSWAP32 pattern, twice) occupies [128..135] and the two schedule shuffle
// masks follow at [136..143] and [144..151]; the SSE2 kernel computes its
// byte-swap and needs no mask. Derived from K256.
K256_Doubled: array [0 .. 151] of UInt32 = (
$428A2F98, $71374491, $B5C0FBCF, $E9B5DBA5,
$428A2F98, $71374491, $B5C0FBCF, $E9B5DBA5,
$3956C25B, $59F111F1, $923F82A4, $AB1C5ED5,
$3956C25B, $59F111F1, $923F82A4, $AB1C5ED5,
$D807AA98, $12835B01, $243185BE, $550C7DC3,
$D807AA98, $12835B01, $243185BE, $550C7DC3,
$72BE5D74, $80DEB1FE, $9BDC06A7, $C19BF174,
$72BE5D74, $80DEB1FE, $9BDC06A7, $C19BF174,
$E49B69C1, $EFBE4786, $0FC19DC6, $240CA1CC,
$E49B69C1, $EFBE4786, $0FC19DC6, $240CA1CC,
$2DE92C6F, $4A7484AA, $5CB0A9DC, $76F988DA,
$2DE92C6F, $4A7484AA, $5CB0A9DC, $76F988DA,
$983E5152, $A831C66D, $B00327C8, $BF597FC7,
$983E5152, $A831C66D, $B00327C8, $BF597FC7,
$C6E00BF3, $D5A79147, $06CA6351, $14292967,
$C6E00BF3, $D5A79147, $06CA6351, $14292967,
$27B70A85, $2E1B2138, $4D2C6DFC, $53380D13,
$27B70A85, $2E1B2138, $4D2C6DFC, $53380D13,
$650A7354, $766A0ABB, $81C2C92E, $92722C85,
$650A7354, $766A0ABB, $81C2C92E, $92722C85,
$A2BFE8A1, $A81A664B, $C24B8B70, $C76C51A3,
$A2BFE8A1, $A81A664B, $C24B8B70, $C76C51A3,
$D192E819, $D6990624, $F40E3585, $106AA070,
$D192E819, $D6990624, $F40E3585, $106AA070,
$19A4C116, $1E376C08, $2748774C, $34B0BCB5,
$19A4C116, $1E376C08, $2748774C, $34B0BCB5,
$391C0CB3, $4ED8AA4A, $5B9CCA4F, $682E6FF3,
$391C0CB3, $4ED8AA4A, $5B9CCA4F, $682E6FF3,
$748F82EE, $78A5636F, $84C87814, $8CC70208,
$748F82EE, $78A5636F, $84C87814, $8CC70208,
$90BEFFFA, $A4506CEB, $BEF9A3F7, $C67178F2,
$90BEFFFA, $A4506CEB, $BEF9A3F7, $C67178F2,
$00010203, $04050607, $08090A0B, $0C0D0E0F,
$00010203, $04050607, $08090A0B, $0C0D0E0F,
$03020100, $0B0A0908, $FFFFFFFF, $FFFFFFFF,
$03020100, $0B0A0908, $FFFFFFFF, $FFFFFFFF,
$FFFFFFFF, $FFFFFFFF, $03020100, $0B0A0908,
$FFFFFFFF, $FFFFFFFF, $03020100, $0B0A0908
);
{$ENDIF HASHLIB_X86_SIMD}

implementation
Expand Down Expand Up @@ -106,8 +155,8 @@ procedure SHA256_Compress_Scalar(AState, AData: Pointer; ANumBlocks: UInt32);
// =============================================================================
// SIMD implementations
//
// i386: SSE2, SSSE3
// x86_64: ShaNi, AVX2, SSSE3, SSE2
// i386: SSE2
// x86_64: ShaNi, AVX2, SSE2
// aarch64: SHA256 Crypto Extensions
// =============================================================================

Expand All @@ -119,12 +168,6 @@ procedure SHA256_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32;
{$I ..\Include\Simd\SHA256\SHA256CompressSse2_i386.inc}
end;

procedure SHA256_Compress_Ssse3(AState, AData: Pointer; ANumBlocks: UInt32;
AConstants, AMask: Pointer);
{$I ..\Include\Simd\Common\SimdProc5Begin_i386.inc}
{$I ..\Include\Simd\SHA256\SHA256CompressSsse3_i386.inc}
end;

{$ENDIF HASHLIB_I386_ASM}

{$IFDEF HASHLIB_X86_64_ASM}
Expand All @@ -146,26 +189,15 @@ procedure SHA256_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32;
{$I ..\Include\Simd\SHA256\SHA256CompressSse2_x86_64.inc}
end;

procedure SHA256_Compress_Ssse3(AState, AData: Pointer; ANumBlocks: UInt32;
AConstants, AMask: Pointer);
{$I ..\Include\Simd\Common\SimdProc5Begin_x86_64.inc}
{$I ..\Include\Simd\SHA256\SHA256CompressSsse3_x86_64.inc}
end;

procedure SHA256_Compress_Ssse3_Wrap(AState, AData: Pointer; ANumBlocks: UInt32);
begin
SHA256_Compress_Ssse3(AState, AData, ANumBlocks, @K256, @BSWAP32_MASK);
end;

procedure SHA256_Compress_Avx2(AState, AData: Pointer; ANumBlocks: UInt32;
AConstants, AMask: Pointer);
{$I ..\Include\Simd\Common\SimdProc5Begin_x86_64.inc}
AConstants: Pointer);
{$I ..\Include\Simd\Common\SimdProc4Begin_x86_64.inc}
{$I ..\Include\Simd\SHA256\SHA256CompressAvx2_x86_64.inc}
end;

procedure SHA256_Compress_Avx2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32);
begin
SHA256_Compress_Avx2(AState, AData, ANumBlocks, @K256, @BSWAP32_MASK);
SHA256_Compress_Avx2(AState, AData, ANumBlocks, @K256_Doubled);
end;

{$ENDIF HASHLIB_X86_64_ASM}
Expand All @@ -174,18 +206,9 @@ procedure SHA256_Compress_Avx2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32);

procedure SHA256_Compress_Sse2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32);
begin
SHA256_Compress_Sse2(AState, AData, ANumBlocks, @K256);
end;

{$IFDEF HASHLIB_I386_ASM}

procedure SHA256_Compress_Ssse3_Wrap(AState, AData: Pointer; ANumBlocks: UInt32);
begin
SHA256_Compress_Ssse3(AState, AData, ANumBlocks, @K256, @BSWAP32_MASK);
SHA256_Compress_Sse2(AState, AData, ANumBlocks, @K256_Doubled);
end;

{$ENDIF HASHLIB_I386_ASM}

{$ENDIF HASHLIB_X86_SIMD}

{$IFDEF HASHLIB_AARCH64_ASM}
Expand All @@ -211,11 +234,7 @@ procedure InitDispatch();
begin
SHA256_Compress := @SHA256_Compress_Scalar;
{$IFDEF HASHLIB_I386_ASM}
case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSSE3, TX86SimdLevel.SSE2]) of
TX86SimdLevel.SSSE3:
begin
SHA256_Compress := @SHA256_Compress_Ssse3_Wrap;
end;
case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of
TX86SimdLevel.SSE2:
begin
SHA256_Compress := @SHA256_Compress_Sse2_Wrap;
Expand All @@ -228,15 +247,11 @@ procedure InitDispatch();
SHA256_Compress := @SHA256_Compress_ShaNi_Wrap;
Exit;
end;
case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, TX86SimdLevel.SSE2]) of
case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of
TX86SimdLevel.AVX2:
begin
SHA256_Compress := @SHA256_Compress_Avx2_Wrap;
end;
TX86SimdLevel.SSSE3:
begin
SHA256_Compress := @SHA256_Compress_Ssse3_Wrap;
end;
TX86SimdLevel.SSE2:
begin
SHA256_Compress := @SHA256_Compress_Sse2_Wrap;
Expand Down
Loading
Loading