From ff0f0070b5d7fc2246cf0c5fb90be4ec4c87e7d0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 16:31:07 +0000 Subject: [PATCH 1/7] Replace ExtractMostSignificantBits+BitOp patterns with Vector helpers Replace patterns of ExtractMostSignificantBits() followed by PopCount/TrailingZeroCount/LeadingZeroCount with the optimized Vector helpers: CountWhereAllBitsSet, IndexOfWhereAllBitsSet, and LastIndexOfWhereAllBitsSet. Remove AdvSimd special paths from Vector64/Vector128 internal helpers (CountMatches, IndexOfFirstMatch, IndexOfLastMatch) and the now-unused AdvSimdExtractBitMask/AdvSimdFixupBitCount methods. Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/fcd7f0bb-7e64-41fa-8773-089705b9a737 Co-authored-by: EgorBo <523221+EgorBo@users.noreply.github.com> --- .../TensorPrimitives.HammingDistance.cs | 12 +-- .../netcore/TensorPrimitives.IndexOfMax.cs | 6 +- .../System/Runtime/Intrinsics/Vector128.cs | 83 +----------------- .../src/System/Runtime/Intrinsics/Vector64.cs | 84 +------------------ .../src/System/SpanHelpers.T.cs | 6 +- 5 files changed, 20 insertions(+), 171 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.HammingDistance.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.HammingDistance.cs index c38b60e5f0ac3b..c9dd58c9af5144 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.HammingDistance.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.HammingDistance.cs @@ -88,7 +88,7 @@ private static int CountUnequalElements(ReadOnlySpan x, ReadOnlySpan y) Vector512 xVec = Vector512.LoadUnsafe(ref xRef, (uint)i); Vector512 yVec = Vector512.LoadUnsafe(ref yRef, (uint)i); - count += BitOperations.PopCount((~Vector512.Equals(xVec, yVec)).ExtractMostSignificantBits()); + count += Vector512.CountWhereAllBitsSet(~Vector512.Equals(xVec, yVec)); i += Vector512.Count; } @@ -104,7 +104,7 @@ private static int CountUnequalElements(ReadOnlySpan x, ReadOnlySpan y) xVec &= remainderMask; yVec &= remainderMask; - count += BitOperations.PopCount((~Vector512.Equals(xVec, yVec)).ExtractMostSignificantBits()); + count += Vector512.CountWhereAllBitsSet(~Vector512.Equals(xVec, yVec)); } } else @@ -120,7 +120,7 @@ private static int CountUnequalElements(ReadOnlySpan x, ReadOnlySpan y) Vector256 xVec = Vector256.LoadUnsafe(ref xRef, (uint)i); Vector256 yVec = Vector256.LoadUnsafe(ref yRef, (uint)i); - count += BitOperations.PopCount((~Vector256.Equals(xVec, yVec)).ExtractMostSignificantBits()); + count += Vector256.CountWhereAllBitsSet(~Vector256.Equals(xVec, yVec)); i += Vector256.Count; } @@ -136,7 +136,7 @@ private static int CountUnequalElements(ReadOnlySpan x, ReadOnlySpan y) xVec &= remainderMask; yVec &= remainderMask; - count += BitOperations.PopCount((~Vector256.Equals(xVec, yVec)).ExtractMostSignificantBits()); + count += Vector256.CountWhereAllBitsSet(~Vector256.Equals(xVec, yVec)); } } } @@ -153,7 +153,7 @@ private static int CountUnequalElements(ReadOnlySpan x, ReadOnlySpan y) Vector128 xVec = Vector128.LoadUnsafe(ref xRef, (uint)i); Vector128 yVec = Vector128.LoadUnsafe(ref yRef, (uint)i); - count += BitOperations.PopCount((~Vector128.Equals(xVec, yVec)).ExtractMostSignificantBits()); + count += Vector128.CountWhereAllBitsSet(~Vector128.Equals(xVec, yVec)); i += Vector128.Count; } @@ -169,7 +169,7 @@ private static int CountUnequalElements(ReadOnlySpan x, ReadOnlySpan y) xVec &= remainderMask; yVec &= remainderMask; - count += BitOperations.PopCount((~Vector128.Equals(xVec, yVec)).ExtractMostSignificantBits()); + count += Vector128.CountWhereAllBitsSet(~Vector128.Equals(xVec, yVec)); } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs index f40f7e1e2e2ba0..4a8778cea5b141 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs @@ -420,13 +420,13 @@ static Vector128 CreateVector128T(int i) => } private static int IndexOfFirstMatch(Vector128 mask) => - BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); + Vector128.IndexOfWhereAllBitsSet(mask); private static int IndexOfFirstMatch(Vector256 mask) => - BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); + Vector256.IndexOfWhereAllBitsSet(mask); private static int IndexOfFirstMatch(Vector512 mask) => - BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); + Vector512.IndexOfWhereAllBitsSet(mask); [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe Vector256 IndexLessThan(Vector256 indices1, Vector256 indices2) => diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs index 1cebc6f55a6b81..626e5d8cebc216 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs @@ -4502,64 +4502,7 @@ public static Vector128 WithUpper(this Vector128 vector, Vector64 va public static Vector128 Xor(Vector128 left, Vector128 right) => left ^ right; [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(AdvSimd))] - internal static ulong AdvSimdExtractBitMask(Vector128 vector) - { - if (!AdvSimd.IsSupported) - { - ThrowHelper.ThrowNotSupportedException(); - } - - // This expects vector to have each element be one of Zero or AllBitsSet - // and will not produce correct results otherwise. - // - // Given this, we can treat it as ushort and do a logical-right-shift by 4 to - // compact the mask into half the space, giving us the following possibilities for - // each pair of bytes: - // * 0x00_00 - 0x00 - // * 0x00_FF - 0x0F - // * 0xFF_00 - 0xF0 - // * 0xFF_FF - 0xFF - // - // This allows us to extract the full metadata as a 64-bit scalar which can then - // be consumed by bit-counting APIs, such as PopCount, LeadingZeroCount, or TrailingZeroCount, - // and then adjusted by AdvSimdFixupBitCount to get the actual count of elements - // that were masked. - - return AdvSimd.ShiftRightLogicalNarrowingLower(vector.AsUInt16(), 4).AsUInt64().ToScalar(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(AdvSimd))] - internal static int AdvSimdFixupBitCount(int bitCount) - { - if (!AdvSimd.IsSupported) - { - ThrowHelper.ThrowNotSupportedException(); - } - - // This API is meant to be consumed alongside AdvSimdExtractBitMask and will - // not produce correct results for arbitrary inputs. It adjusts the bit count - // assuming that sequences of 1 or 0 were in groups of 4 bits per byte. - - unsafe - { - return bitCount >>> (2 + int.Log2(sizeof(T))); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int CountMatches(Vector128 vector) - { - if (AdvSimd.IsSupported) - { - return AdvSimdFixupBitCount(BitOperations.PopCount(AdvSimdExtractBitMask(vector))); - } - else - { - return BitOperations.PopCount(vector.ExtractMostSignificantBits()); - } - } + internal static int CountMatches(Vector128 vector) => BitOperations.PopCount(vector.ExtractMostSignificantBits()); [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static T GetElementUnsafe(in this Vector128 vector, int index) @@ -4572,30 +4515,12 @@ internal static T GetElementUnsafe(in this Vector128 vector, int index) [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static int IndexOfFirstMatch(Vector128 vector) { - if (AdvSimd.IsSupported) - { - int result = AdvSimdFixupBitCount(BitOperations.TrailingZeroCount(AdvSimdExtractBitMask(vector))); - return (result != Vector128.Count) ? result : -1; - } - else - { - int result = BitOperations.TrailingZeroCount(vector.ExtractMostSignificantBits()); - return (result != 32) ? result : -1; - } + int result = BitOperations.TrailingZeroCount(vector.ExtractMostSignificantBits()); + return (result != 32) ? result : -1; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int IndexOfLastMatch(Vector128 vector) - { - if (AdvSimd.IsSupported) - { - return (Vector128.Count - 1) - AdvSimdFixupBitCount(BitOperations.LeadingZeroCount(AdvSimdExtractBitMask(vector))); - } - else - { - return 31 - BitOperations.LeadingZeroCount(vector.ExtractMostSignificantBits()); - } - } + internal static int IndexOfLastMatch(Vector128 vector) => 31 - BitOperations.LeadingZeroCount(vector.ExtractMostSignificantBits()); [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static void SetElementUnsafe(in this Vector128 vector, int index, T value) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs index 8655d9778f0529..1cccde196ab01b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs @@ -6,7 +6,6 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using System.Runtime.Intrinsics.Arm; namespace System.Runtime.Intrinsics { @@ -4398,64 +4397,7 @@ public static Vector64 WithElement(this Vector64 vector, int index, T v public static Vector64 Xor(Vector64 left, Vector64 right) => left ^ right; [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(AdvSimd))] - internal static uint AdvSimdExtractBitMask(Vector64 vector) - { - if (!AdvSimd.IsSupported) - { - ThrowHelper.ThrowNotSupportedException(); - } - - // This expects vector to have each element be one of Zero or AllBitsSet - // and will not produce correct results otherwise. - // - // Given this, we can treat it as ushort and do a logical-right-shift by 4 to - // compact the mask into half the space, giving us the following possibilities for - // each pair of bytes: - // * 0x00_00 - 0x00 - // * 0x00_FF - 0x0F - // * 0xFF_00 - 0xF0 - // * 0xFF_FF - 0xFF - // - // This allows us to extract the full metadata as a 32-bit scalar which can then - // be consumed by bit-counting APIs, such as PopCount, LeadingZeroCount, or TrailingZeroCount, - // and then adjusted by AdvSimdFixupBitCount to get the actual count of elements - // that were masked. - - return AdvSimd.ShiftRightLogicalNarrowingLower(vector.ToVector128().AsUInt16(), 4).AsUInt32().ToScalar(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(AdvSimd))] - internal static int AdvSimdFixupBitCount(int bitCount) - { - if (!AdvSimd.IsSupported) - { - ThrowHelper.ThrowNotSupportedException(); - } - - // This API is meant to be consumed alongside AdvSimdExtractBitMask and will - // not produce correct results for arbitrary inputs. It adjusts the bit count - // assuming that sequences of 1 or 0 were in groups of 4 bits per byte. - - unsafe - { - return bitCount >>> (2 + int.Log2(sizeof(T))); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int CountMatches(Vector64 vector) - { - if (AdvSimd.IsSupported) - { - return AdvSimdFixupBitCount(BitOperations.PopCount(AdvSimdExtractBitMask(vector))); - } - else - { - return BitOperations.PopCount(vector.ExtractMostSignificantBits()); - } - } + internal static int CountMatches(Vector64 vector) => BitOperations.PopCount(vector.ExtractMostSignificantBits()); [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static T GetElementUnsafe(in this Vector64 vector, int index) @@ -4468,30 +4410,12 @@ internal static T GetElementUnsafe(in this Vector64 vector, int index) [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static int IndexOfFirstMatch(Vector64 vector) { - if (AdvSimd.IsSupported) - { - int result = AdvSimdFixupBitCount(BitOperations.TrailingZeroCount(AdvSimdExtractBitMask(vector))); - return (result != Vector64.Count) ? result : -1; - } - else - { - int result = BitOperations.TrailingZeroCount(vector.ExtractMostSignificantBits()); - return (result != 32) ? result : -1; - } + int result = BitOperations.TrailingZeroCount(vector.ExtractMostSignificantBits()); + return (result != 32) ? result : -1; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int IndexOfLastMatch(Vector64 vector) - { - if (AdvSimd.IsSupported) - { - return (Vector64.Count - 1) - AdvSimdFixupBitCount(BitOperations.LeadingZeroCount(AdvSimdExtractBitMask(vector))); - } - else - { - return 31 - BitOperations.LeadingZeroCount(vector.ExtractMostSignificantBits()); - } - } + internal static int IndexOfLastMatch(Vector64 vector) => 31 - BitOperations.LeadingZeroCount(vector.ExtractMostSignificantBits()); [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static void SetElementUnsafe(in this Vector64 vector, int index, T value) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index 8e53a21801ec29..12278446319180 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -4176,7 +4176,7 @@ public static unsafe int CountValueType(ref T current, T value, int length) w ref T oneVectorAwayFromEnd = ref Unsafe.Subtract(ref end, Vector512.Count); while (Unsafe.IsAddressLessThan(ref current, ref oneVectorAwayFromEnd)) { - count += BitOperations.PopCount(Vector512.Equals(Vector512.LoadUnsafe(ref current), targetVector).ExtractMostSignificantBits()); + count += Vector512.CountWhereAllBitsSet(Vector512.Equals(Vector512.LoadUnsafe(ref current), targetVector)); current = ref Unsafe.Add(ref current, Vector512.Count); } @@ -4191,7 +4191,7 @@ public static unsafe int CountValueType(ref T current, T value, int length) w ref T oneVectorAwayFromEnd = ref Unsafe.Subtract(ref end, Vector256.Count); while (Unsafe.IsAddressLessThan(ref current, ref oneVectorAwayFromEnd)) { - count += BitOperations.PopCount(Vector256.Equals(Vector256.LoadUnsafe(ref current), targetVector).ExtractMostSignificantBits()); + count += Vector256.CountWhereAllBitsSet(Vector256.Equals(Vector256.LoadUnsafe(ref current), targetVector)); current = ref Unsafe.Add(ref current, Vector256.Count); } @@ -4206,7 +4206,7 @@ public static unsafe int CountValueType(ref T current, T value, int length) w ref T oneVectorAwayFromEnd = ref Unsafe.Subtract(ref end, Vector128.Count); while (Unsafe.IsAddressLessThan(ref current, ref oneVectorAwayFromEnd)) { - count += BitOperations.PopCount(Vector128.Equals(Vector128.LoadUnsafe(ref current), targetVector).ExtractMostSignificantBits()); + count += Vector128.CountWhereAllBitsSet(Vector128.Equals(Vector128.LoadUnsafe(ref current), targetVector)); current = ref Unsafe.Add(ref current, Vector128.Count); } From 31f94e327960b9f7dc1f26b51820baa1f00083e5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 17:34:54 +0000 Subject: [PATCH 2/7] Remove IndexOfFirstMatch wrappers; inline IndexOfWhereAllBitsSet directly Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/fd3915a5-bbf3-4530-8406-f3983be66ffb Co-authored-by: tannergooding <10487869+tannergooding@users.noreply.github.com> --- .../netcore/TensorPrimitives.IndexOfMax.cs | 25 +++++++------------ .../Tensors/netcore/TensorPrimitives.Max.cs | 18 ++++++------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs index 4a8778cea5b141..884109a8e2057c 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs @@ -176,7 +176,7 @@ static Vector512 CreateVector512T(int i) => nanMask = ~Vector512.Equals(result, result); if (nanMask != Vector512.Zero) { - return IndexOfFirstMatch(nanMask); + return Vector512.IndexOfWhereAllBitsSet(nanMask); } } @@ -195,7 +195,7 @@ static Vector512 CreateVector512T(int i) => nanMask = ~Vector512.Equals(current, current); if (nanMask != Vector512.Zero) { - return i + IndexOfFirstMatch(nanMask); + return i + Vector512.IndexOfWhereAllBitsSet(nanMask); } } @@ -215,7 +215,7 @@ static Vector512 CreateVector512T(int i) => nanMask = ~Vector512.Equals(current, current); if (nanMask != Vector512.Zero) { - int indexInVectorOfFirstMatch = IndexOfFirstMatch(nanMask); + int indexInVectorOfFirstMatch = Vector512.IndexOfWhereAllBitsSet(nanMask); return typeof(T) == typeof(double) ? (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; @@ -260,7 +260,7 @@ static Vector256 CreateVector256T(int i) => nanMask = ~Vector256.Equals(result, result); if (nanMask != Vector256.Zero) { - return IndexOfFirstMatch(nanMask); + return Vector256.IndexOfWhereAllBitsSet(nanMask); } } @@ -279,7 +279,7 @@ static Vector256 CreateVector256T(int i) => nanMask = ~Vector256.Equals(current, current); if (nanMask != Vector256.Zero) { - return i + IndexOfFirstMatch(nanMask); + return i + Vector256.IndexOfWhereAllBitsSet(nanMask); } } @@ -299,7 +299,7 @@ static Vector256 CreateVector256T(int i) => nanMask = ~Vector256.Equals(current, current); if (nanMask != Vector256.Zero) { - int indexInVectorOfFirstMatch = IndexOfFirstMatch(nanMask); + int indexInVectorOfFirstMatch = Vector256.IndexOfWhereAllBitsSet(nanMask); return typeof(T) == typeof(double) ? (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; @@ -344,7 +344,7 @@ static Vector128 CreateVector128T(int i) => nanMask = ~Vector128.Equals(result, result); if (nanMask != Vector128.Zero) { - return IndexOfFirstMatch(nanMask); + return Vector128.IndexOfWhereAllBitsSet(nanMask); } } @@ -363,7 +363,7 @@ static Vector128 CreateVector128T(int i) => nanMask = ~Vector128.Equals(current, current); if (nanMask != Vector128.Zero) { - return i + IndexOfFirstMatch(nanMask); + return i + Vector128.IndexOfWhereAllBitsSet(nanMask); } } @@ -383,7 +383,7 @@ static Vector128 CreateVector128T(int i) => nanMask = ~Vector128.Equals(current, current); if (nanMask != Vector128.Zero) { - int indexInVectorOfFirstMatch = IndexOfFirstMatch(nanMask); + int indexInVectorOfFirstMatch = Vector128.IndexOfWhereAllBitsSet(nanMask); return typeof(T) == typeof(double) ? (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; @@ -419,14 +419,7 @@ static Vector128 CreateVector128T(int i) => return curIn; } - private static int IndexOfFirstMatch(Vector128 mask) => - Vector128.IndexOfWhereAllBitsSet(mask); - private static int IndexOfFirstMatch(Vector256 mask) => - Vector256.IndexOfWhereAllBitsSet(mask); - - private static int IndexOfFirstMatch(Vector512 mask) => - Vector512.IndexOfWhereAllBitsSet(mask); [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe Vector256 IndexLessThan(Vector256 indices1, Vector256 indices2) => diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs index 8e42381c48e165..2f16e8b6b8d4f2 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs @@ -258,7 +258,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = IsNaN(result); if (nanMask != Vector512.Zero) { - return result.GetElement(IndexOfFirstMatch(nanMask)); + return result.GetElement(Vector512.IndexOfWhereAllBitsSet(nanMask)); } } @@ -277,7 +277,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector512.Equals(current, current); if (nanMask != Vector512.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector512.IndexOfWhereAllBitsSet(nanMask)); } } @@ -296,7 +296,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector512.Equals(current, current); if (nanMask != Vector512.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector512.IndexOfWhereAllBitsSet(nanMask)); } } @@ -323,7 +323,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector256.Equals(result, result); if (nanMask != Vector256.Zero) { - return result.GetElement(IndexOfFirstMatch(nanMask)); + return result.GetElement(Vector256.IndexOfWhereAllBitsSet(nanMask)); } } @@ -342,7 +342,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector256.Equals(current, current); if (nanMask != Vector256.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector256.IndexOfWhereAllBitsSet(nanMask)); } } @@ -362,7 +362,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector256.Equals(current, current); if (nanMask != Vector256.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector256.IndexOfWhereAllBitsSet(nanMask)); } } @@ -389,7 +389,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = IsNaN(result); if (nanMask != Vector128.Zero) { - return result.GetElement(IndexOfFirstMatch(nanMask)); + return result.GetElement(Vector128.IndexOfWhereAllBitsSet(nanMask)); } } @@ -408,7 +408,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = IsNaN(current); if (nanMask != Vector128.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector128.IndexOfWhereAllBitsSet(nanMask)); } } @@ -427,7 +427,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = IsNaN(current); if (nanMask != Vector128.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector128.IndexOfWhereAllBitsSet(nanMask)); } } From 499c77ec2f03d6af75198cfe5c12777680caf8d9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 18:02:53 +0000 Subject: [PATCH 3/7] Revert Vector128.cs, Vector64.cs, and HammingDistance.cs changes Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/527f6141-5f1f-41b3-bb40-336677a9c787 Co-authored-by: tannergooding <10487869+tannergooding@users.noreply.github.com> --- .../TensorPrimitives.HammingDistance.cs | 12 +-- .../System/Runtime/Intrinsics/Vector128.cs | 83 +++++++++++++++++- .../src/System/Runtime/Intrinsics/Vector64.cs | 84 ++++++++++++++++++- 3 files changed, 165 insertions(+), 14 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.HammingDistance.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.HammingDistance.cs index c9dd58c9af5144..c38b60e5f0ac3b 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.HammingDistance.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.HammingDistance.cs @@ -88,7 +88,7 @@ private static int CountUnequalElements(ReadOnlySpan x, ReadOnlySpan y) Vector512 xVec = Vector512.LoadUnsafe(ref xRef, (uint)i); Vector512 yVec = Vector512.LoadUnsafe(ref yRef, (uint)i); - count += Vector512.CountWhereAllBitsSet(~Vector512.Equals(xVec, yVec)); + count += BitOperations.PopCount((~Vector512.Equals(xVec, yVec)).ExtractMostSignificantBits()); i += Vector512.Count; } @@ -104,7 +104,7 @@ private static int CountUnequalElements(ReadOnlySpan x, ReadOnlySpan y) xVec &= remainderMask; yVec &= remainderMask; - count += Vector512.CountWhereAllBitsSet(~Vector512.Equals(xVec, yVec)); + count += BitOperations.PopCount((~Vector512.Equals(xVec, yVec)).ExtractMostSignificantBits()); } } else @@ -120,7 +120,7 @@ private static int CountUnequalElements(ReadOnlySpan x, ReadOnlySpan y) Vector256 xVec = Vector256.LoadUnsafe(ref xRef, (uint)i); Vector256 yVec = Vector256.LoadUnsafe(ref yRef, (uint)i); - count += Vector256.CountWhereAllBitsSet(~Vector256.Equals(xVec, yVec)); + count += BitOperations.PopCount((~Vector256.Equals(xVec, yVec)).ExtractMostSignificantBits()); i += Vector256.Count; } @@ -136,7 +136,7 @@ private static int CountUnequalElements(ReadOnlySpan x, ReadOnlySpan y) xVec &= remainderMask; yVec &= remainderMask; - count += Vector256.CountWhereAllBitsSet(~Vector256.Equals(xVec, yVec)); + count += BitOperations.PopCount((~Vector256.Equals(xVec, yVec)).ExtractMostSignificantBits()); } } } @@ -153,7 +153,7 @@ private static int CountUnequalElements(ReadOnlySpan x, ReadOnlySpan y) Vector128 xVec = Vector128.LoadUnsafe(ref xRef, (uint)i); Vector128 yVec = Vector128.LoadUnsafe(ref yRef, (uint)i); - count += Vector128.CountWhereAllBitsSet(~Vector128.Equals(xVec, yVec)); + count += BitOperations.PopCount((~Vector128.Equals(xVec, yVec)).ExtractMostSignificantBits()); i += Vector128.Count; } @@ -169,7 +169,7 @@ private static int CountUnequalElements(ReadOnlySpan x, ReadOnlySpan y) xVec &= remainderMask; yVec &= remainderMask; - count += Vector128.CountWhereAllBitsSet(~Vector128.Equals(xVec, yVec)); + count += BitOperations.PopCount((~Vector128.Equals(xVec, yVec)).ExtractMostSignificantBits()); } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs index 626e5d8cebc216..1cebc6f55a6b81 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs @@ -4502,7 +4502,64 @@ public static Vector128 WithUpper(this Vector128 vector, Vector64 va public static Vector128 Xor(Vector128 left, Vector128 right) => left ^ right; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int CountMatches(Vector128 vector) => BitOperations.PopCount(vector.ExtractMostSignificantBits()); + [CompExactlyDependsOn(typeof(AdvSimd))] + internal static ulong AdvSimdExtractBitMask(Vector128 vector) + { + if (!AdvSimd.IsSupported) + { + ThrowHelper.ThrowNotSupportedException(); + } + + // This expects vector to have each element be one of Zero or AllBitsSet + // and will not produce correct results otherwise. + // + // Given this, we can treat it as ushort and do a logical-right-shift by 4 to + // compact the mask into half the space, giving us the following possibilities for + // each pair of bytes: + // * 0x00_00 - 0x00 + // * 0x00_FF - 0x0F + // * 0xFF_00 - 0xF0 + // * 0xFF_FF - 0xFF + // + // This allows us to extract the full metadata as a 64-bit scalar which can then + // be consumed by bit-counting APIs, such as PopCount, LeadingZeroCount, or TrailingZeroCount, + // and then adjusted by AdvSimdFixupBitCount to get the actual count of elements + // that were masked. + + return AdvSimd.ShiftRightLogicalNarrowingLower(vector.AsUInt16(), 4).AsUInt64().ToScalar(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(AdvSimd))] + internal static int AdvSimdFixupBitCount(int bitCount) + { + if (!AdvSimd.IsSupported) + { + ThrowHelper.ThrowNotSupportedException(); + } + + // This API is meant to be consumed alongside AdvSimdExtractBitMask and will + // not produce correct results for arbitrary inputs. It adjusts the bit count + // assuming that sequences of 1 or 0 were in groups of 4 bits per byte. + + unsafe + { + return bitCount >>> (2 + int.Log2(sizeof(T))); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int CountMatches(Vector128 vector) + { + if (AdvSimd.IsSupported) + { + return AdvSimdFixupBitCount(BitOperations.PopCount(AdvSimdExtractBitMask(vector))); + } + else + { + return BitOperations.PopCount(vector.ExtractMostSignificantBits()); + } + } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static T GetElementUnsafe(in this Vector128 vector, int index) @@ -4515,12 +4572,30 @@ internal static T GetElementUnsafe(in this Vector128 vector, int index) [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static int IndexOfFirstMatch(Vector128 vector) { - int result = BitOperations.TrailingZeroCount(vector.ExtractMostSignificantBits()); - return (result != 32) ? result : -1; + if (AdvSimd.IsSupported) + { + int result = AdvSimdFixupBitCount(BitOperations.TrailingZeroCount(AdvSimdExtractBitMask(vector))); + return (result != Vector128.Count) ? result : -1; + } + else + { + int result = BitOperations.TrailingZeroCount(vector.ExtractMostSignificantBits()); + return (result != 32) ? result : -1; + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int IndexOfLastMatch(Vector128 vector) => 31 - BitOperations.LeadingZeroCount(vector.ExtractMostSignificantBits()); + internal static int IndexOfLastMatch(Vector128 vector) + { + if (AdvSimd.IsSupported) + { + return (Vector128.Count - 1) - AdvSimdFixupBitCount(BitOperations.LeadingZeroCount(AdvSimdExtractBitMask(vector))); + } + else + { + return 31 - BitOperations.LeadingZeroCount(vector.ExtractMostSignificantBits()); + } + } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static void SetElementUnsafe(in this Vector128 vector, int index, T value) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs index 1cccde196ab01b..8655d9778f0529 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs @@ -6,6 +6,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.Arm; namespace System.Runtime.Intrinsics { @@ -4397,7 +4398,64 @@ public static Vector64 WithElement(this Vector64 vector, int index, T v public static Vector64 Xor(Vector64 left, Vector64 right) => left ^ right; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int CountMatches(Vector64 vector) => BitOperations.PopCount(vector.ExtractMostSignificantBits()); + [CompExactlyDependsOn(typeof(AdvSimd))] + internal static uint AdvSimdExtractBitMask(Vector64 vector) + { + if (!AdvSimd.IsSupported) + { + ThrowHelper.ThrowNotSupportedException(); + } + + // This expects vector to have each element be one of Zero or AllBitsSet + // and will not produce correct results otherwise. + // + // Given this, we can treat it as ushort and do a logical-right-shift by 4 to + // compact the mask into half the space, giving us the following possibilities for + // each pair of bytes: + // * 0x00_00 - 0x00 + // * 0x00_FF - 0x0F + // * 0xFF_00 - 0xF0 + // * 0xFF_FF - 0xFF + // + // This allows us to extract the full metadata as a 32-bit scalar which can then + // be consumed by bit-counting APIs, such as PopCount, LeadingZeroCount, or TrailingZeroCount, + // and then adjusted by AdvSimdFixupBitCount to get the actual count of elements + // that were masked. + + return AdvSimd.ShiftRightLogicalNarrowingLower(vector.ToVector128().AsUInt16(), 4).AsUInt32().ToScalar(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(AdvSimd))] + internal static int AdvSimdFixupBitCount(int bitCount) + { + if (!AdvSimd.IsSupported) + { + ThrowHelper.ThrowNotSupportedException(); + } + + // This API is meant to be consumed alongside AdvSimdExtractBitMask and will + // not produce correct results for arbitrary inputs. It adjusts the bit count + // assuming that sequences of 1 or 0 were in groups of 4 bits per byte. + + unsafe + { + return bitCount >>> (2 + int.Log2(sizeof(T))); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int CountMatches(Vector64 vector) + { + if (AdvSimd.IsSupported) + { + return AdvSimdFixupBitCount(BitOperations.PopCount(AdvSimdExtractBitMask(vector))); + } + else + { + return BitOperations.PopCount(vector.ExtractMostSignificantBits()); + } + } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static T GetElementUnsafe(in this Vector64 vector, int index) @@ -4410,12 +4468,30 @@ internal static T GetElementUnsafe(in this Vector64 vector, int index) [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static int IndexOfFirstMatch(Vector64 vector) { - int result = BitOperations.TrailingZeroCount(vector.ExtractMostSignificantBits()); - return (result != 32) ? result : -1; + if (AdvSimd.IsSupported) + { + int result = AdvSimdFixupBitCount(BitOperations.TrailingZeroCount(AdvSimdExtractBitMask(vector))); + return (result != Vector64.Count) ? result : -1; + } + else + { + int result = BitOperations.TrailingZeroCount(vector.ExtractMostSignificantBits()); + return (result != 32) ? result : -1; + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int IndexOfLastMatch(Vector64 vector) => 31 - BitOperations.LeadingZeroCount(vector.ExtractMostSignificantBits()); + internal static int IndexOfLastMatch(Vector64 vector) + { + if (AdvSimd.IsSupported) + { + return (Vector64.Count - 1) - AdvSimdFixupBitCount(BitOperations.LeadingZeroCount(AdvSimdExtractBitMask(vector))); + } + else + { + return 31 - BitOperations.LeadingZeroCount(vector.ExtractMostSignificantBits()); + } + } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static void SetElementUnsafe(in this Vector64 vector, int index, T value) From 8dc533d1608b9d5bfcbba2ab22faf656b184e67d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 18:37:28 +0000 Subject: [PATCH 4/7] Use CountMatches internal helper in SpanHelpers.T.cs to avoid x64 regression Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/83aad277-6910-4c67-8aac-88c2e15047b3 Co-authored-by: tannergooding <10487869+tannergooding@users.noreply.github.com> --- .../System.Private.CoreLib/src/System/SpanHelpers.T.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index 12278446319180..fdba77200e94a7 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -4176,7 +4176,7 @@ public static unsafe int CountValueType(ref T current, T value, int length) w ref T oneVectorAwayFromEnd = ref Unsafe.Subtract(ref end, Vector512.Count); while (Unsafe.IsAddressLessThan(ref current, ref oneVectorAwayFromEnd)) { - count += Vector512.CountWhereAllBitsSet(Vector512.Equals(Vector512.LoadUnsafe(ref current), targetVector)); + count += Vector512.CountMatches(Vector512.Equals(Vector512.LoadUnsafe(ref current), targetVector)); current = ref Unsafe.Add(ref current, Vector512.Count); } @@ -4191,7 +4191,7 @@ public static unsafe int CountValueType(ref T current, T value, int length) w ref T oneVectorAwayFromEnd = ref Unsafe.Subtract(ref end, Vector256.Count); while (Unsafe.IsAddressLessThan(ref current, ref oneVectorAwayFromEnd)) { - count += Vector256.CountWhereAllBitsSet(Vector256.Equals(Vector256.LoadUnsafe(ref current), targetVector)); + count += Vector256.CountMatches(Vector256.Equals(Vector256.LoadUnsafe(ref current), targetVector)); current = ref Unsafe.Add(ref current, Vector256.Count); } @@ -4206,7 +4206,7 @@ public static unsafe int CountValueType(ref T current, T value, int length) w ref T oneVectorAwayFromEnd = ref Unsafe.Subtract(ref end, Vector128.Count); while (Unsafe.IsAddressLessThan(ref current, ref oneVectorAwayFromEnd)) { - count += Vector128.CountWhereAllBitsSet(Vector128.Equals(Vector128.LoadUnsafe(ref current), targetVector)); + count += Vector128.CountMatches(Vector128.Equals(Vector128.LoadUnsafe(ref current), targetVector)); current = ref Unsafe.Add(ref current, Vector128.Count); } From d3a08e41c5945037d24af13b238cc83645caa716 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 18:38:28 +0000 Subject: [PATCH 5/7] Clean up extra blank lines from removed IndexOfFirstMatch methods Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/83aad277-6910-4c67-8aac-88c2e15047b3 Co-authored-by: tannergooding <10487869+tannergooding@users.noreply.github.com> --- .../Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs index 884109a8e2057c..d3114e7becd199 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs @@ -419,8 +419,6 @@ static Vector128 CreateVector128T(int i) => return curIn; } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe Vector256 IndexLessThan(Vector256 indices1, Vector256 indices2) => sizeof(T) == sizeof(long) ? Vector256.LessThan(indices1.AsInt64(), indices2.AsInt64()).As() : From 9b3f39fb0b83d08b635f36ba075f59bd2bcacc40 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 21:10:02 +0000 Subject: [PATCH 6/7] Replace EMSB+BitOp patterns with IndexOfFirstMatch/IndexOfLastMatch in SpanHelpers Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/32b80803-7f44-4863-8845-f76d999fe220 Co-authored-by: EgorBo <523221+EgorBo@users.noreply.github.com> --- .../src/System/SpanHelpers.Byte.cs | 55 +++++++++---------- .../src/System/SpanHelpers.T.cs | 24 ++------ 2 files changed, 31 insertions(+), 48 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index e7194f4098ed50..67cd528d198236 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -531,8 +531,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector128 search = Vector128.Load(searchSpace + offset); // Same method as below - uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector128 cmp = Vector128.Equals(Vector128.Zero, search); + if (cmp == Vector128.Zero) { // Zero flags set so no matches offset += (nuint)Vector128.Count; @@ -540,7 +540,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(cmp)); } } @@ -553,8 +553,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector256 search = Vector256.Load(searchSpace + offset); // Same method as below - uint matches = Vector256.Equals(Vector256.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector256 cmp = Vector256.Equals(Vector256.Zero, search); + if (cmp == Vector256.Zero) { // Zero flags set so no matches offset += (nuint)Vector256.Count; @@ -562,7 +562,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector256.IndexOfFirstMatch(cmp)); } } lengthToExamine = GetByteVector512SpanLength(offset, Length); @@ -571,10 +571,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) do { Vector512 search = Vector512.Load(searchSpace + offset); - ulong matches = Vector512.Equals(Vector512.Zero, search).ExtractMostSignificantBits(); - // Note that MoveMask has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - if (matches == 0) + Vector512 cmp = Vector512.Equals(Vector512.Zero, search); + if (cmp == Vector512.Zero) { // Zero flags set so no matches offset += (nuint)Vector512.Count; @@ -582,7 +580,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) } // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector512.IndexOfFirstMatch(cmp)); } while (lengthToExamine > offset); } @@ -592,8 +590,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector256 search = Vector256.Load(searchSpace + offset); // Same method as above - uint matches = Vector256.Equals(Vector256.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector256 cmp = Vector256.Equals(Vector256.Zero, search); + if (cmp == Vector256.Zero) { // Zero flags set so no matches offset += (nuint)Vector256.Count; @@ -601,7 +599,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector256.IndexOfFirstMatch(cmp)); } } @@ -611,8 +609,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector128 search = Vector128.Load(searchSpace + offset); // Same method as above - uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector128 cmp = Vector128.Equals(Vector128.Zero, search); + if (cmp == Vector128.Zero) { // Zero flags set so no matches offset += (nuint)Vector128.Count; @@ -620,7 +618,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(cmp)); } } @@ -644,8 +642,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector128 search = Vector128.Load(searchSpace + offset); // Same method as below - uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector128 cmp = Vector128.Equals(Vector128.Zero, search); + if (cmp == Vector128.Zero) { // Zero flags set so no matches offset += (nuint)Vector128.Count; @@ -653,7 +651,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(cmp)); } } @@ -663,10 +661,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) do { Vector256 search = Vector256.Load(searchSpace + offset); - uint matches = Vector256.Equals(Vector256.Zero, search).ExtractMostSignificantBits(); - // Note that MoveMask has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - if (matches == 0) + Vector256 cmp = Vector256.Equals(Vector256.Zero, search); + if (cmp == Vector256.Zero) { // Zero flags set so no matches offset += (nuint)Vector256.Count; @@ -674,7 +670,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) } // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector256.IndexOfFirstMatch(cmp)); } while (lengthToExamine > offset); } @@ -684,8 +680,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector128 search = Vector128.Load(searchSpace + offset); // Same method as above - uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector128 cmp = Vector128.Equals(Vector128.Zero, search); + if (cmp == Vector128.Zero) { // Zero flags set so no matches offset += (nuint)Vector128.Count; @@ -693,7 +689,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(cmp)); } } @@ -724,8 +720,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) } // Find bitflag offset of first match and add to current offset - uint matches = compareResult.ExtractMostSignificantBits(); - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(compareResult)); } if (offset < (nuint)(uint)Length) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index fdba77200e94a7..7f562e59d626aa 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -3713,49 +3713,37 @@ private static int LastIndexOfAnyValueType(ref TValue searchSp [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe int ComputeFirstIndex(ref T searchSpace, ref T current, Vector128 equals) where T : struct { - uint notEqualsElements = equals.ExtractMostSignificantBits(); - int index = BitOperations.TrailingZeroCount(notEqualsElements); - return index + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); + return Vector128.IndexOfFirstMatch(equals) + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe int ComputeFirstIndex(ref T searchSpace, ref T current, Vector256 equals) where T : struct { - uint notEqualsElements = equals.ExtractMostSignificantBits(); - int index = BitOperations.TrailingZeroCount(notEqualsElements); - return index + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); + return Vector256.IndexOfFirstMatch(equals) + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe int ComputeFirstIndex(ref T searchSpace, ref T current, Vector512 equals) where T : struct { - ulong notEqualsElements = equals.ExtractMostSignificantBits(); - int index = BitOperations.TrailingZeroCount(notEqualsElements); - return index + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); + return Vector512.IndexOfFirstMatch(equals) + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ComputeLastIndex(nint offset, Vector128 equals) where T : struct { - uint notEqualsElements = equals.ExtractMostSignificantBits(); - int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero) - return (int)offset + index; + return (int)offset + Vector128.IndexOfLastMatch(equals); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ComputeLastIndex(nint offset, Vector256 equals) where T : struct { - uint notEqualsElements = equals.ExtractMostSignificantBits(); - int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero) - return (int)offset + index; + return (int)offset + Vector256.IndexOfLastMatch(equals); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ComputeLastIndex(nint offset, Vector512 equals) where T : struct { - ulong notEqualsElements = equals.ExtractMostSignificantBits(); - int index = 63 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero) - return (int)offset + index; + return (int)offset + Vector512.IndexOfLastMatch(equals); } internal interface INegator where T : struct From 5ff4a0c6de6756fc5483dc3b0c39a969b3daefa2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 25 Jun 2026 00:02:32 +0000 Subject: [PATCH 7/7] Revert TensorPrimitives.IndexOfMax.cs to original state Co-authored-by: EgorBo <523221+EgorBo@users.noreply.github.com> --- .../netcore/TensorPrimitives.IndexOfMax.cs | 408 ++---------------- 1 file changed, 35 insertions(+), 373 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs index d3114e7becd199..7ea6f2f349a797 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs @@ -1,9 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Diagnostics; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; @@ -29,412 +27,76 @@ public static int IndexOfMax(ReadOnlySpan x) IndexOfMinMaxCore>(x); /// Returns the index of MathF.Max(x, y) - internal readonly struct IndexOfMaxOperator : IIndexOfOperator where T : INumber + internal readonly struct IndexOfMaxOperator : IIndexOfMinMaxOperator where T : INumber { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 currentIndex) - { - Vector128 useResult = Vector128.GreaterThan(result, current); - Vector128 equalMask = Vector128.Equals(result, current); - - if (equalMask != Vector128.Zero) - { - Vector128 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); - Vector128 currentNegative = IsNegative(current); - Vector128 sameSign = Vector128.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } - } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } + public static T Aggregate(Vector128 x) => HorizontalAggregate>(x); + public static T Aggregate(Vector256 x) => HorizontalAggregate>(x); + public static T Aggregate(Vector512 x) => HorizontalAggregate>(x); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 currentIndex) + public static bool Compare(T x, T y) { - Vector256 useResult = Vector256.GreaterThan(result, current); - Vector256 equalMask = Vector256.Equals(result, current); - - if (equalMask != Vector256.Zero) + if (x == y) { - Vector256 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); - Vector256 currentNegative = IsNegative(current); - Vector256 sameSign = Vector256.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } + return T.IsPositive(x) && T.IsNegative(y); } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 currentIndex) - { - Vector512 useResult = Vector512.GreaterThan(result, current); - Vector512 equalMask = Vector512.Equals(result, current); - - if (equalMask != Vector512.Zero) + else { - Vector512 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); - Vector512 currentNegative = IsNegative(current); - Vector512 sameSign = Vector512.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } + return x > y; } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Invoke(ref T result, T current, int resultIndex, int currentIndex) - { - if (result == current) - { - bool resultNegative = IsNegative(result); - if ((resultNegative == IsNegative(current)) ? (currentIndex < resultIndex) : resultNegative) - { - result = current; - return currentIndex; - } - } - else if (current > result) - { - result = current; - return currentIndex; - } - - return resultIndex; - } - } - - private static unsafe int IndexOfMinMaxCore(ReadOnlySpan x) - where T : INumber - where TIndexOfMinMax : struct, IIndexOfOperator - { - if (x.IsEmpty) + public static Vector128 Compare(Vector128 x, Vector128 y) { - return -1; - } - - // This matches the IEEE 754:2019 `maximum`/`minimum` functions. - // It propagates NaN inputs back to the caller and - // otherwise returns the index of the greater of the inputs. - // It treats +0 as greater than -0 as per the specification. - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && x.Length >= Vector512.Count) - { - Debug.Assert(sizeof(T) is 1 or 2 or 4 or 8); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static Vector512 CreateVector512T(int i) => - sizeof(T) == sizeof(long) ? Vector512.Create((long)i).As() : - sizeof(T) == sizeof(int) ? Vector512.Create(i).As() : - sizeof(T) == sizeof(short) ? Vector512.Create((short)i).As() : - Vector512.Create((byte)i).As(); - - ref T xRef = ref MemoryMarshal.GetReference(x); - Vector512 resultIndex = - sizeof(T) == sizeof(long) ? Vector512.Indices.As() : - sizeof(T) == sizeof(int) ? Vector512.Indices.As() : - sizeof(T) == sizeof(short) ? Vector512.Indices.As() : - Vector512.Indices.As(); - Vector512 currentIndex = resultIndex; - Vector512 increment = CreateVector512T(Vector512.Count); - - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector512 result = Vector512.LoadUnsafe(ref xRef); - Vector512 current; - - Vector512 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + if (typeof(T) == typeof(double) || typeof(T) == typeof(float)) { - nanMask = ~Vector512.Equals(result, result); - if (nanMask != Vector512.Zero) - { - return Vector512.IndexOfWhereAllBitsSet(nanMask); - } + Vector128 equalResult = IsPositive(x) & IsNegative(y); + return Vector128.GreaterThan(x, y) | (Vector128.Equals(x, y) & equalResult); } - - int oneVectorFromEnd = x.Length - Vector512.Count; - int i = Vector512.Count; - - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) + else { - // Load the next vector, and early exit on NaN. - current = Vector512.LoadUnsafe(ref xRef, (uint)i); - currentIndex += increment; - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector512.Equals(current, current); - if (nanMask != Vector512.Zero) - { - return i + Vector512.IndexOfWhereAllBitsSet(nanMask); - } - } - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); - - i += Vector512.Count; + return Vector128.GreaterThan(x, y); } - - // If any elements remain, handle them in one final vector. - if (i != x.Length) - { - current = Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count)); - currentIndex += CreateVector512T(x.Length - i); - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector512.Equals(current, current); - if (nanMask != Vector512.Zero) - { - int indexInVectorOfFirstMatch = Vector512.IndexOfWhereAllBitsSet(nanMask); - return typeof(T) == typeof(double) ? - (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : - (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; - } - } - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); - } - - // Aggregate the lanes in the vector to create the final scalar result. - return IndexOfFinalAggregate(result, resultIndex); } - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && x.Length >= Vector256.Count) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Compare(Vector256 x, Vector256 y) { - Debug.Assert(sizeof(T) is 1 or 2 or 4 or 8); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static Vector256 CreateVector256T(int i) => - sizeof(T) == sizeof(long) ? Vector256.Create((long)i).As() : - sizeof(T) == sizeof(int) ? Vector256.Create(i).As() : - sizeof(T) == sizeof(short) ? Vector256.Create((short)i).As() : - Vector256.Create((byte)i).As(); - - ref T xRef = ref MemoryMarshal.GetReference(x); - Vector256 resultIndex = - sizeof(T) == sizeof(long) ? Vector256.Indices.As() : - sizeof(T) == sizeof(int) ? Vector256.Indices.As() : - sizeof(T) == sizeof(short) ? Vector256.Indices.As() : - Vector256.Indices.As(); - Vector256 currentIndex = resultIndex; - Vector256 increment = CreateVector256T(Vector256.Count); - - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector256 result = Vector256.LoadUnsafe(ref xRef); - Vector256 current; - - Vector256 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + if (typeof(T) == typeof(double) || typeof(T) == typeof(float)) { - nanMask = ~Vector256.Equals(result, result); - if (nanMask != Vector256.Zero) - { - return Vector256.IndexOfWhereAllBitsSet(nanMask); - } - } - - int oneVectorFromEnd = x.Length - Vector256.Count; - int i = Vector256.Count; - - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) - { - // Load the next vector, and early exit on NaN. - current = Vector256.LoadUnsafe(ref xRef, (uint)i); - currentIndex += increment; - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector256.Equals(current, current); - if (nanMask != Vector256.Zero) - { - return i + Vector256.IndexOfWhereAllBitsSet(nanMask); - } - } - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); - - i += Vector256.Count; + Vector256 equalResult = IsPositive(x) & IsNegative(y); + return Vector256.GreaterThan(x, y) | (Vector256.Equals(x, y) & equalResult); } - - // If any elements remain, handle them in one final vector. - if (i != x.Length) + else { - current = Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count)); - currentIndex += CreateVector256T(x.Length - i); - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector256.Equals(current, current); - if (nanMask != Vector256.Zero) - { - int indexInVectorOfFirstMatch = Vector256.IndexOfWhereAllBitsSet(nanMask); - return typeof(T) == typeof(double) ? - (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : - (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; - } - } - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); + return Vector256.GreaterThan(x, y); } - - // Aggregate the lanes in the vector to create the final scalar result. - return IndexOfFinalAggregate(result, resultIndex); } - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && x.Length >= Vector128.Count) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Compare(Vector512 x, Vector512 y) { - Debug.Assert(sizeof(T) is 1 or 2 or 4 or 8); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static Vector128 CreateVector128T(int i) => - sizeof(T) == sizeof(long) ? Vector128.Create((long)i).As() : - sizeof(T) == sizeof(int) ? Vector128.Create(i).As() : - sizeof(T) == sizeof(short) ? Vector128.Create((short)i).As() : - Vector128.Create((byte)i).As(); - - ref T xRef = ref MemoryMarshal.GetReference(x); - Vector128 resultIndex = - sizeof(T) == sizeof(long) ? Vector128.Indices.As() : - sizeof(T) == sizeof(int) ? Vector128.Indices.As() : - sizeof(T) == sizeof(short) ? Vector128.Indices.As() : - Vector128.Indices.As(); - Vector128 currentIndex = resultIndex; - Vector128 increment = CreateVector128T(Vector128.Count); - - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector128 result = Vector128.LoadUnsafe(ref xRef); - Vector128 current; - - Vector128 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector128.Equals(result, result); - if (nanMask != Vector128.Zero) - { - return Vector128.IndexOfWhereAllBitsSet(nanMask); - } - } - - int oneVectorFromEnd = x.Length - Vector128.Count; - int i = Vector128.Count; - - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) + if (typeof(T) == typeof(double) || typeof(T) == typeof(float)) { - // Load the next vector, and early exit on NaN. - current = Vector128.LoadUnsafe(ref xRef, (uint)i); - currentIndex += increment; - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector128.Equals(current, current); - if (nanMask != Vector128.Zero) - { - return i + Vector128.IndexOfWhereAllBitsSet(nanMask); - } - } - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); - - i += Vector128.Count; - } - - // If any elements remain, handle them in one final vector. - if (i != x.Length) - { - current = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count)); - currentIndex += CreateVector128T(x.Length - i); - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector128.Equals(current, current); - if (nanMask != Vector128.Zero) - { - int indexInVectorOfFirstMatch = Vector128.IndexOfWhereAllBitsSet(nanMask); - return typeof(T) == typeof(double) ? - (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : - (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; - } - } - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); + Vector512 equalResult = IsPositive(x) & IsNegative(y); + return Vector512.GreaterThan(x, y) | (Vector512.Equals(x, y) & equalResult); } - - // Aggregate the lanes in the vector to create the final scalar result. - return IndexOfFinalAggregate(result, resultIndex); - } - - // Scalar path used when either vectorization is not supported or the input is too small to vectorize. - T curResult = x[0]; - int curIn = 0; - if (T.IsNaN(curResult)) - { - return curIn; - } - - for (int i = 1; i < x.Length; i++) - { - T current = x[i]; - if (T.IsNaN(current)) + else { - return i; + return Vector512.GreaterThan(x, y); } - - curIn = TIndexOfMinMax.Invoke(ref curResult, current, curIn, i); } - - return curIn; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe Vector256 IndexLessThan(Vector256 indices1, Vector256 indices2) => - sizeof(T) == sizeof(long) ? Vector256.LessThan(indices1.AsInt64(), indices2.AsInt64()).As() : - sizeof(T) == sizeof(int) ? Vector256.LessThan(indices1.AsInt32(), indices2.AsInt32()).As() : - sizeof(T) == sizeof(short) ? Vector256.LessThan(indices1.AsInt16(), indices2.AsInt16()).As() : - Vector256.LessThan(indices1.AsByte(), indices2.AsByte()).As(); + private static int IndexOfFirstMatch(Vector128 mask) => + BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe Vector512 IndexLessThan(Vector512 indices1, Vector512 indices2) => - sizeof(T) == sizeof(long) ? Vector512.LessThan(indices1.AsInt64(), indices2.AsInt64()).As() : - sizeof(T) == sizeof(int) ? Vector512.LessThan(indices1.AsInt32(), indices2.AsInt32()).As() : - sizeof(T) == sizeof(short) ? Vector512.LessThan(indices1.AsInt16(), indices2.AsInt16()).As() : - Vector512.LessThan(indices1.AsByte(), indices2.AsByte()).As(); + private static int IndexOfFirstMatch(Vector256 mask) => + BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); - /// Gets whether the specified is negative. - private static bool IsNegative(T f) where T : INumberBase => T.IsNegative(f); + private static int IndexOfFirstMatch(Vector512 mask) => + BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe Vector128 ElementWiseSelect(Vector128 mask, Vector128 left, Vector128 right)