From 43012756a619591226892268d21b898734910bd8 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 08:02:53 +1000 Subject: [PATCH 01/19] Update String.Manipulation.cs --- .../System.Private.CoreLib/src/System/String.Manipulation.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index e19298acef92f2..ece82179ed9cf1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2114,7 +2114,7 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r ReadOnlySpan sourceSpanUInt16 = MemoryMarshal.Cast(sourceSpan); ReadOnlySpan remaining = sourceSpanUInt16; - if (Vector512.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector512.Count*2) + if (Vector512.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector512.Count) { Vector512 v1 = Vector512.Create((ushort)c); Vector512 v2 = Vector512.Create((ushort)c2); @@ -2209,7 +2209,7 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r } return; } - else if (Vector256.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector256.Count*2) + else if (Vector256.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector256.Count) { Vector256 v1 = Vector256.Create((ushort)c); Vector256 v2 = Vector256.Create((ushort)c2); From 41dcd92a2fadd9ead42e92414aed73b7978f08d4 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 08:10:51 +1000 Subject: [PATCH 02/19] Optimize handling of last vector's worth on arm64 --- .../src/System/String.Manipulation.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index ece82179ed9cf1..857455fedaebed 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2361,7 +2361,7 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r } } - while ((uint)remaining.Length >= (uint)Vector128.Count) + while ((uint)remaining.Length > (uint)Vector128.Count) { Vector128 vector = Vector128.Create(remaining); Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); @@ -2384,10 +2384,10 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r // Handle the last chunk in a vectorized way also. // We do a whole vector's worth again, but just mask out the bits we've already handled. - if (remaining.Length > 0) + Vector128 vector = Vector128.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector128.Count)); + Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); + if (cmp != Vector128.Zero) { - Vector128 vector = Vector128.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector128.Count)); - Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); int finalIndex = sourceSpanUInt16.Length - Vector128.Count; uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); while (mask != 0) From 180c80d30aecfc0502890b60c8a2d821922cde36 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 08:15:09 +1000 Subject: [PATCH 03/19] Revert previous change --- .../System.Private.CoreLib/src/System/String.Manipulation.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 857455fedaebed..635656bef31d82 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2114,7 +2114,7 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r ReadOnlySpan sourceSpanUInt16 = MemoryMarshal.Cast(sourceSpan); ReadOnlySpan remaining = sourceSpanUInt16; - if (Vector512.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector512.Count) + if (Vector512.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector512.Count*2) { Vector512 v1 = Vector512.Create((ushort)c); Vector512 v2 = Vector512.Create((ushort)c2); @@ -2209,7 +2209,7 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r } return; } - else if (Vector256.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector256.Count) + else if (Vector256.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector256.Count*2) { Vector256 v1 = Vector256.Create((ushort)c); Vector256 v2 = Vector256.Create((ushort)c2); From 4e8d707a996ea25781ce6e3bf483cad30568044f Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 08:21:41 +1000 Subject: [PATCH 04/19] Add same opt to V256/V512 & fix it on xarch --- .../src/System/String.Manipulation.cs | 57 +++++++++++-------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 635656bef31d82..af064c2f0b56bf 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2194,17 +2194,20 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r // Handle the last chunk in a vectorized way also. // We do a whole vector's worth again, but just mask out the bits we've already handled. - if (remaining.Length > 0) + if (!Avx512BW.IsSupported || remaining.Length > 0) { Vector512 vector = Vector512.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector512.Count)); Vector512 cmp = Vector512.Equals(vector, v1).AsByte() | Vector512.Equals(vector, v2).AsByte() | Vector512.Equals(vector, v3).AsByte(); - int finalIndex = sourceSpanUInt16.Length - Vector512.Count; - ulong mask = cmp.ExtractMostSignificantBits() & 0x5555555555555555 & ~((1UL << (Vector512.Count - remaining.Length * sizeof(char))) - 1); - while (mask != 0) + if (cmp != Vector512.Zero) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); + int finalIndex = sourceSpanUInt16.Length - Vector512.Count; + ulong mask = cmp.ExtractMostSignificantBits() & 0x5555555555555555 & ~((1UL << (Vector512.Count - remaining.Length * sizeof(char))) - 1); + while (mask != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } } } return; @@ -2266,7 +2269,7 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r } } - while ((uint)remaining.Length >= (uint)Vector256.Count) + while ((uint)remaining.Length > (uint)Vector256.Count) { Vector256 vector = Vector256.Create(remaining); Vector256 cmp = Vector256.Equals(vector, v1).AsByte() | Vector256.Equals(vector, v2).AsByte() | Vector256.Equals(vector, v3).AsByte(); @@ -2289,17 +2292,20 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r // Handle the last chunk in a vectorized way also. // We do a whole vector's worth again, but just mask out the bits we've already handled. - if (remaining.Length > 0) + if (!Avx2.IsSupported || remaining.Length > 0) { Vector256 vector = Vector256.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector256.Count)); Vector256 cmp = Vector256.Equals(vector, v1).AsByte() | Vector256.Equals(vector, v2).AsByte() | Vector256.Equals(vector, v3).AsByte(); - int finalIndex = sourceSpanUInt16.Length - Vector256.Count; - uint mask = cmp.ExtractMostSignificantBits() & 0x55555555 & ~((1u << (Vector256.Count - remaining.Length * sizeof(char))) - 1); - while (mask != 0) + if (cmp != Vector256.Zero) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); + int finalIndex = sourceSpanUInt16.Length - Vector256.Count; + uint mask = cmp.ExtractMostSignificantBits() & 0x55555555 & ~((1u << (Vector256.Count - remaining.Length * sizeof(char))) - 1); + while (mask != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } } } return; @@ -2384,17 +2390,20 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r // Handle the last chunk in a vectorized way also. // We do a whole vector's worth again, but just mask out the bits we've already handled. - Vector128 vector = Vector128.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector128.Count)); - Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); - if (cmp != Vector128.Zero) + if (!Sse.IsSupported || remaining.Count > 0) { - int finalIndex = sourceSpanUInt16.Length - Vector128.Count; - uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); - while (mask != 0) + Vector128 vector = Vector128.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector128.Count)); + Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); + if (cmp != Vector128.Zero) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); + int finalIndex = sourceSpanUInt16.Length - Vector128.Count; + uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); + while (mask != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } } } return; From 9c9b01ebcfb5840400e69021d6dc9ff4a6d92f27 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 08:27:08 +1000 Subject: [PATCH 05/19] Fix (shouldn't affect benchmark run meaningfully) --- .../System.Private.CoreLib/src/System/String.Manipulation.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index af064c2f0b56bf..3250b1001e050f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2171,7 +2171,7 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r } } - while ((uint)remaining.Length >= (uint)Vector512.Count) + while ((uint)remaining.Length > (uint)Vector512.Count) { Vector512 vector = Vector512.Create(remaining); Vector512 cmp = Vector512.Equals(vector, v1).AsByte() | Vector512.Equals(vector, v2).AsByte() | Vector512.Equals(vector, v3).AsByte(); From 4647df734d1fac9455f3eef0e94bb01938d46cf7 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 08:36:44 +1000 Subject: [PATCH 06/19] Update String.Manipulation.cs --- .../System.Private.CoreLib/src/System/String.Manipulation.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 3250b1001e050f..ff0eaf3b2bebab 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2390,7 +2390,7 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r // Handle the last chunk in a vectorized way also. // We do a whole vector's worth again, but just mask out the bits we've already handled. - if (!Sse.IsSupported || remaining.Count > 0) + if (!Sse.IsSupported || remaining.Length > 0) { Vector128 vector = Vector128.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector128.Count)); Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); From d692d36f23ba0019d14629d0864e5d17c5fb1334 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 09:20:06 +1000 Subject: [PATCH 07/19] Test to see if using V64 on arm conditionally would improve perf further --- .../src/System/String.Manipulation.cs | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index ff0eaf3b2bebab..40b2417a5d9a46 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2392,17 +2392,36 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r // We do a whole vector's worth again, but just mask out the bits we've already handled. if (!Sse.IsSupported || remaining.Length > 0) { - Vector128 vector = Vector128.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector128.Count)); - Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); - if (cmp != Vector128.Zero) + if (Vector64.IsHardwareAccelerated && remaining.Length <= Vector64.Count) { - int finalIndex = sourceSpanUInt16.Length - Vector128.Count; - uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); - while (mask != 0) + Vector64 vector = Vector64.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector64.Count)); + Vector64 cmp = Vector64.Equals(vector, v1).AsByte() | Vector64.Equals(vector, v2).AsByte() | Vector64.Equals(vector, v3).AsByte(); + var cmpU64 = cmp.AsUInt64().GetElement(0); + if (cmpU64 != 0) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); + cmpU64 &= 0x0101010101010101; + while (cmpU64 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(cmpU64) / (sizeof(char) * 8); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } + } + } + else + { + Vector128 vector = Vector128.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector128.Count)); + Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); + if (cmp != Vector128.Zero) + { + int finalIndex = sourceSpanUInt16.Length - Vector128.Count; + uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); + while (mask != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } } } } From 39f111a6fa816e1560998a0a75077545cd38ec39 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 09:42:28 +1000 Subject: [PATCH 08/19] Fixes to V64 path --- .../src/System/String.Manipulation.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 40b2417a5d9a46..2bc49b9e612350 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2395,16 +2395,17 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r if (Vector64.IsHardwareAccelerated && remaining.Length <= Vector64.Count) { Vector64 vector = Vector64.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector64.Count)); - Vector64 cmp = Vector64.Equals(vector, v1).AsByte() | Vector64.Equals(vector, v2).AsByte() | Vector64.Equals(vector, v3).AsByte(); - var cmpU64 = cmp.AsUInt64().GetElement(0); + Vector64 cmp = Vector64.Equals(vector, v1.GetLower()).AsByte() | Vector64.Equals(vector, v2.GetLower()).AsByte() | Vector64.Equals(vector, v3.GetLower()).AsByte(); + ulong cmpU64 = cmp.AsUInt64().GetElement(0); if (cmpU64 != 0) { - cmpU64 &= 0x0101010101010101; + int finalIndex = sourceSpanUInt16.Length - Vector64.Count; + cmpU64 &= 0x0101010101010101UL & ~((1UL << (Vector64.Count - remaining.Length * sizeof(char) * 8)) - 1); while (cmpU64 != 0) { uint bitPos = (uint)BitOperations.TrailingZeroCount(cmpU64) / (sizeof(char) * 8); sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); + cmpU64 = BitOperations.ResetLowestSetBit(cmpU64); } } } From b855b40a1b8963893afd68cd44c8298c378446a9 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 10:02:19 +1000 Subject: [PATCH 09/19] Revert V64 logic --- .../src/System/String.Manipulation.cs | 38 +++++-------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 2bc49b9e612350..ff0eaf3b2bebab 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2392,37 +2392,17 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r // We do a whole vector's worth again, but just mask out the bits we've already handled. if (!Sse.IsSupported || remaining.Length > 0) { - if (Vector64.IsHardwareAccelerated && remaining.Length <= Vector64.Count) - { - Vector64 vector = Vector64.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector64.Count)); - Vector64 cmp = Vector64.Equals(vector, v1.GetLower()).AsByte() | Vector64.Equals(vector, v2.GetLower()).AsByte() | Vector64.Equals(vector, v3.GetLower()).AsByte(); - ulong cmpU64 = cmp.AsUInt64().GetElement(0); - if (cmpU64 != 0) - { - int finalIndex = sourceSpanUInt16.Length - Vector64.Count; - cmpU64 &= 0x0101010101010101UL & ~((1UL << (Vector64.Count - remaining.Length * sizeof(char) * 8)) - 1); - while (cmpU64 != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(cmpU64) / (sizeof(char) * 8); - sepListBuilder.Append(finalIndex + (int)bitPos); - cmpU64 = BitOperations.ResetLowestSetBit(cmpU64); - } - } - } - else + Vector128 vector = Vector128.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector128.Count)); + Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); + if (cmp != Vector128.Zero) { - Vector128 vector = Vector128.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector128.Count)); - Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); - if (cmp != Vector128.Zero) + int finalIndex = sourceSpanUInt16.Length - Vector128.Count; + uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); + while (mask != 0) { - int finalIndex = sourceSpanUInt16.Length - Vector128.Count; - uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); - while (mask != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); - } + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); } } } From 930d7f64bce40b555b1fc9d2cd7a211937b19b9b Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 10:13:01 +1000 Subject: [PATCH 10/19] Manually elide bounds check that made its way in when we converted to safe simd in #125379 --- .../System.Private.CoreLib/src/System/String.Manipulation.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index ff0eaf3b2bebab..984947ce8eb227 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2392,7 +2392,8 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r // We do a whole vector's worth again, but just mask out the bits we've already handled. if (!Sse.IsSupported || remaining.Length > 0) { - Vector128 vector = Vector128.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector128.Count)); + Debug.Assert(sourceSpanUInt16.Length >= Vector128.Count); + Vector128 vector = Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(sourceSpanUInt16), (uint)(sourceSpanUInt16.Length - Vector128.Count))); Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); if (cmp != Vector128.Zero) { From 7fd8a6e986eeaff6e7ed5e2544ff0eee8727bf54 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 10:27:44 +1000 Subject: [PATCH 11/19] Update String.Manipulation.cs --- .../System.Private.CoreLib/src/System/String.Manipulation.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 984947ce8eb227..9353aaca149132 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2393,7 +2393,7 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r if (!Sse.IsSupported || remaining.Length > 0) { Debug.Assert(sourceSpanUInt16.Length >= Vector128.Count); - Vector128 vector = Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(sourceSpanUInt16), (uint)(sourceSpanUInt16.Length - Vector128.Count))); + Vector128 vector = Vector128.LoadUnsafe(in MemoryMarshal.GetReference(sourceSpanUInt16), (uint)(sourceSpanUInt16.Length - Vector128.Count)); Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); if (cmp != Vector128.Zero) { From 358b43d0450929c05aa55ded614696a90ead2706 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 11:48:43 +1000 Subject: [PATCH 12/19] Add comments & test out alternative fixup sequence for arm64 --- .../src/System/String.Manipulation.cs | 50 +++++++++++++++++-- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 9353aaca149132..560ebae0a789b2 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2194,6 +2194,7 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r // Handle the last chunk in a vectorized way also. // We do a whole vector's worth again, but just mask out the bits we've already handled. + // Note: when Avx512BW.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. if (!Avx512BW.IsSupported || remaining.Length > 0) { Vector512 vector = Vector512.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector512.Count)); @@ -2292,6 +2293,7 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r // Handle the last chunk in a vectorized way also. // We do a whole vector's worth again, but just mask out the bits we've already handled. + // Note: when Avx2.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. if (!Avx2.IsSupported || remaining.Length > 0) { Vector256 vector = Vector256.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector256.Count)); @@ -2390,6 +2392,7 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r // Handle the last chunk in a vectorized way also. // We do a whole vector's worth again, but just mask out the bits we've already handled. + // Note: when Sse.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. if (!Sse.IsSupported || remaining.Length > 0) { Debug.Assert(sourceSpanUInt16.Length >= Vector128.Count); @@ -2397,13 +2400,50 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); if (cmp != Vector128.Zero) { + cmp &= Vector128.Create(0x0101010101010101UL).AsByte(); int finalIndex = sourceSpanUInt16.Length - Vector128.Count; - uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); - while (mask != 0) + if (X86Base.IsSupported) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); + // X86 has fast ExtractMostSignificantBits, so we just use that pattern here + uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); + while (mask != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } + } + else + { + // Whereas on arm64, this pattern can be faster + var mask = ~((1UL << (Vector128.Count - remaining.Length * sizeof(char) * 8)) - 1); + ulong cmp0, cmp1; + + if (remaining.Length > Vector128.Count) + { + cmp0 = cmp.GetElement(0) & 0x0101010101010101UL & mask; + + while (cmp0 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(cmp0) / (uint)(sizeof(char) * 8); + sepListBuilder.Append(finalIndex + (int)bitPos); + cmp0 = BitOperations.ResetLowestSetBit(cmp0); + } + + cmp1 = cmp.GetElement(1); + } + else + { + cmp0 = 0; + cmp1 = cmp.GetElement(1) & 0x0101010101010101UL & mask; + } + + while (cmp1 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(cmp1) / (uint)(sizeof(char) * 8); + sepListBuilder.Append(finalIndex + (int)bitPos); + cmp1 = BitOperations.ResetLowestSetBit(cmp1); + } } } } From 56a317d805cdc2e5dfe6c0c100e0834cf833e576 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 11:50:04 +1000 Subject: [PATCH 13/19] Update String.Manipulation.cs --- .../System.Private.CoreLib/src/System/String.Manipulation.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 560ebae0a789b2..605f6967e690cf 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2417,11 +2417,11 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r { // Whereas on arm64, this pattern can be faster var mask = ~((1UL << (Vector128.Count - remaining.Length * sizeof(char) * 8)) - 1); - ulong cmp0, cmp1; + ulong cmp1; if (remaining.Length > Vector128.Count) { - cmp0 = cmp.GetElement(0) & 0x0101010101010101UL & mask; + var cmp0 = cmp.GetElement(0) & 0x0101010101010101UL & mask; while (cmp0 != 0) { @@ -2434,7 +2434,6 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r } else { - cmp0 = 0; cmp1 = cmp.GetElement(1) & 0x0101010101010101UL & mask; } From 655680a7472f5117b73234ec372fed1cfaf43d4c Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 12:11:12 +1000 Subject: [PATCH 14/19] thanks github editor --- .../src/System/String.Manipulation.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 605f6967e690cf..15e1539812ce29 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2418,25 +2418,25 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r // Whereas on arm64, this pattern can be faster var mask = ~((1UL << (Vector128.Count - remaining.Length * sizeof(char) * 8)) - 1); ulong cmp1; - + if (remaining.Length > Vector128.Count) { var cmp0 = cmp.GetElement(0) & 0x0101010101010101UL & mask; - + while (cmp0 != 0) { uint bitPos = (uint)BitOperations.TrailingZeroCount(cmp0) / (uint)(sizeof(char) * 8); sepListBuilder.Append(finalIndex + (int)bitPos); cmp0 = BitOperations.ResetLowestSetBit(cmp0); } - + cmp1 = cmp.GetElement(1); } else { cmp1 = cmp.GetElement(1) & 0x0101010101010101UL & mask; } - + while (cmp1 != 0) { uint bitPos = (uint)BitOperations.TrailingZeroCount(cmp1) / (uint)(sizeof(char) * 8); From c3f656ffce78b5448710414c3513525e2b561e1f Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 12:48:31 +1000 Subject: [PATCH 15/19] Revert alternative enumeration test --- .../src/System/String.Manipulation.cs | 45 +++---------------- 1 file changed, 5 insertions(+), 40 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 15e1539812ce29..69068d80fc614c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2402,47 +2402,12 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r { cmp &= Vector128.Create(0x0101010101010101UL).AsByte(); int finalIndex = sourceSpanUInt16.Length - Vector128.Count; - if (X86Base.IsSupported) - { - // X86 has fast ExtractMostSignificantBits, so we just use that pattern here - uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); - while (mask != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); - } - } - else + uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); + while (mask != 0) { - // Whereas on arm64, this pattern can be faster - var mask = ~((1UL << (Vector128.Count - remaining.Length * sizeof(char) * 8)) - 1); - ulong cmp1; - - if (remaining.Length > Vector128.Count) - { - var cmp0 = cmp.GetElement(0) & 0x0101010101010101UL & mask; - - while (cmp0 != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(cmp0) / (uint)(sizeof(char) * 8); - sepListBuilder.Append(finalIndex + (int)bitPos); - cmp0 = BitOperations.ResetLowestSetBit(cmp0); - } - - cmp1 = cmp.GetElement(1); - } - else - { - cmp1 = cmp.GetElement(1) & 0x0101010101010101UL & mask; - } - - while (cmp1 != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(cmp1) / (uint)(sizeof(char) * 8); - sepListBuilder.Append(finalIndex + (int)bitPos); - cmp1 = BitOperations.ResetLowestSetBit(cmp1); - } + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); } } } From 1c287539f21a09f364b4f51937f2ef90d7c3f2f3 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 17:16:34 +1000 Subject: [PATCH 16/19] Remove holder from previous experiment --- .../System.Private.CoreLib/src/System/String.Manipulation.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 69068d80fc614c..d8e10b8b5074ea 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2400,7 +2400,6 @@ private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, r Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); if (cmp != Vector128.Zero) { - cmp &= Vector128.Create(0x0101010101010101UL).AsByte(); int finalIndex = sourceSpanUInt16.Length - Vector128.Count; uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); while (mask != 0) From fe3175981e5b68c277fb583608294138bde6c4e8 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 23:37:08 +1000 Subject: [PATCH 17/19] attempt to appease egorbo by removing unsafe by moving scalar code into the method --- .../src/System/String.Manipulation.cs | 549 +++++++++--------- 1 file changed, 272 insertions(+), 277 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index d8e10b8b5074ea..74268d1862e593 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2068,20 +2068,7 @@ internal static void MakeSeparatorListAny(ReadOnlySpan source, ReadOnlySpa sep0 = separators[0]; sep1 = separators.Length > 1 ? separators[1] : sep0; sep2 = separators.Length > 2 ? separators[2] : sep1; - if (Vector128.IsHardwareAccelerated && source.Length >= Vector128.Count * 2) - { - MakeSeparatorListVectorized(source, ref sepListBuilder, sep0, sep1, sep2); - return; - } - - for (int i = 0; i < source.Length; i++) - { - char c = source[i]; - if (c == sep0 || c == sep1 || c == sep2) - { - sepListBuilder.Append(i); - } - } + MakeSeparatorListFewChars(source, ref sepListBuilder, sep0, sep1, sep2); } // Handle > 3 separators with a probabilistic map, ala IndexOfAny. @@ -2101,319 +2088,327 @@ internal static void MakeSeparatorListAny(ReadOnlySpan source, ReadOnlySpa } } - private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, ref ValueListBuilder sepListBuilder, char c, char c2, char c3) + private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref ValueListBuilder sepListBuilder, char c, char c2, char c3) { - // Redundant test so we won't prejit remainder of this method - // on platforms where it is not supported - if (!Vector128.IsHardwareAccelerated) - { - throw new PlatformNotSupportedException(); - } - Debug.Assert(sourceSpan.Length >= Vector128.Count*2); - int baseIndex = 0; - ReadOnlySpan sourceSpanUInt16 = MemoryMarshal.Cast(sourceSpan); - ReadOnlySpan remaining = sourceSpanUInt16; - - if (Vector512.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector512.Count*2) + // Attempt to run vectorized version + if (Vector128.IsHardwareAccelerated && sourceSpan.Length >= Vector128.Count*2) { - Vector512 v1 = Vector512.Create((ushort)c); - Vector512 v2 = Vector512.Create((ushort)c2); - Vector512 v3 = Vector512.Create((ushort)c3); - - if (Avx512BW.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) + Debug.Assert(sourceSpan.Length >= Vector128.Count*2); + int baseIndex = 0; + ReadOnlySpan sourceSpanUInt16 = MemoryMarshal.Cast(sourceSpan); + ReadOnlySpan remaining = sourceSpanUInt16; + + if (Vector512.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector512.Count*2) { - // Process in double chunks & check if either chunk is likely to contain matches at once. - // If we get multiple matches in a single chunk, we assume they're likely to be close & - // break out of this logic & use the more optimistic loop below. - // This is similar logic to SpanHelpers.Packed.cs's IndexOf. - Vector512 packedComparand1 = Vector512.Create((byte)c); - Vector512 packedComparand2 = Vector512.Create((byte)c2); - Vector512 packedComparand3 = Vector512.Create((byte)c3); - while ((uint)remaining.Length >= (uint)Vector512.Count*2) + Vector512 v1 = Vector512.Create((ushort)c); + Vector512 v2 = Vector512.Create((ushort)c2); + Vector512 v3 = Vector512.Create((ushort)c3); + + if (Avx512BW.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) { - Vector512 vector1 = Vector512.Create(remaining); - Vector512 vector2 = Vector512.Create(remaining.Slice(Vector512.Count)); - var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); - - if ((Vector512.Equals(packed, packedComparand1) | Vector512.Equals(packed, packedComparand2) | Vector512.Equals(packed, packedComparand3)) != Vector512.Zero) + // Process in double chunks & check if either chunk is likely to contain matches at once. + // If we get multiple matches in a single chunk, we assume they're likely to be close & + // break out of this logic & use the more optimistic loop below. + // This is similar logic to SpanHelpers.Packed.cs's IndexOf. + Vector512 packedComparand1 = Vector512.Create((byte)c); + Vector512 packedComparand2 = Vector512.Create((byte)c2); + Vector512 packedComparand3 = Vector512.Create((byte)c3); + while ((uint)remaining.Length >= (uint)Vector512.Count*2) { - var cmp1 = Vector512.Equals(vector1, v1).AsByte() | Vector512.Equals(vector1, v2).AsByte() | Vector512.Equals(vector1, v3).AsByte(); - var cmp2 = Vector512.Equals(vector2, v1).AsByte() | Vector512.Equals(vector2, v2).AsByte() | Vector512.Equals(vector2, v3).AsByte(); - - // Same logic as below, but for both vectors. - ulong mask1 = cmp1.ExtractMostSignificantBits() & 0x5555555555555555; - ulong mask2 = cmp2.ExtractMostSignificantBits() & 0x5555555555555555; - bool shouldBreak = ulong.PopCount(mask1) + ulong.PopCount(mask2) > 1; - while (mask1 != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask1) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos); - mask1 = BitOperations.ResetLowestSetBit(mask1); - } - while (mask2 != 0) + Vector512 vector1 = Vector512.Create(remaining); + Vector512 vector2 = Vector512.Create(remaining.Slice(Vector512.Count)); + var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); + + if ((Vector512.Equals(packed, packedComparand1) | Vector512.Equals(packed, packedComparand2) | Vector512.Equals(packed, packedComparand3)) != Vector512.Zero) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask2) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos + Vector512.Count); - mask2 = BitOperations.ResetLowestSetBit(mask2); - } - - // Break out of the loop if we had >1 match: - if (shouldBreak) - { - baseIndex += Vector512.Count*2; - remaining = remaining.Slice(Vector512.Count*2); - break; + var cmp1 = Vector512.Equals(vector1, v1).AsByte() | Vector512.Equals(vector1, v2).AsByte() | Vector512.Equals(vector1, v3).AsByte(); + var cmp2 = Vector512.Equals(vector2, v1).AsByte() | Vector512.Equals(vector2, v2).AsByte() | Vector512.Equals(vector2, v3).AsByte(); + + // Same logic as below, but for both vectors. + ulong mask1 = cmp1.ExtractMostSignificantBits() & 0x5555555555555555; + ulong mask2 = cmp2.ExtractMostSignificantBits() & 0x5555555555555555; + bool shouldBreak = ulong.PopCount(mask1) + ulong.PopCount(mask2) > 1; + while (mask1 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask1) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos); + mask1 = BitOperations.ResetLowestSetBit(mask1); + } + while (mask2 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask2) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos + Vector512.Count); + mask2 = BitOperations.ResetLowestSetBit(mask2); + } + + // Break out of the loop if we had >1 match: + if (shouldBreak) + { + baseIndex += Vector512.Count*2; + remaining = remaining.Slice(Vector512.Count*2); + break; + } } + + baseIndex += Vector512.Count*2; + remaining = remaining.Slice(Vector512.Count*2); } - - baseIndex += Vector512.Count*2; - remaining = remaining.Slice(Vector512.Count*2); } - } - - while ((uint)remaining.Length > (uint)Vector512.Count) - { - Vector512 vector = Vector512.Create(remaining); - Vector512 cmp = Vector512.Equals(vector, v1).AsByte() | Vector512.Equals(vector, v2).AsByte() | Vector512.Equals(vector, v3).AsByte(); - - if (cmp != Vector512.Zero) + + while ((uint)remaining.Length > (uint)Vector512.Count) { - // Skip every other bit - ulong mask = cmp.ExtractMostSignificantBits() & 0x5555555555555555; - do + Vector512 vector = Vector512.Create(remaining); + Vector512 cmp = Vector512.Equals(vector, v1).AsByte() | Vector512.Equals(vector, v2).AsByte() | Vector512.Equals(vector, v3).AsByte(); + + if (cmp != Vector512.Zero) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); - } while (mask != 0); + // Skip every other bit + ulong mask = cmp.ExtractMostSignificantBits() & 0x5555555555555555; + do + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } while (mask != 0); + } + + baseIndex += Vector512.Count; + remaining = remaining.Slice(Vector512.Count); } - - baseIndex += Vector512.Count; - remaining = remaining.Slice(Vector512.Count); - } - - // Handle the last chunk in a vectorized way also. - // We do a whole vector's worth again, but just mask out the bits we've already handled. - // Note: when Avx512BW.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. - if (!Avx512BW.IsSupported || remaining.Length > 0) - { - Vector512 vector = Vector512.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector512.Count)); - Vector512 cmp = Vector512.Equals(vector, v1).AsByte() | Vector512.Equals(vector, v2).AsByte() | Vector512.Equals(vector, v3).AsByte(); - if (cmp != Vector512.Zero) + + // Handle the last chunk in a vectorized way also. + // We do a whole vector's worth again, but just mask out the bits we've already handled. + // Note: when Avx512BW.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. + if (!Avx512BW.IsSupported || remaining.Length > 0) { - int finalIndex = sourceSpanUInt16.Length - Vector512.Count; - ulong mask = cmp.ExtractMostSignificantBits() & 0x5555555555555555 & ~((1UL << (Vector512.Count - remaining.Length * sizeof(char))) - 1); - while (mask != 0) + Vector512 vector = Vector512.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector512.Count)); + Vector512 cmp = Vector512.Equals(vector, v1).AsByte() | Vector512.Equals(vector, v2).AsByte() | Vector512.Equals(vector, v3).AsByte(); + if (cmp != Vector512.Zero) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); + int finalIndex = sourceSpanUInt16.Length - Vector512.Count; + ulong mask = cmp.ExtractMostSignificantBits() & 0x5555555555555555 & ~((1UL << (Vector512.Count - remaining.Length * sizeof(char))) - 1); + while (mask != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } } } + return; } - return; - } - else if (Vector256.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector256.Count*2) - { - Vector256 v1 = Vector256.Create((ushort)c); - Vector256 v2 = Vector256.Create((ushort)c2); - Vector256 v3 = Vector256.Create((ushort)c3); - - if (Avx2.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) + else if (Vector256.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector256.Count*2) { - // Process in double chunks & check if either chunk is likely to contain matches at once. - // If we get multiple matches in a single chunk, we assume they're likely to be close & - // break out of this logic & use the more optimistic loop below. - // This is similar logic to SpanHelpers.Packed.cs's IndexOf. - Vector256 packedComparand1 = Vector256.Create((byte)c); - Vector256 packedComparand2 = Vector256.Create((byte)c2); - Vector256 packedComparand3 = Vector256.Create((byte)c3); - while ((uint)remaining.Length >= (uint)Vector256.Count*2) + Vector256 v1 = Vector256.Create((ushort)c); + Vector256 v2 = Vector256.Create((ushort)c2); + Vector256 v3 = Vector256.Create((ushort)c3); + + if (Avx2.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) { - Vector256 vector1 = Vector256.Create(remaining); - Vector256 vector2 = Vector256.Create(remaining.Slice(Vector256.Count)); - var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); - - if ((Vector256.Equals(packed, packedComparand1) | Vector256.Equals(packed, packedComparand2) | Vector256.Equals(packed, packedComparand3)) != Vector256.Zero) + // Process in double chunks & check if either chunk is likely to contain matches at once. + // If we get multiple matches in a single chunk, we assume they're likely to be close & + // break out of this logic & use the more optimistic loop below. + // This is similar logic to SpanHelpers.Packed.cs's IndexOf. + Vector256 packedComparand1 = Vector256.Create((byte)c); + Vector256 packedComparand2 = Vector256.Create((byte)c2); + Vector256 packedComparand3 = Vector256.Create((byte)c3); + while ((uint)remaining.Length >= (uint)Vector256.Count*2) { - var cmp1 = Vector256.Equals(vector1, v1).AsByte() | Vector256.Equals(vector1, v2).AsByte() | Vector256.Equals(vector1, v3).AsByte(); - var cmp2 = Vector256.Equals(vector2, v1).AsByte() | Vector256.Equals(vector2, v2).AsByte() | Vector256.Equals(vector2, v3).AsByte(); - - // Same logic as below, but for both vectors. - uint mask1 = cmp1.ExtractMostSignificantBits() & 0x55555555; - uint mask2 = cmp2.ExtractMostSignificantBits() & 0x55555555; - bool shouldBreak = uint.PopCount(mask1) + uint.PopCount(mask2) > 1; - while (mask1 != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask1) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos); - mask1 = BitOperations.ResetLowestSetBit(mask1); - } - while (mask2 != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask2) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos + Vector256.Count); - mask2 = BitOperations.ResetLowestSetBit(mask2); - } - - // Break out of the loop if we had > 1 match: - if (shouldBreak) + Vector256 vector1 = Vector256.Create(remaining); + Vector256 vector2 = Vector256.Create(remaining.Slice(Vector256.Count)); + var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); + + if ((Vector256.Equals(packed, packedComparand1) | Vector256.Equals(packed, packedComparand2) | Vector256.Equals(packed, packedComparand3)) != Vector256.Zero) { - baseIndex += Vector256.Count*2; - remaining = remaining.Slice(Vector256.Count*2); - break; + var cmp1 = Vector256.Equals(vector1, v1).AsByte() | Vector256.Equals(vector1, v2).AsByte() | Vector256.Equals(vector1, v3).AsByte(); + var cmp2 = Vector256.Equals(vector2, v1).AsByte() | Vector256.Equals(vector2, v2).AsByte() | Vector256.Equals(vector2, v3).AsByte(); + + // Same logic as below, but for both vectors. + uint mask1 = cmp1.ExtractMostSignificantBits() & 0x55555555; + uint mask2 = cmp2.ExtractMostSignificantBits() & 0x55555555; + bool shouldBreak = uint.PopCount(mask1) + uint.PopCount(mask2) > 1; + while (mask1 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask1) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos); + mask1 = BitOperations.ResetLowestSetBit(mask1); + } + while (mask2 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask2) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos + Vector256.Count); + mask2 = BitOperations.ResetLowestSetBit(mask2); + } + + // Break out of the loop if we had > 1 match: + if (shouldBreak) + { + baseIndex += Vector256.Count*2; + remaining = remaining.Slice(Vector256.Count*2); + break; + } } + + baseIndex += Vector256.Count*2; + remaining = remaining.Slice(Vector256.Count*2); } - - baseIndex += Vector256.Count*2; - remaining = remaining.Slice(Vector256.Count*2); } - } - - while ((uint)remaining.Length > (uint)Vector256.Count) - { - Vector256 vector = Vector256.Create(remaining); - Vector256 cmp = Vector256.Equals(vector, v1).AsByte() | Vector256.Equals(vector, v2).AsByte() | Vector256.Equals(vector, v3).AsByte(); - - if (cmp != Vector256.Zero) + + while ((uint)remaining.Length > (uint)Vector256.Count) { - // Skip every other bit - uint mask = cmp.ExtractMostSignificantBits() & 0x55555555; - do + Vector256 vector = Vector256.Create(remaining); + Vector256 cmp = Vector256.Equals(vector, v1).AsByte() | Vector256.Equals(vector, v2).AsByte() | Vector256.Equals(vector, v3).AsByte(); + + if (cmp != Vector256.Zero) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); - } while (mask != 0); + // Skip every other bit + uint mask = cmp.ExtractMostSignificantBits() & 0x55555555; + do + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } while (mask != 0); + } + + baseIndex += Vector256.Count; + remaining = remaining.Slice(Vector256.Count); } - - baseIndex += Vector256.Count; - remaining = remaining.Slice(Vector256.Count); - } - - // Handle the last chunk in a vectorized way also. - // We do a whole vector's worth again, but just mask out the bits we've already handled. - // Note: when Avx2.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. - if (!Avx2.IsSupported || remaining.Length > 0) - { - Vector256 vector = Vector256.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector256.Count)); - Vector256 cmp = Vector256.Equals(vector, v1).AsByte() | Vector256.Equals(vector, v2).AsByte() | Vector256.Equals(vector, v3).AsByte(); - if (cmp != Vector256.Zero) + + // Handle the last chunk in a vectorized way also. + // We do a whole vector's worth again, but just mask out the bits we've already handled. + // Note: when Avx2.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. + if (!Avx2.IsSupported || remaining.Length > 0) { - int finalIndex = sourceSpanUInt16.Length - Vector256.Count; - uint mask = cmp.ExtractMostSignificantBits() & 0x55555555 & ~((1u << (Vector256.Count - remaining.Length * sizeof(char))) - 1); - while (mask != 0) + Vector256 vector = Vector256.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector256.Count)); + Vector256 cmp = Vector256.Equals(vector, v1).AsByte() | Vector256.Equals(vector, v2).AsByte() | Vector256.Equals(vector, v3).AsByte(); + if (cmp != Vector256.Zero) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); + int finalIndex = sourceSpanUInt16.Length - Vector256.Count; + uint mask = cmp.ExtractMostSignificantBits() & 0x55555555 & ~((1u << (Vector256.Count - remaining.Length * sizeof(char))) - 1); + while (mask != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } } } + return; } - return; - } - else if (Vector128.IsHardwareAccelerated) - { - Vector128 v1 = Vector128.Create((ushort)c); - Vector128 v2 = Vector128.Create((ushort)c2); - Vector128 v3 = Vector128.Create((ushort)c3); - - if (Sse2.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) + else if (Vector128.IsHardwareAccelerated) { - // Process in double chunks & check if either chunk is likely to contain matches at once. - // If we get multiple matches in a single chunk, we assume they're likely to be close & - // break out of this logic & use the more optimistic loop below. - // This is similar logic to SpanHelpers.Packed.cs's IndexOf. - Vector128 packedComparand1 = Vector128.Create((byte)c); - Vector128 packedComparand2 = Vector128.Create((byte)c2); - Vector128 packedComparand3 = Vector128.Create((byte)c3); - while ((uint)remaining.Length >= (uint)Vector128.Count*2) + Vector128 v1 = Vector128.Create((ushort)c); + Vector128 v2 = Vector128.Create((ushort)c2); + Vector128 v3 = Vector128.Create((ushort)c3); + + if (Sse2.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) { - Vector128 vector1 = Vector128.Create(remaining); - Vector128 vector2 = Vector128.Create(remaining.Slice(Vector128.Count)); - var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); - - if ((Vector128.Equals(packed, packedComparand1) | Vector128.Equals(packed, packedComparand2) | Vector128.Equals(packed, packedComparand3)) != Vector128.Zero) + // Process in double chunks & check if either chunk is likely to contain matches at once. + // If we get multiple matches in a single chunk, we assume they're likely to be close & + // break out of this logic & use the more optimistic loop below. + // This is similar logic to SpanHelpers.Packed.cs's IndexOf. + Vector128 packedComparand1 = Vector128.Create((byte)c); + Vector128 packedComparand2 = Vector128.Create((byte)c2); + Vector128 packedComparand3 = Vector128.Create((byte)c3); + while ((uint)remaining.Length >= (uint)Vector128.Count*2) { - var cmp1 = Vector128.Equals(vector1, v1).AsByte() | Vector128.Equals(vector1, v2).AsByte() | Vector128.Equals(vector1, v3).AsByte(); - var cmp2 = Vector128.Equals(vector2, v1).AsByte() | Vector128.Equals(vector2, v2).AsByte() | Vector128.Equals(vector2, v3).AsByte(); - - // Same logic as below, but for both vectors. - uint mask1 = cmp1.ExtractMostSignificantBits() & 0x5555; - uint mask2 = cmp2.ExtractMostSignificantBits() & 0x5555; - bool shouldBreak = uint.PopCount(mask1) + uint.PopCount(mask2) > 1; - while (mask1 != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask1) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos); - mask1 = BitOperations.ResetLowestSetBit(mask1); - } - while (mask2 != 0) + Vector128 vector1 = Vector128.Create(remaining); + Vector128 vector2 = Vector128.Create(remaining.Slice(Vector128.Count)); + var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); + + if ((Vector128.Equals(packed, packedComparand1) | Vector128.Equals(packed, packedComparand2) | Vector128.Equals(packed, packedComparand3)) != Vector128.Zero) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask2) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos + Vector128.Count); - mask2 = BitOperations.ResetLowestSetBit(mask2); + var cmp1 = Vector128.Equals(vector1, v1).AsByte() | Vector128.Equals(vector1, v2).AsByte() | Vector128.Equals(vector1, v3).AsByte(); + var cmp2 = Vector128.Equals(vector2, v1).AsByte() | Vector128.Equals(vector2, v2).AsByte() | Vector128.Equals(vector2, v3).AsByte(); + + // Same logic as below, but for both vectors. + uint mask1 = cmp1.ExtractMostSignificantBits() & 0x5555; + uint mask2 = cmp2.ExtractMostSignificantBits() & 0x5555; + bool shouldBreak = uint.PopCount(mask1) + uint.PopCount(mask2) > 1; + while (mask1 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask1) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos); + mask1 = BitOperations.ResetLowestSetBit(mask1); + } + while (mask2 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask2) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos + Vector128.Count); + mask2 = BitOperations.ResetLowestSetBit(mask2); + } + + // Break out of the loop if we had > 1 match: + if (shouldBreak) + { + baseIndex += Vector128.Count*2; + remaining = remaining.Slice(Vector128.Count*2); + break; + } } - - // Break out of the loop if we had > 1 match: - if (shouldBreak) + + baseIndex += Vector128.Count*2; + remaining = remaining.Slice(Vector128.Count*2); + } + } + + while ((uint)remaining.Length > (uint)Vector128.Count) + { + Vector128 vector = Vector128.Create(remaining); + Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); + + if (cmp != Vector128.Zero) + { + // Skip every other bit + uint mask = cmp.ExtractMostSignificantBits() & 0x5555; + do { - baseIndex += Vector128.Count*2; - remaining = remaining.Slice(Vector128.Count*2); - break; - } + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } while (mask != 0); } - - baseIndex += Vector128.Count*2; - remaining = remaining.Slice(Vector128.Count*2); + + baseIndex += Vector128.Count; + remaining = remaining.Slice(Vector128.Count); } - } - - while ((uint)remaining.Length > (uint)Vector128.Count) - { - Vector128 vector = Vector128.Create(remaining); - Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); - - if (cmp != Vector128.Zero) + + // Handle the last chunk in a vectorized way also. + // We do a whole vector's worth again, but just mask out the bits we've already handled. + // Note: when Sse.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. + if (!Sse.IsSupported || remaining.Length > 0) { - // Skip every other bit - uint mask = cmp.ExtractMostSignificantBits() & 0x5555; - do + Vector128 vector = Vector128.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector128.Count)); + Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); + if (cmp != Vector128.Zero) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); - } while (mask != 0); + int finalIndex = sourceSpanUInt16.Length - Vector128.Count; + uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); + while (mask != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } + } } - - baseIndex += Vector128.Count; - remaining = remaining.Slice(Vector128.Count); + return; } - - // Handle the last chunk in a vectorized way also. - // We do a whole vector's worth again, but just mask out the bits we've already handled. - // Note: when Sse.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. - if (!Sse.IsSupported || remaining.Length > 0) + } + else + { + for (int i = 0; i < source.Length; i++) { - Debug.Assert(sourceSpanUInt16.Length >= Vector128.Count); - Vector128 vector = Vector128.LoadUnsafe(in MemoryMarshal.GetReference(sourceSpanUInt16), (uint)(sourceSpanUInt16.Length - Vector128.Count)); - Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); - if (cmp != Vector128.Zero) + char c = source[i]; + if (c == sep0 || c == sep1 || c == sep2) { - int finalIndex = sourceSpanUInt16.Length - Vector128.Count; - uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); - while (mask != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); - } + sepListBuilder.Append(i); } } + return; } - - Debug.Fail("We should not be able to reach this point of MakeSeparatorListVectorized."); } /// From b0a1d81a659886720196c764bad0a1a31e09d17a Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Fri, 10 Jul 2026 23:39:36 +1000 Subject: [PATCH 18/19] thanks github desktop again --- .../src/System/String.Manipulation.cs | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 74268d1862e593..887207eabf25b7 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2097,13 +2097,13 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref int baseIndex = 0; ReadOnlySpan sourceSpanUInt16 = MemoryMarshal.Cast(sourceSpan); ReadOnlySpan remaining = sourceSpanUInt16; - + if (Vector512.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector512.Count*2) { Vector512 v1 = Vector512.Create((ushort)c); Vector512 v2 = Vector512.Create((ushort)c2); Vector512 v3 = Vector512.Create((ushort)c3); - + if (Avx512BW.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) { // Process in double chunks & check if either chunk is likely to contain matches at once. @@ -2118,12 +2118,12 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref Vector512 vector1 = Vector512.Create(remaining); Vector512 vector2 = Vector512.Create(remaining.Slice(Vector512.Count)); var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); - + if ((Vector512.Equals(packed, packedComparand1) | Vector512.Equals(packed, packedComparand2) | Vector512.Equals(packed, packedComparand3)) != Vector512.Zero) { var cmp1 = Vector512.Equals(vector1, v1).AsByte() | Vector512.Equals(vector1, v2).AsByte() | Vector512.Equals(vector1, v3).AsByte(); var cmp2 = Vector512.Equals(vector2, v1).AsByte() | Vector512.Equals(vector2, v2).AsByte() | Vector512.Equals(vector2, v3).AsByte(); - + // Same logic as below, but for both vectors. ulong mask1 = cmp1.ExtractMostSignificantBits() & 0x5555555555555555; ulong mask2 = cmp2.ExtractMostSignificantBits() & 0x5555555555555555; @@ -2140,7 +2140,7 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref sepListBuilder.Append(baseIndex + (int)bitPos + Vector512.Count); mask2 = BitOperations.ResetLowestSetBit(mask2); } - + // Break out of the loop if we had >1 match: if (shouldBreak) { @@ -2149,17 +2149,17 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref break; } } - + baseIndex += Vector512.Count*2; remaining = remaining.Slice(Vector512.Count*2); } } - + while ((uint)remaining.Length > (uint)Vector512.Count) { Vector512 vector = Vector512.Create(remaining); Vector512 cmp = Vector512.Equals(vector, v1).AsByte() | Vector512.Equals(vector, v2).AsByte() | Vector512.Equals(vector, v3).AsByte(); - + if (cmp != Vector512.Zero) { // Skip every other bit @@ -2171,11 +2171,11 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref mask = BitOperations.ResetLowestSetBit(mask); } while (mask != 0); } - + baseIndex += Vector512.Count; remaining = remaining.Slice(Vector512.Count); } - + // Handle the last chunk in a vectorized way also. // We do a whole vector's worth again, but just mask out the bits we've already handled. // Note: when Avx512BW.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. @@ -2202,7 +2202,7 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref Vector256 v1 = Vector256.Create((ushort)c); Vector256 v2 = Vector256.Create((ushort)c2); Vector256 v3 = Vector256.Create((ushort)c3); - + if (Avx2.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) { // Process in double chunks & check if either chunk is likely to contain matches at once. @@ -2217,12 +2217,12 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref Vector256 vector1 = Vector256.Create(remaining); Vector256 vector2 = Vector256.Create(remaining.Slice(Vector256.Count)); var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); - + if ((Vector256.Equals(packed, packedComparand1) | Vector256.Equals(packed, packedComparand2) | Vector256.Equals(packed, packedComparand3)) != Vector256.Zero) { var cmp1 = Vector256.Equals(vector1, v1).AsByte() | Vector256.Equals(vector1, v2).AsByte() | Vector256.Equals(vector1, v3).AsByte(); var cmp2 = Vector256.Equals(vector2, v1).AsByte() | Vector256.Equals(vector2, v2).AsByte() | Vector256.Equals(vector2, v3).AsByte(); - + // Same logic as below, but for both vectors. uint mask1 = cmp1.ExtractMostSignificantBits() & 0x55555555; uint mask2 = cmp2.ExtractMostSignificantBits() & 0x55555555; @@ -2239,7 +2239,7 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref sepListBuilder.Append(baseIndex + (int)bitPos + Vector256.Count); mask2 = BitOperations.ResetLowestSetBit(mask2); } - + // Break out of the loop if we had > 1 match: if (shouldBreak) { @@ -2248,17 +2248,17 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref break; } } - + baseIndex += Vector256.Count*2; remaining = remaining.Slice(Vector256.Count*2); } } - + while ((uint)remaining.Length > (uint)Vector256.Count) { Vector256 vector = Vector256.Create(remaining); Vector256 cmp = Vector256.Equals(vector, v1).AsByte() | Vector256.Equals(vector, v2).AsByte() | Vector256.Equals(vector, v3).AsByte(); - + if (cmp != Vector256.Zero) { // Skip every other bit @@ -2270,11 +2270,11 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref mask = BitOperations.ResetLowestSetBit(mask); } while (mask != 0); } - + baseIndex += Vector256.Count; remaining = remaining.Slice(Vector256.Count); } - + // Handle the last chunk in a vectorized way also. // We do a whole vector's worth again, but just mask out the bits we've already handled. // Note: when Avx2.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. @@ -2301,7 +2301,7 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref Vector128 v1 = Vector128.Create((ushort)c); Vector128 v2 = Vector128.Create((ushort)c2); Vector128 v3 = Vector128.Create((ushort)c3); - + if (Sse2.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) { // Process in double chunks & check if either chunk is likely to contain matches at once. @@ -2316,12 +2316,12 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref Vector128 vector1 = Vector128.Create(remaining); Vector128 vector2 = Vector128.Create(remaining.Slice(Vector128.Count)); var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); - + if ((Vector128.Equals(packed, packedComparand1) | Vector128.Equals(packed, packedComparand2) | Vector128.Equals(packed, packedComparand3)) != Vector128.Zero) { var cmp1 = Vector128.Equals(vector1, v1).AsByte() | Vector128.Equals(vector1, v2).AsByte() | Vector128.Equals(vector1, v3).AsByte(); var cmp2 = Vector128.Equals(vector2, v1).AsByte() | Vector128.Equals(vector2, v2).AsByte() | Vector128.Equals(vector2, v3).AsByte(); - + // Same logic as below, but for both vectors. uint mask1 = cmp1.ExtractMostSignificantBits() & 0x5555; uint mask2 = cmp2.ExtractMostSignificantBits() & 0x5555; @@ -2338,7 +2338,7 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref sepListBuilder.Append(baseIndex + (int)bitPos + Vector128.Count); mask2 = BitOperations.ResetLowestSetBit(mask2); } - + // Break out of the loop if we had > 1 match: if (shouldBreak) { @@ -2347,17 +2347,17 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref break; } } - + baseIndex += Vector128.Count*2; remaining = remaining.Slice(Vector128.Count*2); } } - + while ((uint)remaining.Length > (uint)Vector128.Count) { Vector128 vector = Vector128.Create(remaining); Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); - + if (cmp != Vector128.Zero) { // Skip every other bit @@ -2369,11 +2369,11 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref mask = BitOperations.ResetLowestSetBit(mask); } while (mask != 0); } - + baseIndex += Vector128.Count; remaining = remaining.Slice(Vector128.Count); } - + // Handle the last chunk in a vectorized way also. // We do a whole vector's worth again, but just mask out the bits we've already handled. // Note: when Sse.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. From c5f242f9ddbbf9265424a8e7f7b330d068e58e14 Mon Sep 17 00:00:00 2001 From: Hamish Arblaster Date: Sat, 11 Jul 2026 00:15:54 +1000 Subject: [PATCH 19/19] fix build --- .../src/System/String.Manipulation.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 887207eabf25b7..e8c3f7e21998d7 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2398,10 +2398,10 @@ private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref } else { - for (int i = 0; i < source.Length; i++) + for (int i = 0; i < sourceSpan.Length; i++) { - char c = source[i]; - if (c == sep0 || c == sep1 || c == sep2) + char v = sourceSpan[i]; + if (v == c || v == c2 || v == c3) { sepListBuilder.Append(i); }