diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index e19298acef92f2..e8c3f7e21998d7 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2068,20 +2068,7 @@ internal static void MakeSeparatorListAny(ReadOnlySpan source, ReadOnlySpa sep0 = separators[0]; sep1 = separators.Length > 1 ? separators[1] : sep0; sep2 = separators.Length > 2 ? separators[2] : sep1; - if (Vector128.IsHardwareAccelerated && source.Length >= Vector128.Count * 2) - { - MakeSeparatorListVectorized(source, ref sepListBuilder, sep0, sep1, sep2); - return; - } - - for (int i = 0; i < source.Length; i++) - { - char c = source[i]; - if (c == sep0 || c == sep1 || c == sep2) - { - sepListBuilder.Append(i); - } - } + MakeSeparatorListFewChars(source, ref sepListBuilder, sep0, sep1, sep2); } // Handle > 3 separators with a probabilistic map, ala IndexOfAny. @@ -2101,306 +2088,327 @@ internal static void MakeSeparatorListAny(ReadOnlySpan source, ReadOnlySpa } } - private static void MakeSeparatorListVectorized(ReadOnlySpan sourceSpan, ref ValueListBuilder sepListBuilder, char c, char c2, char c3) + private static void MakeSeparatorListFewChars(ReadOnlySpan sourceSpan, ref ValueListBuilder sepListBuilder, char c, char c2, char c3) { - // Redundant test so we won't prejit remainder of this method - // on platforms where it is not supported - if (!Vector128.IsHardwareAccelerated) + // Attempt to run vectorized version + if (Vector128.IsHardwareAccelerated && sourceSpan.Length >= Vector128.Count*2) { - throw new PlatformNotSupportedException(); - } - Debug.Assert(sourceSpan.Length >= Vector128.Count*2); - int baseIndex = 0; - ReadOnlySpan sourceSpanUInt16 = MemoryMarshal.Cast(sourceSpan); - ReadOnlySpan remaining = sourceSpanUInt16; + Debug.Assert(sourceSpan.Length >= Vector128.Count*2); + int baseIndex = 0; + ReadOnlySpan sourceSpanUInt16 = MemoryMarshal.Cast(sourceSpan); + ReadOnlySpan remaining = sourceSpanUInt16; - if (Vector512.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector512.Count*2) - { - Vector512 v1 = Vector512.Create((ushort)c); - Vector512 v2 = Vector512.Create((ushort)c2); - Vector512 v3 = Vector512.Create((ushort)c3); - - if (Avx512BW.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) + if (Vector512.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector512.Count*2) { - // Process in double chunks & check if either chunk is likely to contain matches at once. - // If we get multiple matches in a single chunk, we assume they're likely to be close & - // break out of this logic & use the more optimistic loop below. - // This is similar logic to SpanHelpers.Packed.cs's IndexOf. - Vector512 packedComparand1 = Vector512.Create((byte)c); - Vector512 packedComparand2 = Vector512.Create((byte)c2); - Vector512 packedComparand3 = Vector512.Create((byte)c3); - while ((uint)remaining.Length >= (uint)Vector512.Count*2) - { - Vector512 vector1 = Vector512.Create(remaining); - Vector512 vector2 = Vector512.Create(remaining.Slice(Vector512.Count)); - var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); + Vector512 v1 = Vector512.Create((ushort)c); + Vector512 v2 = Vector512.Create((ushort)c2); + Vector512 v3 = Vector512.Create((ushort)c3); - if ((Vector512.Equals(packed, packedComparand1) | Vector512.Equals(packed, packedComparand2) | Vector512.Equals(packed, packedComparand3)) != Vector512.Zero) + if (Avx512BW.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) + { + // Process in double chunks & check if either chunk is likely to contain matches at once. + // If we get multiple matches in a single chunk, we assume they're likely to be close & + // break out of this logic & use the more optimistic loop below. + // This is similar logic to SpanHelpers.Packed.cs's IndexOf. + Vector512 packedComparand1 = Vector512.Create((byte)c); + Vector512 packedComparand2 = Vector512.Create((byte)c2); + Vector512 packedComparand3 = Vector512.Create((byte)c3); + while ((uint)remaining.Length >= (uint)Vector512.Count*2) { - var cmp1 = Vector512.Equals(vector1, v1).AsByte() | Vector512.Equals(vector1, v2).AsByte() | Vector512.Equals(vector1, v3).AsByte(); - var cmp2 = Vector512.Equals(vector2, v1).AsByte() | Vector512.Equals(vector2, v2).AsByte() | Vector512.Equals(vector2, v3).AsByte(); - - // Same logic as below, but for both vectors. - ulong mask1 = cmp1.ExtractMostSignificantBits() & 0x5555555555555555; - ulong mask2 = cmp2.ExtractMostSignificantBits() & 0x5555555555555555; - bool shouldBreak = ulong.PopCount(mask1) + ulong.PopCount(mask2) > 1; - while (mask1 != 0) + Vector512 vector1 = Vector512.Create(remaining); + Vector512 vector2 = Vector512.Create(remaining.Slice(Vector512.Count)); + var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); + + if ((Vector512.Equals(packed, packedComparand1) | Vector512.Equals(packed, packedComparand2) | Vector512.Equals(packed, packedComparand3)) != Vector512.Zero) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask1) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos); - mask1 = BitOperations.ResetLowestSetBit(mask1); + var cmp1 = Vector512.Equals(vector1, v1).AsByte() | Vector512.Equals(vector1, v2).AsByte() | Vector512.Equals(vector1, v3).AsByte(); + var cmp2 = Vector512.Equals(vector2, v1).AsByte() | Vector512.Equals(vector2, v2).AsByte() | Vector512.Equals(vector2, v3).AsByte(); + + // Same logic as below, but for both vectors. + ulong mask1 = cmp1.ExtractMostSignificantBits() & 0x5555555555555555; + ulong mask2 = cmp2.ExtractMostSignificantBits() & 0x5555555555555555; + bool shouldBreak = ulong.PopCount(mask1) + ulong.PopCount(mask2) > 1; + while (mask1 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask1) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos); + mask1 = BitOperations.ResetLowestSetBit(mask1); + } + while (mask2 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask2) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos + Vector512.Count); + mask2 = BitOperations.ResetLowestSetBit(mask2); + } + + // Break out of the loop if we had >1 match: + if (shouldBreak) + { + baseIndex += Vector512.Count*2; + remaining = remaining.Slice(Vector512.Count*2); + break; + } } - while (mask2 != 0) + + baseIndex += Vector512.Count*2; + remaining = remaining.Slice(Vector512.Count*2); + } + } + + while ((uint)remaining.Length > (uint)Vector512.Count) + { + Vector512 vector = Vector512.Create(remaining); + Vector512 cmp = Vector512.Equals(vector, v1).AsByte() | Vector512.Equals(vector, v2).AsByte() | Vector512.Equals(vector, v3).AsByte(); + + if (cmp != Vector512.Zero) + { + // Skip every other bit + ulong mask = cmp.ExtractMostSignificantBits() & 0x5555555555555555; + do { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask2) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos + Vector512.Count); - mask2 = BitOperations.ResetLowestSetBit(mask2); - } + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } while (mask != 0); + } + + baseIndex += Vector512.Count; + remaining = remaining.Slice(Vector512.Count); + } - // Break out of the loop if we had >1 match: - if (shouldBreak) + // Handle the last chunk in a vectorized way also. + // We do a whole vector's worth again, but just mask out the bits we've already handled. + // Note: when Avx512BW.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. + if (!Avx512BW.IsSupported || remaining.Length > 0) + { + Vector512 vector = Vector512.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector512.Count)); + Vector512 cmp = Vector512.Equals(vector, v1).AsByte() | Vector512.Equals(vector, v2).AsByte() | Vector512.Equals(vector, v3).AsByte(); + if (cmp != Vector512.Zero) + { + int finalIndex = sourceSpanUInt16.Length - Vector512.Count; + ulong mask = cmp.ExtractMostSignificantBits() & 0x5555555555555555 & ~((1UL << (Vector512.Count - remaining.Length * sizeof(char))) - 1); + while (mask != 0) { - baseIndex += Vector512.Count*2; - remaining = remaining.Slice(Vector512.Count*2); - break; + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); } } - - baseIndex += Vector512.Count*2; - remaining = remaining.Slice(Vector512.Count*2); } + return; } - - while ((uint)remaining.Length >= (uint)Vector512.Count) + else if (Vector256.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector256.Count*2) { - Vector512 vector = Vector512.Create(remaining); - Vector512 cmp = Vector512.Equals(vector, v1).AsByte() | Vector512.Equals(vector, v2).AsByte() | Vector512.Equals(vector, v3).AsByte(); + Vector256 v1 = Vector256.Create((ushort)c); + Vector256 v2 = Vector256.Create((ushort)c2); + Vector256 v3 = Vector256.Create((ushort)c3); - if (cmp != Vector512.Zero) + if (Avx2.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) { - // Skip every other bit - ulong mask = cmp.ExtractMostSignificantBits() & 0x5555555555555555; - do + // Process in double chunks & check if either chunk is likely to contain matches at once. + // If we get multiple matches in a single chunk, we assume they're likely to be close & + // break out of this logic & use the more optimistic loop below. + // This is similar logic to SpanHelpers.Packed.cs's IndexOf. + Vector256 packedComparand1 = Vector256.Create((byte)c); + Vector256 packedComparand2 = Vector256.Create((byte)c2); + Vector256 packedComparand3 = Vector256.Create((byte)c3); + while ((uint)remaining.Length >= (uint)Vector256.Count*2) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); - } while (mask != 0); - } + Vector256 vector1 = Vector256.Create(remaining); + Vector256 vector2 = Vector256.Create(remaining.Slice(Vector256.Count)); + var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); - baseIndex += Vector512.Count; - remaining = remaining.Slice(Vector512.Count); - } + if ((Vector256.Equals(packed, packedComparand1) | Vector256.Equals(packed, packedComparand2) | Vector256.Equals(packed, packedComparand3)) != Vector256.Zero) + { + var cmp1 = Vector256.Equals(vector1, v1).AsByte() | Vector256.Equals(vector1, v2).AsByte() | Vector256.Equals(vector1, v3).AsByte(); + var cmp2 = Vector256.Equals(vector2, v1).AsByte() | Vector256.Equals(vector2, v2).AsByte() | Vector256.Equals(vector2, v3).AsByte(); + + // Same logic as below, but for both vectors. + uint mask1 = cmp1.ExtractMostSignificantBits() & 0x55555555; + uint mask2 = cmp2.ExtractMostSignificantBits() & 0x55555555; + bool shouldBreak = uint.PopCount(mask1) + uint.PopCount(mask2) > 1; + while (mask1 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask1) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos); + mask1 = BitOperations.ResetLowestSetBit(mask1); + } + while (mask2 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask2) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos + Vector256.Count); + mask2 = BitOperations.ResetLowestSetBit(mask2); + } + + // Break out of the loop if we had > 1 match: + if (shouldBreak) + { + baseIndex += Vector256.Count*2; + remaining = remaining.Slice(Vector256.Count*2); + break; + } + } - // Handle the last chunk in a vectorized way also. - // We do a whole vector's worth again, but just mask out the bits we've already handled. - if (remaining.Length > 0) - { - Vector512 vector = Vector512.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector512.Count)); - Vector512 cmp = Vector512.Equals(vector, v1).AsByte() | Vector512.Equals(vector, v2).AsByte() | Vector512.Equals(vector, v3).AsByte(); - int finalIndex = sourceSpanUInt16.Length - Vector512.Count; - ulong mask = cmp.ExtractMostSignificantBits() & 0x5555555555555555 & ~((1UL << (Vector512.Count - remaining.Length * sizeof(char))) - 1); - while (mask != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); + baseIndex += Vector256.Count*2; + remaining = remaining.Slice(Vector256.Count*2); + } } - } - return; - } - else if (Vector256.IsHardwareAccelerated && (uint)remaining.Length >= (uint)Vector256.Count*2) - { - Vector256 v1 = Vector256.Create((ushort)c); - Vector256 v2 = Vector256.Create((ushort)c2); - Vector256 v3 = Vector256.Create((ushort)c3); - if (Avx2.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) - { - // Process in double chunks & check if either chunk is likely to contain matches at once. - // If we get multiple matches in a single chunk, we assume they're likely to be close & - // break out of this logic & use the more optimistic loop below. - // This is similar logic to SpanHelpers.Packed.cs's IndexOf. - Vector256 packedComparand1 = Vector256.Create((byte)c); - Vector256 packedComparand2 = Vector256.Create((byte)c2); - Vector256 packedComparand3 = Vector256.Create((byte)c3); - while ((uint)remaining.Length >= (uint)Vector256.Count*2) + while ((uint)remaining.Length > (uint)Vector256.Count) { - Vector256 vector1 = Vector256.Create(remaining); - Vector256 vector2 = Vector256.Create(remaining.Slice(Vector256.Count)); - var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); + Vector256 vector = Vector256.Create(remaining); + Vector256 cmp = Vector256.Equals(vector, v1).AsByte() | Vector256.Equals(vector, v2).AsByte() | Vector256.Equals(vector, v3).AsByte(); - if ((Vector256.Equals(packed, packedComparand1) | Vector256.Equals(packed, packedComparand2) | Vector256.Equals(packed, packedComparand3)) != Vector256.Zero) + if (cmp != Vector256.Zero) { - var cmp1 = Vector256.Equals(vector1, v1).AsByte() | Vector256.Equals(vector1, v2).AsByte() | Vector256.Equals(vector1, v3).AsByte(); - var cmp2 = Vector256.Equals(vector2, v1).AsByte() | Vector256.Equals(vector2, v2).AsByte() | Vector256.Equals(vector2, v3).AsByte(); - - // Same logic as below, but for both vectors. - uint mask1 = cmp1.ExtractMostSignificantBits() & 0x55555555; - uint mask2 = cmp2.ExtractMostSignificantBits() & 0x55555555; - bool shouldBreak = uint.PopCount(mask1) + uint.PopCount(mask2) > 1; - while (mask1 != 0) + // Skip every other bit + uint mask = cmp.ExtractMostSignificantBits() & 0x55555555; + do { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask1) / sizeof(char); + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); sepListBuilder.Append(baseIndex + (int)bitPos); - mask1 = BitOperations.ResetLowestSetBit(mask1); - } - while (mask2 != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask2) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos + Vector256.Count); - mask2 = BitOperations.ResetLowestSetBit(mask2); - } + mask = BitOperations.ResetLowestSetBit(mask); + } while (mask != 0); + } + + baseIndex += Vector256.Count; + remaining = remaining.Slice(Vector256.Count); + } - // Break out of the loop if we had > 1 match: - if (shouldBreak) + // Handle the last chunk in a vectorized way also. + // We do a whole vector's worth again, but just mask out the bits we've already handled. + // Note: when Avx2.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. + if (!Avx2.IsSupported || remaining.Length > 0) + { + Vector256 vector = Vector256.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector256.Count)); + Vector256 cmp = Vector256.Equals(vector, v1).AsByte() | Vector256.Equals(vector, v2).AsByte() | Vector256.Equals(vector, v3).AsByte(); + if (cmp != Vector256.Zero) + { + int finalIndex = sourceSpanUInt16.Length - Vector256.Count; + uint mask = cmp.ExtractMostSignificantBits() & 0x55555555 & ~((1u << (Vector256.Count - remaining.Length * sizeof(char))) - 1); + while (mask != 0) { - baseIndex += Vector256.Count*2; - remaining = remaining.Slice(Vector256.Count*2); - break; + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); } } - - baseIndex += Vector256.Count*2; - remaining = remaining.Slice(Vector256.Count*2); } + return; } - - while ((uint)remaining.Length >= (uint)Vector256.Count) + else if (Vector128.IsHardwareAccelerated) { - Vector256 vector = Vector256.Create(remaining); - Vector256 cmp = Vector256.Equals(vector, v1).AsByte() | Vector256.Equals(vector, v2).AsByte() | Vector256.Equals(vector, v3).AsByte(); + Vector128 v1 = Vector128.Create((ushort)c); + Vector128 v2 = Vector128.Create((ushort)c2); + Vector128 v3 = Vector128.Create((ushort)c3); - if (cmp != Vector256.Zero) + if (Sse2.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) { - // Skip every other bit - uint mask = cmp.ExtractMostSignificantBits() & 0x55555555; - do + // Process in double chunks & check if either chunk is likely to contain matches at once. + // If we get multiple matches in a single chunk, we assume they're likely to be close & + // break out of this logic & use the more optimistic loop below. + // This is similar logic to SpanHelpers.Packed.cs's IndexOf. + Vector128 packedComparand1 = Vector128.Create((byte)c); + Vector128 packedComparand2 = Vector128.Create((byte)c2); + Vector128 packedComparand3 = Vector128.Create((byte)c3); + while ((uint)remaining.Length >= (uint)Vector128.Count*2) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); - } while (mask != 0); - } + Vector128 vector1 = Vector128.Create(remaining); + Vector128 vector2 = Vector128.Create(remaining.Slice(Vector128.Count)); + var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); - baseIndex += Vector256.Count; - remaining = remaining.Slice(Vector256.Count); - } + if ((Vector128.Equals(packed, packedComparand1) | Vector128.Equals(packed, packedComparand2) | Vector128.Equals(packed, packedComparand3)) != Vector128.Zero) + { + var cmp1 = Vector128.Equals(vector1, v1).AsByte() | Vector128.Equals(vector1, v2).AsByte() | Vector128.Equals(vector1, v3).AsByte(); + var cmp2 = Vector128.Equals(vector2, v1).AsByte() | Vector128.Equals(vector2, v2).AsByte() | Vector128.Equals(vector2, v3).AsByte(); + + // Same logic as below, but for both vectors. + uint mask1 = cmp1.ExtractMostSignificantBits() & 0x5555; + uint mask2 = cmp2.ExtractMostSignificantBits() & 0x5555; + bool shouldBreak = uint.PopCount(mask1) + uint.PopCount(mask2) > 1; + while (mask1 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask1) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos); + mask1 = BitOperations.ResetLowestSetBit(mask1); + } + while (mask2 != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask2) / sizeof(char); + sepListBuilder.Append(baseIndex + (int)bitPos + Vector128.Count); + mask2 = BitOperations.ResetLowestSetBit(mask2); + } + + // Break out of the loop if we had > 1 match: + if (shouldBreak) + { + baseIndex += Vector128.Count*2; + remaining = remaining.Slice(Vector128.Count*2); + break; + } + } - // Handle the last chunk in a vectorized way also. - // We do a whole vector's worth again, but just mask out the bits we've already handled. - if (remaining.Length > 0) - { - Vector256 vector = Vector256.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector256.Count)); - Vector256 cmp = Vector256.Equals(vector, v1).AsByte() | Vector256.Equals(vector, v2).AsByte() | Vector256.Equals(vector, v3).AsByte(); - int finalIndex = sourceSpanUInt16.Length - Vector256.Count; - uint mask = cmp.ExtractMostSignificantBits() & 0x55555555 & ~((1u << (Vector256.Count - remaining.Length * sizeof(char))) - 1); - while (mask != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); + baseIndex += Vector128.Count*2; + remaining = remaining.Slice(Vector128.Count*2); + } } - } - return; - } - else if (Vector128.IsHardwareAccelerated) - { - Vector128 v1 = Vector128.Create((ushort)c); - Vector128 v2 = Vector128.Create((ushort)c2); - Vector128 v3 = Vector128.Create((ushort)c3); - if (Sse2.IsSupported && PackedSpanHelpers.CanUsePackedIndexOf(c) && PackedSpanHelpers.CanUsePackedIndexOf(c2) && PackedSpanHelpers.CanUsePackedIndexOf(c3)) - { - // Process in double chunks & check if either chunk is likely to contain matches at once. - // If we get multiple matches in a single chunk, we assume they're likely to be close & - // break out of this logic & use the more optimistic loop below. - // This is similar logic to SpanHelpers.Packed.cs's IndexOf. - Vector128 packedComparand1 = Vector128.Create((byte)c); - Vector128 packedComparand2 = Vector128.Create((byte)c2); - Vector128 packedComparand3 = Vector128.Create((byte)c3); - while ((uint)remaining.Length >= (uint)Vector128.Count*2) + while ((uint)remaining.Length > (uint)Vector128.Count) { - Vector128 vector1 = Vector128.Create(remaining); - Vector128 vector2 = Vector128.Create(remaining.Slice(Vector128.Count)); - var packed = PackedSpanHelpers.PackSources(vector1.AsInt16(), vector2.AsInt16()); + Vector128 vector = Vector128.Create(remaining); + Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); - if ((Vector128.Equals(packed, packedComparand1) | Vector128.Equals(packed, packedComparand2) | Vector128.Equals(packed, packedComparand3)) != Vector128.Zero) + if (cmp != Vector128.Zero) { - var cmp1 = Vector128.Equals(vector1, v1).AsByte() | Vector128.Equals(vector1, v2).AsByte() | Vector128.Equals(vector1, v3).AsByte(); - var cmp2 = Vector128.Equals(vector2, v1).AsByte() | Vector128.Equals(vector2, v2).AsByte() | Vector128.Equals(vector2, v3).AsByte(); - - // Same logic as below, but for both vectors. - uint mask1 = cmp1.ExtractMostSignificantBits() & 0x5555; - uint mask2 = cmp2.ExtractMostSignificantBits() & 0x5555; - bool shouldBreak = uint.PopCount(mask1) + uint.PopCount(mask2) > 1; - while (mask1 != 0) + // Skip every other bit + uint mask = cmp.ExtractMostSignificantBits() & 0x5555; + do { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask1) / sizeof(char); + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); sepListBuilder.Append(baseIndex + (int)bitPos); - mask1 = BitOperations.ResetLowestSetBit(mask1); - } - while (mask2 != 0) - { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask2) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos + Vector128.Count); - mask2 = BitOperations.ResetLowestSetBit(mask2); - } - - // Break out of the loop if we had > 1 match: - if (shouldBreak) - { - baseIndex += Vector128.Count*2; - remaining = remaining.Slice(Vector128.Count*2); - break; - } + mask = BitOperations.ResetLowestSetBit(mask); + } while (mask != 0); } - baseIndex += Vector128.Count*2; - remaining = remaining.Slice(Vector128.Count*2); + baseIndex += Vector128.Count; + remaining = remaining.Slice(Vector128.Count); } - } - while ((uint)remaining.Length >= (uint)Vector128.Count) - { - Vector128 vector = Vector128.Create(remaining); - Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); - - if (cmp != Vector128.Zero) + // Handle the last chunk in a vectorized way also. + // We do a whole vector's worth again, but just mask out the bits we've already handled. + // Note: when Sse.IsSupported is false, we never enter the first block, and thus remaining.Length is always > 0. + if (!Sse.IsSupported || remaining.Length > 0) { - // Skip every other bit - uint mask = cmp.ExtractMostSignificantBits() & 0x5555; - do + Vector128 vector = Vector128.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector128.Count)); + Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); + if (cmp != Vector128.Zero) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(baseIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); - } while (mask != 0); + int finalIndex = sourceSpanUInt16.Length - Vector128.Count; + uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); + while (mask != 0) + { + uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); + sepListBuilder.Append(finalIndex + (int)bitPos); + mask = BitOperations.ResetLowestSetBit(mask); + } + } } - - baseIndex += Vector128.Count; - remaining = remaining.Slice(Vector128.Count); + return; } - - // Handle the last chunk in a vectorized way also. - // We do a whole vector's worth again, but just mask out the bits we've already handled. - if (remaining.Length > 0) + } + else + { + for (int i = 0; i < sourceSpan.Length; i++) { - Vector128 vector = Vector128.Create(sourceSpanUInt16.Slice(sourceSpanUInt16.Length - Vector128.Count)); - Vector128 cmp = Vector128.Equals(vector, v1).AsByte() | Vector128.Equals(vector, v2).AsByte() | Vector128.Equals(vector, v3).AsByte(); - int finalIndex = sourceSpanUInt16.Length - Vector128.Count; - uint mask = cmp.ExtractMostSignificantBits() & 0x5555 & ~((1u << (Vector128.Count - remaining.Length * sizeof(char))) - 1); - while (mask != 0) + char v = sourceSpan[i]; + if (v == c || v == c2 || v == c3) { - uint bitPos = (uint)BitOperations.TrailingZeroCount(mask) / sizeof(char); - sepListBuilder.Append(finalIndex + (int)bitPos); - mask = BitOperations.ResetLowestSetBit(mask); + sepListBuilder.Append(i); } } + return; } - - Debug.Fail("We should not be able to reach this point of MakeSeparatorListVectorized."); } ///