diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index 6f51248f60d9c1..1b546f40cf0eb7 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -80,6 +80,68 @@ static Range GetRange(Compiler* comp, GenTree* tree, BasicBlock* block, ASSERT_V return Limit(Limit::keUnknown); } +#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_ARM64) +//---------------------------------------------------------------------------------------------- +// optAssertionProp_HWIntrinsic: Propagate VN-derived facts to local var metadata. +// +// Arguments: +// comp - The compiler instance +// tree - The hwintrinsic node +// +static void optAssertionProp_HWIntrinsic(Compiler* comp, GenTreeHWIntrinsic* tree) +{ + NamedIntrinsic intrinsic = tree->GetHWIntrinsicId(); + + if ((intrinsic != NI_Vector64_ExtractMostSignificantBits) && (intrinsic != NI_Vector128_ExtractMostSignificantBits)) + { + return; + } + + assert(tree->GetOperandCount() == 1); + + GenTree* op1 = tree->Op(1); + + if (!op1->OperIs(GT_LCL_VAR)) + { + return; + } + + LclVarDsc* varDsc = comp->lvaGetDesc(op1->AsLclVar()); + + if (!varDsc->lvSingleDef) + { + return; + } + + ValueNum op1VN = comp->vnStore->VNConservativeNormalValue(op1->gtVNPair); + + auto vnVisitor = [comp, tree](ValueNum vn) -> ValueNumStore::VNVisit { + if (vn == ValueNumStore::NoVN) + { + return ValueNumStore::VNVisit::Abort; + } + + vn = comp->vnStore->VNNormalValue(vn); + var_types type = comp->vnStore->TypeOfVN(vn); + unsigned simdSize = tree->GetSimdSize(); + + if (!varTypeIsSIMD(type) || (genTypeSize(type) != simdSize)) + { + return ValueNumStore::VNVisit::Abort; + } + + return comp->vnStore->IsVectorPerElementMask(vn, tree->GetSimdBaseType(), simdSize) + ? ValueNumStore::VNVisit::Continue + : ValueNumStore::VNVisit::Abort; + }; + + if (comp->vnStore->VNVisitReachingVNs(op1VN, vnVisitor) == ValueNumStore::VNVisit::Continue) + { + varDsc->SetIsVectorPerElementMask(tree->GetSimdBaseType()); + } +} +#endif // FEATURE_HW_INTRINSICS && TARGET_ARM64 + //------------------------------------------------------------------------ // SymbolicToRealValue: Convert a symbolic value to a 64-bit signed integer. // @@ -5822,6 +5884,12 @@ GenTree* Compiler::optAssertionProp(ASSERT_VALARG_TP assertions, GenTree* tree, case GT_CALL: return optAssertionProp_Call(assertions, tree->AsCall(), stmt); +#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_ARM64) + case GT_HWINTRINSIC: + optAssertionProp_HWIntrinsic(this, tree->AsHWIntrinsic()); + return nullptr; +#endif // FEATURE_HW_INTRINSICS && TARGET_ARM64 + case GT_EQ: case GT_NE: case GT_LT: diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 5e7ef8beb9c7ee..8ffa7617b6e531 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -668,6 +668,9 @@ class LclVarDsc unsigned char lvIsSpan : 1; // The local is a Span + unsigned char lvIsVectorPerElementMask : 1; // The local is known to be a per-element mask + unsigned char lvVectorPerElementMaskElemSizeLog2 : 2; // Maximum log2(element size) for the local mask + public: union { @@ -869,6 +872,58 @@ class LclVarDsc lvIsSpan = value; } +#ifdef FEATURE_HW_INTRINSICS + // Is this local a per-element mask compatible with the given base type? + bool IsVectorPerElementMask(var_types simdBaseType) const + { + return lvIsVectorPerElementMask && + (GetVectorPerElementMaskElemSizeLog2(simdBaseType) <= lvVectorPerElementMaskElemSizeLog2); + } + + // Mark this local as a per-element mask with the given base type. + void SetIsVectorPerElementMask(var_types simdBaseType) + { + unsigned elemSizeLog2 = GetVectorPerElementMaskElemSizeLog2(simdBaseType); + + if (!lvIsVectorPerElementMask || (elemSizeLog2 > lvVectorPerElementMaskElemSizeLog2)) + { + lvVectorPerElementMaskElemSizeLog2 = static_cast(elemSizeLog2); + } + + lvIsVectorPerElementMask = true; + } + +private: + static unsigned GetVectorPerElementMaskElemSizeLog2(var_types simdBaseType) + { + switch (simdBaseType) + { + case TYP_BYTE: + case TYP_UBYTE: + return 0; + + case TYP_SHORT: + case TYP_USHORT: + return 1; + + case TYP_INT: + case TYP_UINT: + case TYP_FLOAT: + return 2; + + case TYP_LONG: + case TYP_ULONG: + case TYP_DOUBLE: + return 3; + + default: + unreached(); + } + } + +public: +#endif // FEATURE_HW_INTRINSICS + ///////////////////// regNumber GetArgInitReg() const @@ -10409,7 +10464,7 @@ class Compiler return simdType; } - static var_types getIndexTypeForShuffle(var_types simdBaseType) + static var_types getUnsignedSimdBaseType(var_types simdBaseType) { switch (simdBaseType) { diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 9ad468aa31c7b2..3d2a653c16f0c7 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27342,7 +27342,7 @@ GenTree* Compiler::gtNewSimdZipNode( } GenTreeVecCon* shuffle = gtNewVconNode(type); - var_types indexBaseType = getIndexTypeForShuffle(simdBaseType); + var_types indexBaseType = getUnsignedSimdBaseType(simdBaseType); uint32_t start = upper ? (simdCount / 2) : 0; for (uint32_t index = 0; index < simdCount; index++) @@ -27450,7 +27450,7 @@ GenTree* Compiler::gtNewSimdUnzipNode( #elif defined(TARGET_XARCH) unsigned elementSize = genTypeSize(simdBaseType); - var_types indexBaseType = getIndexTypeForShuffle(simdBaseType); + var_types indexBaseType = getUnsignedSimdBaseType(simdBaseType); if (simdSize == 16) { @@ -27605,7 +27605,7 @@ GenTree* Compiler::gtNewSimdReverseNode(var_types type, GenTree* op1, var_types // return Shuffle(op1, indices); GenTreeVecCon* shuffle = gtNewVconNode(type); - var_types indexBaseType = getIndexTypeForShuffle(simdBaseType); + var_types indexBaseType = getUnsignedSimdBaseType(simdBaseType); for (uint32_t index = 0; index < simdCount; index++) { @@ -33767,6 +33767,11 @@ bool GenTree::IsInvariant() const // True if this node is a per-element mask compatible with simdBaseType and simdSize // bool GenTree::IsVectorPerElementMask(var_types simdBaseType, unsigned simdSize) const +{ + return IsVectorPerElementMask(nullptr, simdBaseType, simdSize); +} + +bool GenTree::IsVectorPerElementMask(Compiler* comp, var_types simdBaseType, unsigned simdSize) const { #ifdef FEATURE_SIMD // This should be kept in sync with ValueNumStore::IsVectorPerElementMask @@ -33774,8 +33779,10 @@ bool GenTree::IsVectorPerElementMask(var_types simdBaseType, unsigned simdSize) var_types simdType = TypeGet(); unsigned elementCount = GenTreeVecCon::ElementCount(simdSize, simdBaseType); - assert(varTypeIsSIMD(simdType)); - assert(genTypeSize(simdType) == simdSize); + if (!varTypeIsSIMD(simdType) || (genTypeSize(simdType) != simdSize)) + { + return false; + } if (IsCnsVec()) { @@ -33783,6 +33790,11 @@ bool GenTree::IsVectorPerElementMask(var_types simdBaseType, unsigned simdSize) return ElementsAreAllBitsSetOrZero(&vecCon->gtSimdVal, simdBaseType, elementCount); } + if ((comp != nullptr) && OperIs(GT_LCL_VAR)) + { + return comp->lvaGetDesc(AsLclVar())->IsVectorPerElementMask(simdBaseType); + } + if (!OperIsHWIntrinsic()) { return false; @@ -33821,6 +33833,11 @@ bool GenTree::IsVectorPerElementMask(var_types simdBaseType, unsigned simdSize) bool isScalar = false; genTreeOps oper = GenTreeHWIntrinsic::GetOperForHWIntrinsicId(intrinsicId, simdBaseType, &isScalar); + if (!isScalar && GenTree::OperIsCmpCompare(oper)) + { + return genTypeSize(intrinsicSimdBaseType) >= genTypeSize(simdBaseType); + } + switch (oper) { case GT_AND: @@ -33836,14 +33853,14 @@ bool GenTree::IsVectorPerElementMask(var_types simdBaseType, unsigned simdSize) // there isn't any way to statically determine this for non-constants and // the constant cases should've already been folded. - return intrinsic->Op(1)->IsVectorPerElementMask(simdBaseType, simdSize) && - intrinsic->Op(2)->IsVectorPerElementMask(simdBaseType, simdSize); + return intrinsic->Op(1)->IsVectorPerElementMask(comp, simdBaseType, simdSize) && + intrinsic->Op(2)->IsVectorPerElementMask(comp, simdBaseType, simdSize); } case GT_NOT: { // We are a unary bitwise operation where the input is a per-element mask - return intrinsic->Op(1)->IsVectorPerElementMask(simdBaseType, simdSize); + return intrinsic->Op(1)->IsVectorPerElementMask(comp, simdBaseType, simdSize); } default: diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index e7e3e0f2bf608c..f3aa57e6a64d72 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -2537,6 +2537,7 @@ struct GenTree bool IsInvariant() const; bool IsVectorPerElementMask(var_types simdBaseType, unsigned simdSize) const; + bool IsVectorPerElementMask(class Compiler* comp, var_types simdBaseType, unsigned simdSize) const; bool IsNeverNegative(Compiler* comp) const; bool IsNeverNegativeOne(Compiler* comp) const; diff --git a/src/coreclr/jit/rationalize.cpp b/src/coreclr/jit/rationalize.cpp index 87460c47c622a9..1160d007a69be9 100644 --- a/src/coreclr/jit/rationalize.cpp +++ b/src/coreclr/jit/rationalize.cpp @@ -1335,6 +1335,531 @@ bool Rationalizer::ShouldRewriteToNonMaskHWIntrinsic(GenTree* node) } #endif // TARGET_XARCH +#if defined(TARGET_ARM64) +//---------------------------------------------------------------------------------------------- +// NormalizeCmpMaskSimdBaseType: Normalize a SIMD comparison mask's base type to unsigned. +// +// Arguments: +// simdBaseType - The SIMD base type. +// +// Return Value: +// The normalized SIMD base type, or TYP_UNDEF if it is unsupported. +// +static var_types NormalizeCmpMaskSimdBaseType(var_types simdBaseType) +{ + switch (simdBaseType) + { + case TYP_BYTE: + case TYP_UBYTE: + case TYP_SHORT: + case TYP_USHORT: + case TYP_INT: + case TYP_UINT: + { + return Compiler::getUnsignedSimdBaseType(simdBaseType); + } + + default: + { + return TYP_UNDEF; + } + } +} + +//---------------------------------------------------------------------------------------------- +// IsHWIntrinsicCmpMaskExtractMsb: Checks if an ExtractMostSignificantBits node consumes a SIMD +// comparison mask. +// +// Arguments: +// comp - The compiler instance. +// node - The hwintrinsic node. +// +// Return Value: +// True if the node is an ExtractMostSignificantBits over a SIMD comparison mask. +// +static bool IsHWIntrinsicCmpMaskExtractMsb(Compiler* comp, GenTreeHWIntrinsic* node) +{ + if ((node->GetHWIntrinsicId() != NI_Vector64_ExtractMostSignificantBits) && + (node->GetHWIntrinsicId() != NI_Vector128_ExtractMostSignificantBits)) + { + return false; + } + + var_types simdBaseType = NormalizeCmpMaskSimdBaseType(node->GetSimdBaseType()); + + if (simdBaseType == TYP_UNDEF) + { + return false; + } + + GenTree* op1 = node->Op(1); + + return op1->IsVectorPerElementMask(comp, simdBaseType, node->GetSimdSize()); +} + +//---------------------------------------------------------------------------------------------- +// IsPrimitivePopCount: Checks if a node is a primitive PopCount intrinsic. +// +// Arguments: +// node - The node to check. +// +// Return Value: +// True if the node is a primitive PopCount intrinsic. +// +static bool IsPrimitivePopCount(GenTree* node) +{ + return node->OperIs(GT_INTRINSIC) && (node->AsIntrinsic()->gtIntrinsicName == NI_PRIMITIVE_PopCount); +} + +//---------------------------------------------------------------------------------------------- +// IsZeroCount: Checks if a node is a scalar zero-count intrinsic. +// +// Arguments: +// node - The node to check. +// +// Return Value: +// True if the node is a TrailingZeroCount or LeadingZeroCount intrinsic. +// +static bool IsZeroCount(GenTree* node) +{ + if (node->OperIs(GT_INTRINSIC)) + { + return (node->AsIntrinsic()->gtIntrinsicName == NI_PRIMITIVE_TrailingZeroCount) || + (node->AsIntrinsic()->gtIntrinsicName == NI_PRIMITIVE_LeadingZeroCount); + } + + if (node->OperIsHWIntrinsic()) + { + return node->AsHWIntrinsic()->GetHWIntrinsicId() == NI_ArmBase_LeadingZeroCount; + } + + return false; +} + +//---------------------------------------------------------------------------------------------- +// ReplaceHWIntrinsicCmpMaskExtractMsbUse: Replace a scalarized comparison mask extraction with +// the specified replacement node. +// +// Arguments: +// use - A pointer to the node being replaced +// parents - A reference to tree walk data providing the context +// oldNode - The node being replaced +// replacement - The node that replaces *use +// +static void ReplaceHWIntrinsicCmpMaskExtractMsbUse(GenTree** use, + Compiler::GenTreeStack& parents, + GenTree* oldNode, + GenTree* replacement) +{ + if (parents.Height() > 1) + { + parents.Top(1)->ReplaceOperand(use, replacement); + } + else + { + *use = replacement; + } + + // Adjust the parent stack + assert(parents.Top() == oldNode); + (void)parents.Pop(); + parents.Push(replacement); +} + +//---------------------------------------------------------------------------------------------- +// ScalarizeHWIntrinsicCmpMaskReduction: Update an ExtractMostSignificantBits node so it scalarizes +// a vector reduction result. +// +// Arguments: +// node - The ExtractMostSignificantBits node to update +// reduction - The vector reduction node +// simdBaseType - The SIMD base type of the reduction +// simdSize - The SIMD size of the original input +// +static void ScalarizeHWIntrinsicCmpMaskReduction(GenTreeHWIntrinsic* node, + GenTree* reduction, + var_types simdBaseType, + unsigned simdSize) +{ + NamedIntrinsic intrinsic = (simdSize == 8) ? NI_Vector64_ToScalar : NI_Vector128_ToScalar; + + node->gtType = genActualType(simdBaseType); + node->ChangeHWIntrinsicId(intrinsic); + node->SetSimdSize(8); + node->SetSimdBaseType(simdBaseType); + node->Op(1) = reduction; +} + +//---------------------------------------------------------------------------------------------- +// RewriteHWIntrinsicCmpMaskExtractMsb: +// Rewrites an ExtractMostSignificantBits operation when the input is known to be a SIMD comparison +// mask and the result is only checked for zero. +// +// Matches: +// ExtractMostSignificantBits(cmpMask) == 0 +// ExtractMostSignificantBits(cmpMask) != 0 +// +// Replaces the ExtractMostSignificantBits with: +// MaxAcross(cmpMask) +// +// This computes whether any all-bits-set comparison element exists without materializing the full +// bitmask. For Vector64, the reduction is implemented with MaxPairwise. +// +// Arguments: +// use - A pointer to the hwintrinsic node +// parents - A reference to tree walk data providing the context +// +// Return Value: +// True if the node was rewritten; otherwise false. +// +bool Rationalizer::RewriteHWIntrinsicCmpMaskExtractMsb(GenTree** use, Compiler::GenTreeStack& parents) +{ + GenTreeHWIntrinsic* node = (*use)->AsHWIntrinsic(); + + var_types simdBaseType = node->GetSimdBaseType(); + unsigned simdSize = node->GetSimdSize(); + + simdBaseType = NormalizeCmpMaskSimdBaseType(simdBaseType); + + if (simdBaseType == TYP_UNDEF) + { + return false; + } + + if (parents.Height() <= 1) + { + return false; + } + + GenTree* parent = parents.Top(1); + + if (!parent->OperIs(GT_EQ, GT_NE)) + { + return false; + } + + GenTree* parentOp1 = parent->gtGetOp1(); + GenTree* parentOp2 = parent->gtGetOp2(); + + if (!(((parentOp1 == node) && parentOp2->IsIntegralConst(0)) || + ((parentOp2 == node) && parentOp1->IsIntegralConst(0)))) + { + return false; + } + + GenTree* op1 = node->Op(1); + + if (!IsHWIntrinsicCmpMaskExtractMsb(m_compiler, node)) + { + return false; + } + + // A comparison produces elements whose value is either all-bits-set or zero. When the + // ExtractMostSignificantBits result is only being compared against zero, use a horizontal max + // reduction to determine if any element was all-bits-set without materializing the full mask. + + GenTree* tmp; + + if ((simdSize == 8) && (simdBaseType == TYP_UINT)) + { + // Vector64 has only two lanes and AdvSimd does not provide a 2S MaxAcross form. + // Use a pairwise reduction with the same vector as both operands instead. + + LIR::Use op1Use; + LIR::Use::MakeDummyUse(BlockRange(), op1, &op1Use); + + // The pairwise form consumes op1 twice, so spill it to a temp before cloning the use. + op1Use.ReplaceWithLclVar(m_compiler); + op1 = op1Use.Def(); + + GenTree* op2 = m_compiler->gtClone(op1); + BlockRange().InsertAfter(op1, op2); + + tmp = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, op2, NI_AdvSimd_MaxPairwise, simdBaseType, simdSize); + BlockRange().InsertAfter(op2, tmp); + } + else + { + tmp = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_MaxAcross, simdBaseType, simdSize); + BlockRange().InsertAfter(op1, tmp); + } + + op1 = tmp; + + ScalarizeHWIntrinsicCmpMaskReduction(node, op1, simdBaseType, simdSize); + + GenTree* castNode = m_compiler->gtNewCastNode(TYP_INT, node, /* isUnsigned */ true, TYP_INT); + BlockRange().InsertAfter(node, castNode); + + ReplaceHWIntrinsicCmpMaskExtractMsbUse(use, parents, node, castNode); + + return true; +} + +//---------------------------------------------------------------------------------------------- +// RewriteHWIntrinsicCmpMaskExtractMsbPopCount: +// Rewrites PopCount(ExtractMostSignificantBits(...)) when the input is known to be a SIMD +// comparison mask. +// +// Matches: +// PopCount(ExtractMostSignificantBits(cmpMask)) +// +// Replaces it with: +// AddAcross(ShiftRightLogical(cmpMask, elementBits - 1)) +// +// This converts each all-bits-set comparison element to 1, each zero element to 0, then horizontally +// sums those per-element counts. For Vector64, the reduction is implemented with AddPairwise. +// +// Arguments: +// use - A pointer to the intrinsic node +// parents - A reference to tree walk data providing the context +// +// Return Value: +// True if the node was rewritten; otherwise false. +// +bool Rationalizer::RewriteHWIntrinsicCmpMaskExtractMsbPopCount(GenTree** use, Compiler::GenTreeStack& parents) +{ + GenTreeIntrinsic* popCount = (*use)->AsIntrinsic(); + assert(popCount->gtIntrinsicName == NI_PRIMITIVE_PopCount); + + GenTree* extract = popCount->gtGetOp1(); + + if (!extract->OperIsHWIntrinsic()) + { + return false; + } + + GenTreeHWIntrinsic* extractNode = extract->AsHWIntrinsic(); + var_types simdBaseType = NormalizeCmpMaskSimdBaseType(extractNode->GetSimdBaseType()); + + if (simdBaseType == TYP_UNDEF) + { + return false; + } + + if (!IsHWIntrinsicCmpMaskExtractMsb(m_compiler, extractNode)) + { + return false; + } + + unsigned simdSize = extractNode->GetSimdSize(); + unsigned elementBitSize = genTypeSize(simdBaseType) * BITS_PER_BYTE; + + // A comparison produces elements whose value is either all-bits-set or zero. For a Count-style + // consumer, normalize each element to one or zero and then horizontally sum the elements. + + GenTree* op1 = extractNode->Op(1); + + GenTree* shiftAmount = m_compiler->gtNewIconNode(elementBitSize - 1); + BlockRange().InsertAfter(op1, shiftAmount); + + GenTree* shift = m_compiler->gtNewSimdHWIntrinsicNode(Compiler::getSIMDTypeForSize(simdSize), op1, shiftAmount, + NI_AdvSimd_ShiftRightLogical, simdBaseType, simdSize); + BlockRange().InsertAfter(shiftAmount, shift); + op1 = shift; + + GenTree* add; + + if ((simdSize == 8) && (simdBaseType == TYP_UINT)) + { + // Vector64 has only two lanes and AdvSimd does not provide a 2S AddAcross form. + // Use a pairwise reduction with the same vector as both operands instead. + + LIR::Use op1Use; + LIR::Use::MakeDummyUse(BlockRange(), op1, &op1Use); + + // The pairwise form consumes op1 twice, so spill it to a temp before cloning the use. + op1Use.ReplaceWithLclVar(m_compiler); + op1 = op1Use.Def(); + + GenTree* op2 = m_compiler->gtClone(op1); + BlockRange().InsertAfter(op1, op2); + + add = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, op2, NI_AdvSimd_AddPairwise, simdBaseType, simdSize); + BlockRange().InsertAfter(op2, add); + } + else + { + add = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_AddAcross, simdBaseType, simdSize); + BlockRange().InsertAfter(op1, add); + } + + op1 = add; + + ScalarizeHWIntrinsicCmpMaskReduction(extractNode, op1, simdBaseType, simdSize); + + GenTree* castNode = m_compiler->gtNewCastNode(TYP_INT, extractNode, /* isUnsigned */ true, TYP_INT); + BlockRange().InsertAfter(extractNode, castNode); + + BlockRange().Remove(popCount); + + ReplaceHWIntrinsicCmpMaskExtractMsbUse(use, parents, popCount, castNode); + + return true; +} + +//---------------------------------------------------------------------------------------------- +// RewriteHWIntrinsicCmpMaskExtractMsbZeroCount: +// Rewrites TrailingZeroCount(ExtractMostSignificantBits(...)) and +// LeadingZeroCount(ExtractMostSignificantBits(...)) when the input is known to be a SIMD comparison mask. +// +// Matches: +// TrailingZeroCount(ExtractMostSignificantBits(cmpMask)) +// LeadingZeroCount(ExtractMostSignificantBits(cmpMask)) +// +// Replaces it with: +// TrailingZeroCount: MinAcross(BitwiseSelect(cmpMask, IndexVector, SentinelVector)) - 1 +// LeadingZeroCount: MinAcross(BitwiseSelect(cmpMask, IndexVector, SentinelVector)) +// +// For TrailingZeroCount, IndexVector holds one-based element indexes and SentinelVector holds 33. +// The selected minimum is therefore the first matching element index plus one, or 33 if no element +// matched. Subtracting one preserves the zero-mask result of 32. +// +// For LeadingZeroCount, IndexVector holds 31 minus the element index and SentinelVector holds 32. +// The selected minimum is therefore the leading-zero-count result directly, including 32 for the +// zero-mask case. For Vector64, the reduction is implemented with MinPairwise. +// +// Arguments: +// use - A pointer to the intrinsic node +// parents - A reference to tree walk data providing the context +// +// Return Value: +// True if the node was rewritten; otherwise false. +// +bool Rationalizer::RewriteHWIntrinsicCmpMaskExtractMsbZeroCount(GenTree** use, Compiler::GenTreeStack& parents) +{ + GenTree* zeroCount = *use; + assert(IsZeroCount(zeroCount)); + + const bool isTrailingZeroCount = zeroCount->OperIs(GT_INTRINSIC) && + (zeroCount->AsIntrinsic()->gtIntrinsicName == NI_PRIMITIVE_TrailingZeroCount); + + GenTree* extract = zeroCount->OperIs(GT_INTRINSIC) ? zeroCount->gtGetOp1() : zeroCount->AsHWIntrinsic()->Op(1); + + if (!extract->OperIsHWIntrinsic()) + { + return false; + } + + GenTreeHWIntrinsic* extractNode = extract->AsHWIntrinsic(); + var_types simdBaseType = NormalizeCmpMaskSimdBaseType(extractNode->GetSimdBaseType()); + + if (simdBaseType == TYP_UNDEF) + { + return false; + } + + if (!IsHWIntrinsicCmpMaskExtractMsb(m_compiler, extractNode)) + { + return false; + } + + unsigned simdSize = extractNode->GetSimdSize(); + var_types simdType = Compiler::getSIMDTypeForSize(simdSize); + + // A comparison produces elements whose value is either all-bits-set or zero. Select an element + // index value when the comparison is true and a sentinel when false. The horizontal min reduction + // then finds the zero-count result directly or with a final subtract, as described above. + + GenTree* op1 = extractNode->Op(1); + + GenTreeVecCon* indexVec = m_compiler->gtNewVconNode(simdType); + GenTreeVecCon* otherVec = m_compiler->gtNewVconNode(simdType); + + const unsigned elementSize = genTypeSize(simdBaseType); + const unsigned elementCount = simdSize / elementSize; + + for (unsigned index = 0; index < elementCount; index++) + { + switch (simdBaseType) + { + case TYP_UBYTE: + { + indexVec->gtSimdVal.u8[index] = static_cast(isTrailingZeroCount ? index + 1 : 31 - index); + otherVec->gtSimdVal.u8[index] = static_cast(isTrailingZeroCount ? 33 : 32); + break; + } + + case TYP_USHORT: + { + indexVec->gtSimdVal.u16[index] = static_cast(isTrailingZeroCount ? index + 1 : 31 - index); + otherVec->gtSimdVal.u16[index] = static_cast(isTrailingZeroCount ? 33 : 32); + break; + } + + case TYP_UINT: + { + indexVec->gtSimdVal.u32[index] = static_cast(isTrailingZeroCount ? index + 1 : 31 - index); + otherVec->gtSimdVal.u32[index] = static_cast(isTrailingZeroCount ? 33 : 32); + break; + } + + default: + { + unreached(); + } + } + } + + BlockRange().InsertAfter(op1, indexVec); + BlockRange().InsertAfter(indexVec, otherVec); + + GenTree* select = m_compiler->gtNewSimdCndSelNode(simdType, op1, indexVec, otherVec, simdBaseType, simdSize); + BlockRange().InsertAfter(otherVec, select); + op1 = select; + + GenTree* min; + + if ((simdSize == 8) && (simdBaseType == TYP_UINT)) + { + // Vector64 has only two lanes and AdvSimd does not provide a 2S MinAcross form. + // Use a pairwise reduction with the same vector as both operands instead. + + LIR::Use op1Use; + LIR::Use::MakeDummyUse(BlockRange(), op1, &op1Use); + + // The pairwise form consumes op1 twice, so spill it to a temp before cloning the use. + op1Use.ReplaceWithLclVar(m_compiler); + op1 = op1Use.Def(); + + GenTree* op2 = m_compiler->gtClone(op1); + BlockRange().InsertAfter(op1, op2); + + min = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, op2, NI_AdvSimd_MinPairwise, simdBaseType, simdSize); + BlockRange().InsertAfter(op2, min); + } + else + { + min = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_MinAcross, simdBaseType, simdSize); + BlockRange().InsertAfter(op1, min); + } + + op1 = min; + + ScalarizeHWIntrinsicCmpMaskReduction(extractNode, op1, simdBaseType, simdSize); + + GenTree* castNode = m_compiler->gtNewCastNode(TYP_INT, extractNode, /* isUnsigned */ true, TYP_INT); + BlockRange().InsertAfter(extractNode, castNode); + + GenTree* result = castNode; + + if (isTrailingZeroCount) + { + GenTree* one = m_compiler->gtNewIconNode(1); + BlockRange().InsertAfter(castNode, one); + + result = m_compiler->gtNewOperNode(GT_SUB, TYP_INT, castNode, one); + BlockRange().InsertAfter(one, result); + } + + BlockRange().Remove(zeroCount); + + ReplaceHWIntrinsicCmpMaskExtractMsbUse(use, parents, zeroCount, result); + + return true; +} +#endif // TARGET_ARM64 + //---------------------------------------------------------------------------------------------- // RewriteHWIntrinsicExtractMsb: Rewrites a hwintrinsic ExtractMostSignificantBytes operation // @@ -1354,6 +1879,17 @@ void Rationalizer::RewriteHWIntrinsicExtractMsb(GenTree** use, Compiler::GenTree GenTree* op1 = node->Op(1); #if defined(TARGET_ARM64) + if (RewriteHWIntrinsicCmpMaskExtractMsb(use, parents)) + { + return; + } + + if ((parents.Height() > 1) && (IsPrimitivePopCount(parents.Top(1)) || IsZeroCount(parents.Top(1))) && + IsHWIntrinsicCmpMaskExtractMsb(m_compiler, node)) + { + return; + } + // ARM64 doesn't have a single instruction that performs the behavior so we'll emulate it instead. // To do this, we effectively perform the following steps: // 1. tmp = input & 0x80 ; and the input to clear all but the most significant bit @@ -1834,10 +2370,37 @@ Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, Compiler::Ge case GT_INTRINSIC: // Non-target intrinsics should have already been rewritten back into user calls. assert(m_compiler->IsTargetIntrinsic(node->AsIntrinsic()->gtIntrinsicName)); +#if defined(TARGET_ARM64) && defined(FEATURE_HW_INTRINSICS) + if (node->AsIntrinsic()->gtIntrinsicName == NI_PRIMITIVE_PopCount) + { + if (RewriteHWIntrinsicCmpMaskExtractMsbPopCount(useEdge, parentStack)) + { + node = *useEdge; + } + } + else if ((node->AsIntrinsic()->gtIntrinsicName == NI_PRIMITIVE_TrailingZeroCount) || + (node->AsIntrinsic()->gtIntrinsicName == NI_PRIMITIVE_LeadingZeroCount)) + { + if (RewriteHWIntrinsicCmpMaskExtractMsbZeroCount(useEdge, parentStack)) + { + node = *useEdge; + } + } +#endif // TARGET_ARM64 && FEATURE_HW_INTRINSICS break; #if defined(FEATURE_HW_INTRINSICS) case GT_HWINTRINSIC: +#if defined(TARGET_ARM64) + if (IsZeroCount(node)) + { + if (RewriteHWIntrinsicCmpMaskExtractMsbZeroCount(useEdge, parentStack)) + { + node = *useEdge; + break; + } + } +#endif // TARGET_ARM64 RewriteHWIntrinsic(useEdge, parentStack); break; #endif // FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/rationalize.h b/src/coreclr/jit/rationalize.h index d0449be7c70a56..06674cc1f7ca94 100644 --- a/src/coreclr/jit/rationalize.h +++ b/src/coreclr/jit/rationalize.h @@ -64,6 +64,12 @@ class Rationalizer final : public Phase bool ShouldRewriteToNonMaskHWIntrinsic(GenTree* node); #endif // TARGET_XARCH +#if defined(TARGET_ARM64) + bool RewriteHWIntrinsicCmpMaskExtractMsb(GenTree** use, Compiler::GenTreeStack& parents); + bool RewriteHWIntrinsicCmpMaskExtractMsbPopCount(GenTree** use, Compiler::GenTreeStack& parents); + bool RewriteHWIntrinsicCmpMaskExtractMsbZeroCount(GenTree** use, Compiler::GenTreeStack& parents); +#endif // TARGET_ARM64 + void RewriteHWIntrinsicExtractMsb(GenTree** use, Compiler::GenTreeStack& parents); #endif // FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index cff6a6e66e85f1..2c60501268bbd0 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -9836,6 +9836,11 @@ bool ValueNumStore::IsVectorPerElementMask(ValueNum vn, var_types simdBaseType, bool isScalar = false; genTreeOps oper = GenTreeHWIntrinsic::GetOperForHWIntrinsicId(intrinsicId, simdBaseType, &isScalar); + if (!isScalar && GenTree::OperIsCmpCompare(oper)) + { + return genTypeSize(intrinsicSimdBaseType) >= genTypeSize(simdBaseType); + } + switch (oper) { case GT_AND: diff --git a/src/tests/JIT/opt/InstructionCombining/ExtractMostSignificantBits.cs b/src/tests/JIT/opt/InstructionCombining/ExtractMostSignificantBits.cs new file mode 100644 index 00000000000000..b9f26046adbcec --- /dev/null +++ b/src/tests/JIT/opt/InstructionCombining/ExtractMostSignificantBits.cs @@ -0,0 +1,592 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using Xunit; + +namespace TestExtractMostSignificantBits +{ + public class Program + { + [Fact] + public static int TestEntryPoint() + { + bool fail = false; + + Vector128 utf16Data = Vector128.Create( + (ushort)0x0000, (ushort)0x0800, (ushort)0x07FF, (ushort)0x8000, + (ushort)0xD7FF, (ushort)0xD800, (ushort)0x0001, (ushort)0xFFFF); + + if (LessThanUInt16Mask(utf16Data, 0x0800) != 0x45) + { + fail = true; + } + + if (GreaterThanOrEqualUInt16Mask(utf16Data, 0x0800) != 0xBA) + { + fail = true; + } + + if (!AnyLessThanUInt16(utf16Data, 0x0800)) + { + fail = true; + } + + if (NoneLessThanUInt16(Vector128.Create((ushort)0x0800), 0x0800) != true) + { + fail = true; + } + + if (CountGreaterThanOrEqualUInt16(utf16Data, 0x0800) != 5) + { + fail = true; + } + + if (IndexOfFirstGreaterThanOrEqualUInt16(utf16Data, 0x0800) != 1) + { + fail = true; + } + + if (IndexOfFirstGreaterThanOrEqualUInt16(Vector128.Zero, 0x0800) != 32) + { + fail = true; + } + + Vector128 intData = Vector128.Create(-2, 0, 7, 8); + + if (LessThanInt32Mask(intData, 7) != 0x3) + { + fail = true; + } + + if (!AnyLessThanInt32(intData, 7)) + { + fail = true; + } + + if (NoneLessThanInt32(Vector128.Create(7), 7) != true) + { + fail = true; + } + + if (CountLessThanInt32(intData, 7) != 2) + { + fail = true; + } + + if (IndexOfFirstLessThanInt32(intData, 7) != 0) + { + fail = true; + } + + if (IndexOfFirstLessThanInt32(Vector128.Create(7), 7) != 32) + { + fail = true; + } + + Vector128 byteData = Vector128.Create( + (byte)0x00, (byte)0x80, (byte)0x7F, (byte)0xFF, + (byte)0x01, (byte)0x81, (byte)0x40, (byte)0xC0, + (byte)0x02, (byte)0x82, (byte)0x20, (byte)0xA0, + (byte)0x04, (byte)0x84, (byte)0x10, (byte)0x90); + + if (GreaterThanOrEqualByteMask(byteData, 0x80) != 0xAAAA) + { + fail = true; + } + + if (!AnyGreaterThanOrEqualByte(byteData, 0x80)) + { + fail = true; + } + + if (NoneGreaterThanOrEqualByte(Vector128.Zero, 0x80) != true) + { + fail = true; + } + + if (CountGreaterThanOrEqualByte(byteData, 0x80) != 8) + { + fail = true; + } + + if (CountGreaterThanOrEqualByteViaLocal(byteData, 0x80) != 8) + { + fail = true; + } + + if (IndexOfFirstGreaterThanOrEqualByte(byteData, 0x80) != 1) + { + fail = true; + } + + if (IndexOfFirstGreaterThanOrEqualByte(Vector128.Zero, 0x80) != 32) + { + fail = true; + } + + if (LeadingZeroCountGreaterThanOrEqualByte(byteData, 0x80) != 16) + { + fail = true; + } + + if (LeadingZeroCountGreaterThanOrEqualByte(Vector128.Zero, 0x80) != 32) + { + fail = true; + } + + Vector64 utf16Data64 = Vector64.Create( + (ushort)0x0000, (ushort)0x0800, (ushort)0x07FF, (ushort)0xFFFF); + + if (LessThanUInt16Mask64(utf16Data64, 0x0800) != 0x5) + { + fail = true; + } + + if (!AnyLessThanUInt1664(utf16Data64, 0x0800)) + { + fail = true; + } + + if (NoneLessThanUInt1664(Vector64.Create((ushort)0x0800), 0x0800) != true) + { + fail = true; + } + + if (CountGreaterThanOrEqualUInt1664(utf16Data64, 0x0800) != 2) + { + fail = true; + } + + if (IndexOfFirstGreaterThanOrEqualUInt1664(utf16Data64, 0x0800) != 1) + { + fail = true; + } + + if (IndexOfFirstGreaterThanOrEqualUInt1664(Vector64.Zero, 0x0800) != 32) + { + fail = true; + } + + Vector64 intData64 = Vector64.Create(-2, 8); + + if (LessThanInt32Mask64(intData64, 7) != 0x1) + { + fail = true; + } + + if (!AnyLessThanInt3264(intData64, 7)) + { + fail = true; + } + + if (NoneLessThanInt3264(Vector64.Create(7), 7) != true) + { + fail = true; + } + + if (CountLessThanInt3264(intData64, 7) != 1) + { + fail = true; + } + + if (IndexOfFirstLessThanInt3264(intData64, 7) != 0) + { + fail = true; + } + + if (IndexOfFirstLessThanInt3264(Vector64.Create(7), 7) != 32) + { + fail = true; + } + + Vector64 byteData64 = Vector64.Create( + (byte)0x00, (byte)0x80, (byte)0x7F, (byte)0xFF, + (byte)0x01, (byte)0x81, (byte)0x40, (byte)0xC0); + + if (GreaterThanOrEqualByteMask64(byteData64, 0x80) != 0xAA) + { + fail = true; + } + + if (!AnyGreaterThanOrEqualByte64(byteData64, 0x80)) + { + fail = true; + } + + if (NoneGreaterThanOrEqualByte64(Vector64.Zero, 0x80) != true) + { + fail = true; + } + + if (CountGreaterThanOrEqualByte64(byteData64, 0x80) != 4) + { + fail = true; + } + + if (IndexOfFirstGreaterThanOrEqualByte64(byteData64, 0x80) != 1) + { + fail = true; + } + + if (IndexOfFirstGreaterThanOrEqualByte64(Vector64.Zero, 0x80) != 32) + { + fail = true; + } + + if (LeadingZeroCountGreaterThanOrEqualByte64(byteData64, 0x80) != 24) + { + fail = true; + } + + if (LeadingZeroCountGreaterThanOrEqualByte64(Vector64.Zero, 0x80) != 32) + { + fail = true; + } + + return fail ? 101 : 100; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint LessThanUInt16Mask(Vector128 value, ushort limit) + { + // ARM64-FULL-LINE: cmhi {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: and {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: addv {{h[0-9]+}}, {{v[0-9]+}}.8h + return Vector128.LessThan(value, Vector128.Create(limit)).ExtractMostSignificantBits(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint GreaterThanOrEqualUInt16Mask(Vector128 value, ushort limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: and {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: addv {{h[0-9]+}}, {{v[0-9]+}}.8h + return Vector128.GreaterThanOrEqual(value, Vector128.Create(limit)).ExtractMostSignificantBits(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool AnyLessThanUInt16(Vector128 value, ushort limit) + { + // ARM64-FULL-LINE: cmhi {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: umaxv {{h[0-9]+}}, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.h[0] + // ARM64-FULL-LINE: cset {{[wx][0-9]+}}, ne + return Vector128.LessThan(value, Vector128.Create(limit)).ExtractMostSignificantBits() != 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool NoneLessThanUInt16(Vector128 value, ushort limit) + { + // ARM64-FULL-LINE: cmhi {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: umaxv {{h[0-9]+}}, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.h[0] + // ARM64-FULL-LINE: cset {{[wx][0-9]+}}, eq + return Vector128.LessThan(value, Vector128.Create(limit)).ExtractMostSignificantBits() == 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int CountGreaterThanOrEqualUInt16(Vector128 value, ushort limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: ushr {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, #15 + // ARM64-FULL-LINE: addv {{h[0-9]+}}, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.h[0] + return BitOperations.PopCount(Vector128.GreaterThanOrEqual(value, Vector128.Create(limit)).ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int IndexOfFirstGreaterThanOrEqualUInt16(Vector128 value, ushort limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: bsl {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: uminv {{h[0-9]+}}, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.h[0] + // ARM64-FULL-LINE: sub {{w[0-9]+}}, {{w[0-9]+}}, #1 + return BitOperations.TrailingZeroCount(Vector128.GreaterThanOrEqual(value, Vector128.Create(limit)).ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint LessThanInt32Mask(Vector128 value, int limit) + { + // ARM64-FULL-LINE: cmgt {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s + // ARM64-FULL-LINE: and {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s + // ARM64-FULL-LINE: addv {{s[0-9]+}}, {{v[0-9]+}}.4s + // ARM64-FULL-LINE: smov {{x[0-9]+}}, {{v[0-9]+}}.s[0] + return Vector128.LessThan(value, Vector128.Create(limit)).ExtractMostSignificantBits(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool AnyLessThanInt32(Vector128 value, int limit) + { + // ARM64-FULL-LINE: cmgt {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s + // ARM64-FULL-LINE: umaxv {{s[0-9]+}}, {{v[0-9]+}}.4s + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.s[0] + // ARM64-FULL-LINE: cset {{[wx][0-9]+}}, ne + return Vector128.LessThan(value, Vector128.Create(limit)).ExtractMostSignificantBits() != 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool NoneLessThanInt32(Vector128 value, int limit) + { + // ARM64-FULL-LINE: cmgt {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s + // ARM64-FULL-LINE: umaxv {{s[0-9]+}}, {{v[0-9]+}}.4s + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.s[0] + // ARM64-FULL-LINE: cset {{[wx][0-9]+}}, eq + return Vector128.LessThan(value, Vector128.Create(limit)).ExtractMostSignificantBits() == 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int CountLessThanInt32(Vector128 value, int limit) + { + // ARM64-FULL-LINE: cmgt {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s + // ARM64-FULL-LINE: ushr {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, #31 + // ARM64-FULL-LINE: addv {{s[0-9]+}}, {{v[0-9]+}}.4s + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.s[0] + return BitOperations.PopCount(Vector128.LessThan(value, Vector128.Create(limit)).ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int IndexOfFirstLessThanInt32(Vector128 value, int limit) + { + // ARM64-FULL-LINE: cmgt {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s + // ARM64-FULL-LINE: bsl {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s + // ARM64-FULL-LINE: uminv {{s[0-9]+}}, {{v[0-9]+}}.4s + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.s[0] + // ARM64-FULL-LINE: sub {{w[0-9]+}}, {{w[0-9]+}}, #1 + return BitOperations.TrailingZeroCount(Vector128.LessThan(value, Vector128.Create(limit)).ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint GreaterThanOrEqualByteMask(Vector128 value, byte limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: and {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: addv {{h[0-9]+}}, {{v[0-9]+}}.8h + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.h[0] + return Vector128.GreaterThanOrEqual(value, Vector128.Create(limit)).ExtractMostSignificantBits(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool AnyGreaterThanOrEqualByte(Vector128 value, byte limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: umaxv {{b[0-9]+}}, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.b[0] + // ARM64-FULL-LINE: cset {{[wx][0-9]+}}, ne + return Vector128.GreaterThanOrEqual(value, Vector128.Create(limit)).ExtractMostSignificantBits() != 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool NoneGreaterThanOrEqualByte(Vector128 value, byte limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: umaxv {{b[0-9]+}}, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.b[0] + // ARM64-FULL-LINE: cset {{[wx][0-9]+}}, eq + return Vector128.GreaterThanOrEqual(value, Vector128.Create(limit)).ExtractMostSignificantBits() == 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int CountGreaterThanOrEqualByte(Vector128 value, byte limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: ushr {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, #7 + // ARM64-FULL-LINE: addv {{b[0-9]+}}, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.b[0] + return BitOperations.PopCount(Vector128.GreaterThanOrEqual(value, Vector128.Create(limit)).ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int CountGreaterThanOrEqualByteViaLocal(Vector128 value, byte limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: ushr {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, #7 + // ARM64-FULL-LINE: addv {{b[0-9]+}}, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.b[0] + Vector128 mask = Vector128.GreaterThanOrEqual(value, Vector128.Create(limit)); + return BitOperations.PopCount(mask.ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int IndexOfFirstGreaterThanOrEqualByte(Vector128 value, byte limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: bsl {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: uminv {{b[0-9]+}}, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.b[0] + // ARM64-FULL-LINE: sub {{w[0-9]+}}, {{w[0-9]+}}, #1 + return BitOperations.TrailingZeroCount(Vector128.GreaterThanOrEqual(value, Vector128.Create(limit)).ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int LeadingZeroCountGreaterThanOrEqualByte(Vector128 value, byte limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: bsl {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: uminv {{b[0-9]+}}, {{v[0-9]+}}.16b + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.b[0] + return BitOperations.LeadingZeroCount(Vector128.GreaterThanOrEqual(value, Vector128.Create(limit)).ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint LessThanUInt16Mask64(Vector64 value, ushort limit) + { + // ARM64-FULL-LINE: cmhi {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h + // ARM64-FULL-LINE: and {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h + return Vector64.LessThan(value, Vector64.Create(limit)).ExtractMostSignificantBits(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool AnyLessThanUInt1664(Vector64 value, ushort limit) + { + // ARM64-FULL-LINE: cmhi {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h + // ARM64-FULL-LINE: umaxv {{h[0-9]+}}, {{v[0-9]+}}.4h + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.h[0] + // ARM64-FULL-LINE: cset {{[wx][0-9]+}}, ne + return Vector64.LessThan(value, Vector64.Create(limit)).ExtractMostSignificantBits() != 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool NoneLessThanUInt1664(Vector64 value, ushort limit) + { + // ARM64-FULL-LINE: cmhi {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h + // ARM64-FULL-LINE: umaxv {{h[0-9]+}}, {{v[0-9]+}}.4h + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.h[0] + // ARM64-FULL-LINE: cset {{[wx][0-9]+}}, eq + return Vector64.LessThan(value, Vector64.Create(limit)).ExtractMostSignificantBits() == 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int CountGreaterThanOrEqualUInt1664(Vector64 value, ushort limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h + // ARM64-FULL-LINE: ushr {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, #15 + // ARM64-FULL-LINE: addv {{h[0-9]+}}, {{v[0-9]+}}.4h + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.h[0] + return BitOperations.PopCount(Vector64.GreaterThanOrEqual(value, Vector64.Create(limit)).ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int IndexOfFirstGreaterThanOrEqualUInt1664(Vector64 value, ushort limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h + // ARM64-FULL-LINE: bsl {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h + // ARM64-FULL-LINE: uminv {{h[0-9]+}}, {{v[0-9]+}}.4h + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.h[0] + // ARM64-FULL-LINE: sub {{w[0-9]+}}, {{w[0-9]+}}, #1 + return BitOperations.TrailingZeroCount(Vector64.GreaterThanOrEqual(value, Vector64.Create(limit)).ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint LessThanInt32Mask64(Vector64 value, int limit) + { + // ARM64-FULL-LINE: cmgt {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s + // ARM64-FULL-LINE: and {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s + return Vector64.LessThan(value, Vector64.Create(limit)).ExtractMostSignificantBits(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool AnyLessThanInt3264(Vector64 value, int limit) + { + // ARM64-FULL-LINE: cmgt {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s + // ARM64-FULL-LINE: umaxp {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.s[0] + // ARM64-FULL-LINE: cset {{[wx][0-9]+}}, ne + return Vector64.LessThan(value, Vector64.Create(limit)).ExtractMostSignificantBits() != 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool NoneLessThanInt3264(Vector64 value, int limit) + { + // ARM64-FULL-LINE: cmgt {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s + // ARM64-FULL-LINE: umaxp {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.s[0] + // ARM64-FULL-LINE: cset {{[wx][0-9]+}}, eq + return Vector64.LessThan(value, Vector64.Create(limit)).ExtractMostSignificantBits() == 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int CountLessThanInt3264(Vector64 value, int limit) + { + // ARM64-FULL-LINE: cmgt {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s + // ARM64-FULL-LINE: ushr {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, #31 + // ARM64-FULL-LINE: addp {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.s[0] + return BitOperations.PopCount(Vector64.LessThan(value, Vector64.Create(limit)).ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int IndexOfFirstLessThanInt3264(Vector64 value, int limit) + { + // ARM64-FULL-LINE: cmgt {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s + // ARM64-FULL-LINE: bsl {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s + // ARM64-FULL-LINE: uminp {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.s[0] + // ARM64-FULL-LINE: sub {{w[0-9]+}}, {{w[0-9]+}}, #1 + return BitOperations.TrailingZeroCount(Vector64.LessThan(value, Vector64.Create(limit)).ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint GreaterThanOrEqualByteMask64(Vector64 value, byte limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b + // ARM64-FULL-LINE: and {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b + return Vector64.GreaterThanOrEqual(value, Vector64.Create(limit)).ExtractMostSignificantBits(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool AnyGreaterThanOrEqualByte64(Vector64 value, byte limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b + // ARM64-FULL-LINE: umaxv {{b[0-9]+}}, {{v[0-9]+}}.8b + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.b[0] + // ARM64-FULL-LINE: cset {{[wx][0-9]+}}, ne + return Vector64.GreaterThanOrEqual(value, Vector64.Create(limit)).ExtractMostSignificantBits() != 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool NoneGreaterThanOrEqualByte64(Vector64 value, byte limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b + // ARM64-FULL-LINE: umaxv {{b[0-9]+}}, {{v[0-9]+}}.8b + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.b[0] + // ARM64-FULL-LINE: cset {{[wx][0-9]+}}, eq + return Vector64.GreaterThanOrEqual(value, Vector64.Create(limit)).ExtractMostSignificantBits() == 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int CountGreaterThanOrEqualByte64(Vector64 value, byte limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b + // ARM64-FULL-LINE: ushr {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, #7 + // ARM64-FULL-LINE: addv {{b[0-9]+}}, {{v[0-9]+}}.8b + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.b[0] + return BitOperations.PopCount(Vector64.GreaterThanOrEqual(value, Vector64.Create(limit)).ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int IndexOfFirstGreaterThanOrEqualByte64(Vector64 value, byte limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b + // ARM64-FULL-LINE: bsl {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b + // ARM64-FULL-LINE: uminv {{b[0-9]+}}, {{v[0-9]+}}.8b + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.b[0] + // ARM64-FULL-LINE: sub {{w[0-9]+}}, {{w[0-9]+}}, #1 + return BitOperations.TrailingZeroCount(Vector64.GreaterThanOrEqual(value, Vector64.Create(limit)).ExtractMostSignificantBits()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int LeadingZeroCountGreaterThanOrEqualByte64(Vector64 value, byte limit) + { + // ARM64-FULL-LINE: cmhs {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b + // ARM64-FULL-LINE: bsl {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b + // ARM64-FULL-LINE: uminv {{b[0-9]+}}, {{v[0-9]+}}.8b + // ARM64-FULL-LINE: umov {{w[0-9]+}}, {{v[0-9]+}}.b[0] + return BitOperations.LeadingZeroCount(Vector64.GreaterThanOrEqual(value, Vector64.Create(limit)).ExtractMostSignificantBits()); + } + } +} diff --git a/src/tests/JIT/opt/InstructionCombining/ExtractMostSignificantBits.csproj b/src/tests/JIT/opt/InstructionCombining/ExtractMostSignificantBits.csproj new file mode 100644 index 00000000000000..46f205a5ec8387 --- /dev/null +++ b/src/tests/JIT/opt/InstructionCombining/ExtractMostSignificantBits.csproj @@ -0,0 +1,17 @@ + + + + true + + + None + True + + + + true + + + + +