Skip to content

Commit ec1016f

Browse files
fhahnfhahn
authored
[IVDescriptors] Support reductions with minimumnum/maximumnum. (#137335)
Add a new reduction recurrence kind for reductions with minimumnum/maximumnum. Such reductions can be vectorized without nsz/nnans, same as reductions with maximum/minimum intrinsics. Note that a new reduction kind is needed to make sure partial reductions are also combined with minimumnum/maximumnum. Note that the final reduction to a scalar value is performed with vector.reduce.fmin/fmax. This should be fine, as the results of the partial reductions with maximumnum/minimumnum silences any sNaNs. In-loop and reductions in SLP are not supported yet, as there's no reduction version of maximumnum/minimumnum yet and fmax may be incorrect. PR: #137335
1 parent cf17ee1 commit ec1016f

File tree

6 files changed

+249
-34
lines changed

6 files changed

+249
-34
lines changed

‎llvm/include/llvm/Analysis/IVDescriptors.h

+4-1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ enum class RecurKind {
4747
FMax, ///< FP max implemented in terms of select(cmp()).
4848
FMinimum, ///< FP min with llvm.minimum semantics
4949
FMaximum, ///< FP max with llvm.maximum semantics
50+
FMinimumNum, ///< FP min with llvm.minimumnum semantics
51+
FMaximumNum, ///< FP max with llvm.maximumnum semantics
5052
FMulAdd, ///< Sum of float products with llvm.fmuladd(a * b + sum).
5153
IAnyOf, ///< Any_of reduction with select(icmp(),x,y) where one of (x,y) is
5254
///< loop invariant, and both x and y are integer type.
@@ -239,7 +241,8 @@ class RecurrenceDescriptor {
239241
/// Returns true if the recurrence kind is a floating-point min/max kind.
240242
static bool isFPMinMaxRecurrenceKind(RecurKind Kind) {
241243
return Kind == RecurKind::FMin || Kind == RecurKind::FMax ||
242-
Kind == RecurKind::FMinimum || Kind == RecurKind::FMaximum;
244+
Kind == RecurKind::FMinimum || Kind == RecurKind::FMaximum ||
245+
Kind == RecurKind::FMinimumNum || Kind == RecurKind::FMaximumNum;
243246
}
244247

245248
/// Returns true if the recurrence kind is any min/max kind.

‎llvm/lib/Analysis/IVDescriptors.cpp

+26-3
Original file line numberDiff line numberDiff line change
@@ -788,6 +788,10 @@ RecurrenceDescriptor::isMinMaxPattern(Instruction *I, RecurKind Kind,
788788
return InstDesc(Kind == RecurKind::FMin, I);
789789
if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
790790
return InstDesc(Kind == RecurKind::FMax, I);
791+
if (match(I, m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())))
792+
return InstDesc(Kind == RecurKind::FMinimumNum, I);
793+
if (match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value())))
794+
return InstDesc(Kind == RecurKind::FMaximumNum, I);
791795
if (match(I, m_Intrinsic<Intrinsic::minimum>(m_Value(), m_Value())))
792796
return InstDesc(Kind == RecurKind::FMinimum, I);
793797
if (match(I, m_Intrinsic<Intrinsic::maximum>(m_Value(), m_Value())))
@@ -892,10 +896,14 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
892896
return true;
893897
if (isa<FPMathOperator>(I) && I->hasNoNaNs() && I->hasNoSignedZeros())
894898
return true;
895-
// minimum and maximum intrinsics do not require nsz and nnan flags since
896-
// NaN and signed zeroes are propagated in the intrinsic implementation.
899+
// minimum/minnum and maximum/maxnum intrinsics do not require nsz and nnan
900+
// flags since NaN and signed zeroes are propagated in the intrinsic
901+
// implementation.
897902
return match(I, m_Intrinsic<Intrinsic::minimum>(m_Value(), m_Value())) ||
898-
match(I, m_Intrinsic<Intrinsic::maximum>(m_Value(), m_Value()));
903+
match(I, m_Intrinsic<Intrinsic::maximum>(m_Value(), m_Value())) ||
904+
match(I,
905+
m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
906+
match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
899907
};
900908
if (isIntMinMaxRecurrenceKind(Kind) ||
901909
(HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
@@ -1035,6 +1043,19 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
10351043
LLVM_DEBUG(dbgs() << "Found a float MINIMUM reduction PHI." << *Phi << "\n");
10361044
return true;
10371045
}
1046+
if (AddReductionVar(Phi, RecurKind::FMaximumNum, TheLoop, FMF, RedDes, DB, AC,
1047+
DT, SE)) {
1048+
LLVM_DEBUG(dbgs() << "Found a float MAXIMUMNUM reduction PHI." << *Phi
1049+
<< "\n");
1050+
return true;
1051+
}
1052+
if (AddReductionVar(Phi, RecurKind::FMinimumNum, TheLoop, FMF, RedDes, DB, AC,
1053+
DT, SE)) {
1054+
LLVM_DEBUG(dbgs() << "Found a float MINIMUMNUM reduction PHI." << *Phi
1055+
<< "\n");
1056+
return true;
1057+
}
1058+
10381059
// Not a reduction of known type.
10391060
return false;
10401061
}
@@ -1155,6 +1176,8 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
11551176
case RecurKind::FMin:
11561177
case RecurKind::FMaximum:
11571178
case RecurKind::FMinimum:
1179+
case RecurKind::FMaximumNum:
1180+
case RecurKind::FMinimumNum:
11581181
case RecurKind::FAnyOf:
11591182
case RecurKind::FFindLastIV:
11601183
return Instruction::FCmp;

‎llvm/lib/Transforms/Utils/LoopUtils.cpp

+12-1
Original file line numberDiff line numberDiff line change
@@ -958,6 +958,10 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
958958
return Intrinsic::vector_reduce_fmaximum;
959959
case RecurKind::FMinimum:
960960
return Intrinsic::vector_reduce_fminimum;
961+
case RecurKind::FMaximumNum:
962+
return Intrinsic::vector_reduce_fmax;
963+
case RecurKind::FMinimumNum:
964+
return Intrinsic::vector_reduce_fmin;
961965
}
962966
}
963967

@@ -1053,6 +1057,10 @@ Intrinsic::ID llvm::getMinMaxReductionIntrinsicOp(RecurKind RK) {
10531057
return Intrinsic::minimum;
10541058
case RecurKind::FMaximum:
10551059
return Intrinsic::maximum;
1060+
case RecurKind::FMinimumNum:
1061+
return Intrinsic::minimumnum;
1062+
case RecurKind::FMaximumNum:
1063+
return Intrinsic::maximumnum;
10561064
}
10571065
}
10581066

@@ -1101,7 +1109,8 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
11011109
Value *Right) {
11021110
Type *Ty = Left->getType();
11031111
if (Ty->isIntOrIntVectorTy() ||
1104-
(RK == RecurKind::FMinimum || RK == RecurKind::FMaximum)) {
1112+
(RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
1113+
RK == RecurKind::FMinimumNum || RK == RecurKind::FMaximumNum)) {
11051114
// TODO: Add float minnum/maxnum support when FMF nnan is set.
11061115
Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RK);
11071116
return Builder.CreateIntrinsic(Ty, Id, {Left, Right}, nullptr,
@@ -1320,6 +1329,8 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
13201329
case RecurKind::FMin:
13211330
case RecurKind::FMinimum:
13221331
case RecurKind::FMaximum:
1332+
case RecurKind::FMinimumNum:
1333+
case RecurKind::FMaximumNum:
13231334
return Builder.CreateUnaryIntrinsic(getReductionIntrinsicID(RdxKind), Src);
13241335
case RecurKind::FMulAdd:
13251336
case RecurKind::FAdd:

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

+9-1
Original file line numberDiff line numberDiff line change
@@ -21774,7 +21774,9 @@ class HorizontalReduction {
2177421774
case RecurKind::FMax:
2177521775
case RecurKind::FMin:
2177621776
case RecurKind::FMaximum:
21777-
case RecurKind::FMinimum: {
21777+
case RecurKind::FMinimum:
21778+
case RecurKind::FMaximumNum:
21779+
case RecurKind::FMinimumNum: {
2177821780
Intrinsic::ID Id = llvm::getMinMaxReductionIntrinsicOp(Kind);
2177921781
return Builder.CreateBinaryIntrinsic(Id, LHS, RHS);
2178021782
}
@@ -23086,6 +23088,8 @@ class HorizontalReduction {
2308623088
case RecurKind::FAnyOf:
2308723089
case RecurKind::IFindLastIV:
2308823090
case RecurKind::FFindLastIV:
23091+
case RecurKind::FMaximumNum:
23092+
case RecurKind::FMinimumNum:
2308923093
case RecurKind::None:
2309023094
llvm_unreachable("Unexpected reduction kind for repeated scalar.");
2309123095
}
@@ -23220,6 +23224,8 @@ class HorizontalReduction {
2322023224
case RecurKind::FAnyOf:
2322123225
case RecurKind::IFindLastIV:
2322223226
case RecurKind::FFindLastIV:
23227+
case RecurKind::FMaximumNum:
23228+
case RecurKind::FMinimumNum:
2322323229
case RecurKind::None:
2322423230
llvm_unreachable("Unexpected reduction kind for repeated scalar.");
2322523231
}
@@ -23319,6 +23325,8 @@ class HorizontalReduction {
2331923325
case RecurKind::FAnyOf:
2332023326
case RecurKind::IFindLastIV:
2332123327
case RecurKind::FFindLastIV:
23328+
case RecurKind::FMaximumNum:
23329+
case RecurKind::FMinimumNum:
2332223330
case RecurKind::None:
2332323331
llvm_unreachable("Unexpected reduction kind for reused scalars.");
2332423332
}

0 commit comments

Comments
 (0)