Skip to content

Commit 701223a

Browse files
authored
[IR] Add llvm.sincospi intrinsic (#125873)
This adds the `llvm.sincospi` intrinsic, legalization, and lowering (mostly reusing the lowering for sincos and frexp). The `llvm.sincospi` intrinsic takes a floating-point value and returns both the sine and cosine of the value multiplied by pi. It computes the result more accurately than the naive approach of doing the multiplication ahead of time, especially for large input values. ``` declare { float, float } @llvm.sincospi.f32(float %Val) declare { double, double } @llvm.sincospi.f64(double %Val) declare { x86_fp80, x86_fp80 } @llvm.sincospi.f80(x86_fp80 %Val) declare { fp128, fp128 } @llvm.sincospi.f128(fp128 %Val) declare { ppc_fp128, ppc_fp128 } @llvm.sincospi.ppcf128(ppc_fp128 %Val) declare { <4 x float>, <4 x float> } @llvm.sincospi.v4f32(<4 x float> %Val) ``` Currently, the default lowering of this intrinsic relies on the `sincospi[f|l]` functions being available in the target's runtime (e.g. libc).
1 parent 6a961dc commit 701223a

16 files changed

+490
-24
lines changed

llvm/docs/LangRef.rst

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16126,6 +16126,52 @@ of the argument.
1612616126
When specified with the fast-math-flag 'afn', the result may be approximated
1612716127
using a less accurate calculation.
1612816128

16129+
'``llvm.sincospi.*``' Intrinsic
16130+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
16131+
16132+
Syntax:
16133+
"""""""
16134+
16135+
This is an overloaded intrinsic. You can use ``llvm.sincospi`` on any
16136+
floating-point or vector of floating-point type. Not all targets support
16137+
all types however.
16138+
16139+
::
16140+
16141+
declare { float, float } @llvm.sincospi.f32(float %Val)
16142+
declare { double, double } @llvm.sincospi.f64(double %Val)
16143+
declare { x86_fp80, x86_fp80 } @llvm.sincospi.f80(x86_fp80 %Val)
16144+
declare { fp128, fp128 } @llvm.sincospi.f128(fp128 %Val)
16145+
declare { ppc_fp128, ppc_fp128 } @llvm.sincospi.ppcf128(ppc_fp128 %Val)
16146+
declare { <4 x float>, <4 x float> } @llvm.sincospi.v4f32(<4 x float> %Val)
16147+
16148+
Overview:
16149+
"""""""""
16150+
16151+
The '``llvm.sincospi.*``' intrinsics returns the sine and cosine of pi*operand.
16152+
16153+
Arguments:
16154+
""""""""""
16155+
16156+
The argument is a :ref:`floating-point <t_floating>` value or
16157+
:ref:`vector <t_vector>` of floating-point values. Returns two values matching
16158+
the argument type in a struct.
16159+
16160+
Semantics:
16161+
""""""""""
16162+
16163+
This is equivalent to the ``llvm.sincos.*`` intrinsic where the argument has been
16164+
multiplied by pi, however, it computes the result more accurately especially
16165+
for large input values.
16166+
16167+
.. note::
16168+
16169+
Currently, the default lowering of this intrinsic relies on the ``sincospi[f|l]``
16170+
functions being available in the target's runtime (e.g. libc).
16171+
16172+
When specified with the fast-math-flag 'afn', the result may be approximated
16173+
using a less accurate calculation.
16174+
1612916175
'``llvm.modf.*``' Intrinsic
1613016176
^^^^^^^^^^^^^^^^^^^^^^^^^^^
1613116177

llvm/include/llvm/CodeGen/BasicTTIImpl.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2101,6 +2101,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
21012101
case Intrinsic::sincos:
21022102
ISD = ISD::FSINCOS;
21032103
break;
2104+
case Intrinsic::sincospi:
2105+
ISD = ISD::FSINCOSPI;
2106+
break;
21042107
case Intrinsic::modf:
21052108
ISD = ISD::FMODF;
21062109
break;

llvm/include/llvm/CodeGen/ISDOpcodes.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1058,6 +1058,10 @@ enum NodeType {
10581058
/// FSINCOS - Compute both fsin and fcos as a single operation.
10591059
FSINCOS,
10601060

1061+
/// FSINCOSPI - Compute both the sine and cosine times pi more accurately
1062+
/// than FSINCOS(pi*x), especially for large x.
1063+
FSINCOSPI,
1064+
10611065
/// FMODF - Decomposes the operand into integral and fractional parts, each
10621066
/// having the same type and sign as the operand.
10631067
FMODF,

llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ Libcall getFREXP(EVT RetVT);
6666
/// UNKNOWN_LIBCALL if there is none.
6767
Libcall getFSINCOS(EVT RetVT);
6868

69+
/// getSINCOSPI - Return the SINCOSPI_* value for the given types, or
70+
/// UNKNOWN_LIBCALL if there is none.
71+
Libcall getSINCOSPI(EVT RetVT);
72+
6973
/// getMODF - Return the MODF_* value for the given types, or
7074
/// UNKNOWN_LIBCALL if there is none.
7175
Libcall getMODF(EVT RetVT);

llvm/include/llvm/IR/Intrinsics.td

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1075,6 +1075,8 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in {
10751075
def int_roundeven : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
10761076
def int_sincos : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
10771077
[llvm_anyfloat_ty]>;
1078+
def int_sincospi : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
1079+
[llvm_anyfloat_ty]>;
10781080
def int_modf : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
10791081
[llvm_anyfloat_ty]>;
10801082

llvm/include/llvm/IR/RuntimeLibcalls.def

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,11 @@ HANDLE_LIBCALL(FREXP_F64, "frexp")
354354
HANDLE_LIBCALL(FREXP_F80, "frexpl")
355355
HANDLE_LIBCALL(FREXP_F128, "frexpl")
356356
HANDLE_LIBCALL(FREXP_PPCF128, "frexpl")
357+
HANDLE_LIBCALL(SINCOSPI_F32, "sincospif")
358+
HANDLE_LIBCALL(SINCOSPI_F64, "sincospi")
359+
HANDLE_LIBCALL(SINCOSPI_F80, "sincospil")
360+
HANDLE_LIBCALL(SINCOSPI_F128, "sincospil")
361+
HANDLE_LIBCALL(SINCOSPI_PPCF128, "sincospil")
357362
HANDLE_LIBCALL(MODF_F32, "modff")
358363
HANDLE_LIBCALL(MODF_F64, "modf")
359364
HANDLE_LIBCALL(MODF_F80, "modfl")

llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4520,11 +4520,15 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
45204520
ExpandFPLibCall(Node, RTLIB::TANH_F32, RTLIB::TANH_F64, RTLIB::TANH_F80,
45214521
RTLIB::TANH_F128, RTLIB::TANH_PPCF128, Results);
45224522
break;
4523-
case ISD::FSINCOS: {
4524-
RTLIB::Libcall LC = RTLIB::getFSINCOS(Node->getValueType(0));
4523+
case ISD::FSINCOS:
4524+
case ISD::FSINCOSPI: {
4525+
EVT VT = Node->getValueType(0);
4526+
RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
4527+
? RTLIB::getFSINCOS(VT)
4528+
: RTLIB::getSINCOSPI(VT);
45254529
bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results);
45264530
if (!Expanded)
4527-
llvm_unreachable("Expected scalar FSINCOS to expand to libcall!");
4531+
llvm_unreachable("Expected scalar FSINCOS[PI] to expand to libcall!");
45284532
break;
45294533
}
45304534
case ISD::FLOG:
@@ -5507,7 +5511,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
55075511
break;
55085512
}
55095513
case ISD::FMODF:
5510-
case ISD::FSINCOS: {
5514+
case ISD::FSINCOS:
5515+
case ISD::FSINCOSPI: {
55115516
Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
55125517
Tmp2 = DAG.getNode(Node->getOpcode(), dl, DAG.getVTList(NVT, NVT), Tmp1,
55135518
Node->getFlags());

llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2768,6 +2768,7 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
27682768

27692769
case ISD::FMODF:
27702770
case ISD::FSINCOS:
2771+
case ISD::FSINCOSPI:
27712772
R = PromoteFloatRes_UnaryWithTwoFPResults(N);
27722773
break;
27732774
case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
@@ -3230,6 +3231,7 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
32303231

32313232
case ISD::FMODF:
32323233
case ISD::FSINCOS:
3234+
case ISD::FSINCOSPI:
32333235
R = SoftPromoteHalfRes_UnaryWithTwoFPResults(N);
32343236
break;
32353237

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -456,6 +456,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
456456
case ISD::FFREXP:
457457
case ISD::FMODF:
458458
case ISD::FSINCOS:
459+
case ISD::FSINCOSPI:
459460
case ISD::SADDSAT:
460461
case ISD::UADDSAT:
461462
case ISD::SSUBSAT:
@@ -1217,9 +1218,12 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
12171218
return;
12181219

12191220
break;
1220-
case ISD::FSINCOS: {
1221-
RTLIB::Libcall LC =
1222-
RTLIB::getFSINCOS(Node->getValueType(0).getVectorElementType());
1221+
case ISD::FSINCOS:
1222+
case ISD::FSINCOSPI: {
1223+
EVT VT = Node->getValueType(0).getVectorElementType();
1224+
RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
1225+
? RTLIB::getFSINCOS(VT)
1226+
: RTLIB::getSINCOSPI(VT);
12231227
if (DAG.expandMultipleResultFPLibCall(LC, Node, Results))
12241228
return;
12251229
break;

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
136136
case ISD::FMODF:
137137
case ISD::FFREXP:
138138
case ISD::FSINCOS:
139+
case ISD::FSINCOSPI:
139140
R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
140141
break;
141142
case ISD::ADD:
@@ -1265,6 +1266,7 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
12651266
case ISD::FMODF:
12661267
case ISD::FFREXP:
12671268
case ISD::FSINCOS:
1269+
case ISD::FSINCOSPI:
12681270
SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
12691271
break;
12701272

@@ -4815,7 +4817,8 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
48154817
break;
48164818
case ISD::FMODF:
48174819
case ISD::FFREXP:
4818-
case ISD::FSINCOS: {
4820+
case ISD::FSINCOS:
4821+
case ISD::FSINCOSPI: {
48194822
if (!unrollExpandedOp())
48204823
Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
48214824
break;

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6978,6 +6978,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
69786978
return;
69796979
case Intrinsic::modf:
69806980
case Intrinsic::sincos:
6981+
case Intrinsic::sincospi:
69816982
case Intrinsic::frexp: {
69826983
unsigned Opcode;
69836984
switch (Intrinsic) {
@@ -6986,6 +6987,9 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
69866987
case Intrinsic::sincos:
69876988
Opcode = ISD::FSINCOS;
69886989
break;
6990+
case Intrinsic::sincospi:
6991+
Opcode = ISD::FSINCOSPI;
6992+
break;
69896993
case Intrinsic::modf:
69906994
Opcode = ISD::FMODF;
69916995
break;

llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
216216
case ISD::FCOS: return "fcos";
217217
case ISD::STRICT_FCOS: return "strict_fcos";
218218
case ISD::FSINCOS: return "fsincos";
219+
case ISD::FSINCOSPI: return "fsincospi";
219220
case ISD::FMODF: return "fmodf";
220221
case ISD::FTAN: return "ftan";
221222
case ISD::STRICT_FTAN: return "strict_ftan";

llvm/lib/CodeGen/TargetLoweringBase.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,11 @@ RTLIB::Libcall RTLIB::getFSINCOS(EVT RetVT) {
407407
SINCOS_PPCF128);
408408
}
409409

410+
RTLIB::Libcall RTLIB::getSINCOSPI(EVT RetVT) {
411+
return getFPLibCall(RetVT, SINCOSPI_F32, SINCOSPI_F64, SINCOSPI_F80,
412+
SINCOSPI_F128, SINCOSPI_PPCF128);
413+
}
414+
410415
RTLIB::Libcall RTLIB::getMODF(EVT RetVT) {
411416
return getFPLibCall(RetVT, MODF_F32, MODF_F64, MODF_F80, MODF_F128,
412417
MODF_PPCF128);
@@ -781,7 +786,7 @@ void TargetLoweringBase::initActions() {
781786

782787
// These library functions default to expand.
783788
setOperationAction({ISD::FROUND, ISD::FPOWI, ISD::FLDEXP, ISD::FFREXP,
784-
ISD::FSINCOS, ISD::FMODF},
789+
ISD::FSINCOS, ISD::FSINCOSPI, ISD::FMODF},
785790
VT, Expand);
786791

787792
// These operations default to expand for vector types.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -735,19 +735,20 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
735735
setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Promote);
736736
}
737737

738-
for (auto Op : {ISD::FREM, ISD::FPOW, ISD::FPOWI,
739-
ISD::FCOS, ISD::FSIN, ISD::FSINCOS,
740-
ISD::FMODF, ISD::FACOS, ISD::FASIN,
741-
ISD::FATAN, ISD::FATAN2, ISD::FCOSH,
742-
ISD::FSINH, ISD::FTANH, ISD::FTAN,
743-
ISD::FEXP, ISD::FEXP2, ISD::FEXP10,
744-
ISD::FLOG, ISD::FLOG2, ISD::FLOG10,
745-
ISD::STRICT_FREM, ISD::STRICT_FPOW, ISD::STRICT_FPOWI,
746-
ISD::STRICT_FCOS, ISD::STRICT_FSIN, ISD::STRICT_FACOS,
747-
ISD::STRICT_FASIN, ISD::STRICT_FATAN, ISD::STRICT_FATAN2,
748-
ISD::STRICT_FCOSH, ISD::STRICT_FSINH, ISD::STRICT_FTANH,
749-
ISD::STRICT_FEXP, ISD::STRICT_FEXP2, ISD::STRICT_FLOG,
750-
ISD::STRICT_FLOG2, ISD::STRICT_FLOG10, ISD::STRICT_FTAN}) {
738+
for (auto Op : {ISD::FREM, ISD::FPOW, ISD::FPOWI,
739+
ISD::FCOS, ISD::FSIN, ISD::FSINCOS,
740+
ISD::FSINCOSPI, ISD::FMODF, ISD::FACOS,
741+
ISD::FASIN, ISD::FATAN, ISD::FATAN2,
742+
ISD::FCOSH, ISD::FSINH, ISD::FTANH,
743+
ISD::FTAN, ISD::FEXP, ISD::FEXP2,
744+
ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
745+
ISD::FLOG10, ISD::STRICT_FREM, ISD::STRICT_FPOW,
746+
ISD::STRICT_FPOWI, ISD::STRICT_FCOS, ISD::STRICT_FSIN,
747+
ISD::STRICT_FACOS, ISD::STRICT_FASIN, ISD::STRICT_FATAN,
748+
ISD::STRICT_FATAN2, ISD::STRICT_FCOSH, ISD::STRICT_FSINH,
749+
ISD::STRICT_FTANH, ISD::STRICT_FEXP, ISD::STRICT_FEXP2,
750+
ISD::STRICT_FLOG, ISD::STRICT_FLOG2, ISD::STRICT_FLOG10,
751+
ISD::STRICT_FTAN}) {
751752
setOperationAction(Op, MVT::f16, Promote);
752753
setOperationAction(Op, MVT::v4f16, Expand);
753754
setOperationAction(Op, MVT::v8f16, Expand);
@@ -1208,7 +1209,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
12081209
ISD::FSIN, ISD::FCOS, ISD::FTAN,
12091210
ISD::FASIN, ISD::FACOS, ISD::FATAN,
12101211
ISD::FSINH, ISD::FCOSH, ISD::FTANH,
1211-
ISD::FPOW, ISD::FLOG, ISD::FLOG2,
1212+
ISD::FPOW, ISD::FLOG, ISD::FLOG2,
12121213
ISD::FLOG10, ISD::FEXP, ISD::FEXP2,
12131214
ISD::FEXP10, ISD::FRINT, ISD::FROUND,
12141215
ISD::FROUNDEVEN, ISD::FTRUNC, ISD::FMINNUM,
@@ -1217,7 +1218,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
12171218
ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
12181219
ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FCEIL,
12191220
ISD::STRICT_FFLOOR, ISD::STRICT_FSQRT, ISD::STRICT_FRINT,
1220-
ISD::STRICT_FNEARBYINT, ISD::STRICT_FROUND, ISD::STRICT_FTRUNC,
1221+
ISD::STRICT_FNEARBYINT, ISD::STRICT_FROUND, ISD::STRICT_FTRUNC,
12211222
ISD::STRICT_FROUNDEVEN, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM,
12221223
ISD::STRICT_FMINIMUM, ISD::STRICT_FMAXIMUM})
12231224
setOperationAction(Op, MVT::v1f64, Expand);

0 commit comments

Comments
 (0)