Lines Matching refs:Value
47 static ArrayRef<int64_t> vectorShape(Value value) { in vectorShape()
62 static Value broadcast(ImplicitLocOpBuilder &builder, Value value, in broadcast()
88 static Value
91 llvm::function_ref<Value(ValueRange)> compute) { in handleMultidimensionalVectors()
112 SmallVector<Value> expandedOperands(operands); in handleMultidimensionalVectors()
136 SmallVector<Value> results(maxLinearIndex); in handleMultidimensionalVectors()
141 SmallVector<Value> extracted(expandedOperands.size()); in handleMultidimensionalVectors()
152 Value result = builder.create<arith::ConstantOp>( in handleMultidimensionalVectors()
168 static Value f32Cst(ImplicitLocOpBuilder &builder, float value) { in f32Cst()
172 static Value i32Cst(ImplicitLocOpBuilder &builder, int32_t value) { in i32Cst()
176 static Value f32FromBits(ImplicitLocOpBuilder &builder, uint32_t bits) { in f32FromBits()
177 Value i32Value = i32Cst(builder, static_cast<int32_t>(bits)); in f32FromBits()
186 static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound) { in min()
193 static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound) { in max()
200 static Value clamp(ImplicitLocOpBuilder &builder, Value value, Value lowerBound, in clamp()
201 Value upperBound) { in clamp()
207 static std::pair<Value, Value> frexp(ImplicitLocOpBuilder &builder, Value arg, in frexp()
212 auto bcast = [&](Value value) -> Value { in frexp()
220 Value cst126f = f32Cst(builder, 126.0f); in frexp()
221 Value cstHalf = f32Cst(builder, 0.5f); in frexp()
222 Value cstInvMantMask = f32FromBits(builder, ~0x7f800000u); in frexp()
225 Value i32Half = builder.create<arith::BitcastOp>(i32, cstHalf); in frexp()
226 Value i32InvMantMask = builder.create<arith::BitcastOp>(i32, cstInvMantMask); in frexp()
227 Value i32Arg = builder.create<arith::BitcastOp>(i32Vec, arg); in frexp()
230 Value tmp0 = builder.create<arith::AndIOp>(i32Arg, bcast(i32InvMantMask)); in frexp()
231 Value tmp1 = builder.create<arith::OrIOp>(tmp0, bcast(i32Half)); in frexp()
232 Value normalizedFraction = builder.create<arith::BitcastOp>(f32Vec, tmp1); in frexp()
235 Value arg0 = isPositive ? arg : builder.create<math::AbsOp>(arg); in frexp()
236 Value biasedExponentBits = builder.create<arith::ShRUIOp>( in frexp()
239 Value biasedExponent = in frexp()
241 Value exponent = in frexp()
248 static Value exp2I32(ImplicitLocOpBuilder &builder, Value arg) { in exp2I32()
252 auto bcast = [&](Value value) -> Value { in exp2I32()
262 Value biasedArg = builder.create<arith::AddIOp>(arg, bias); in exp2I32()
263 Value exp2ValueInt = in exp2I32()
265 Value exp2ValueF32 = builder.create<arith::BitcastOp>(f32Vec, exp2ValueInt); in exp2I32()
271 Value makePolynomialCalculation(ImplicitLocOpBuilder &builder, in makePolynomialCalculation()
272 llvm::ArrayRef<Value> coeffs, Value x) { in makePolynomialCalculation()
282 Value res = builder.create<math::FmaOp>(x, coeffs[coeffs.size() - 1], in makePolynomialCalculation()
323 SmallVector<Value> operands; in insertCasts()
378 Value abs = builder.create<math::AbsOp>(operand); in matchAndRewrite()
379 Value reciprocal = builder.create<arith::DivFOp>(one, abs); in matchAndRewrite()
380 Value compare = in matchAndRewrite()
382 Value x = builder.create<arith::SelectOp>(compare, abs, reciprocal); in matchAndRewrite()
391 Value p = builder.create<math::FmaOp>(x, n1, n2); in matchAndRewrite()
398 Value sub = builder.create<arith::SubFOp>(halfPi, p); in matchAndRewrite()
399 Value select = builder.create<arith::SelectOp>(compare, p, sub); in matchAndRewrite()
446 Value result = builder.create<arith::SelectOp>(xGt, atan, flippedAtan); in matchAndRewrite()
449 Value xZero = in matchAndRewrite()
451 Value yGt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, y, zero); in matchAndRewrite()
452 Value isHalfPi = builder.create<arith::AndIOp>(xZero, yGt); in matchAndRewrite()
457 Value yLt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, y, zero); in matchAndRewrite()
458 Value isNegativeHalfPiPi = builder.create<arith::AndIOp>(xZero, yLt); in matchAndRewrite()
465 Value yZero = in matchAndRewrite()
467 Value isNan = builder.create<arith::AndIOp>(xZero, yZero); in matchAndRewrite()
468 Value cstNan = broadcast(builder, f32FromBits(builder, 0x7fc00000), shape); in matchAndRewrite()
498 auto bcast = [&](Value value) -> Value { in matchAndRewrite()
503 Value minusClamp = bcast(f32Cst(builder, -7.99881172180175781f)); in matchAndRewrite()
504 Value plusClamp = bcast(f32Cst(builder, 7.99881172180175781f)); in matchAndRewrite()
505 Value x = clamp(builder, op.getOperand(), minusClamp, plusClamp); in matchAndRewrite()
508 Value tiny = bcast(f32Cst(builder, 0.0004f)); in matchAndRewrite()
509 Value tinyMask = builder.create<arith::CmpFOp>( in matchAndRewrite()
514 Value alpha1 = bcast(f32Cst(builder, 4.89352455891786e-03f)); in matchAndRewrite()
515 Value alpha3 = bcast(f32Cst(builder, 6.37261928875436e-04f)); in matchAndRewrite()
516 Value alpha5 = bcast(f32Cst(builder, 1.48572235717979e-05f)); in matchAndRewrite()
517 Value alpha7 = bcast(f32Cst(builder, 5.12229709037114e-08f)); in matchAndRewrite()
518 Value alpha9 = bcast(f32Cst(builder, -8.60467152213735e-11f)); in matchAndRewrite()
519 Value alpha11 = bcast(f32Cst(builder, 2.00018790482477e-13f)); in matchAndRewrite()
520 Value alpha13 = bcast(f32Cst(builder, -2.76076847742355e-16f)); in matchAndRewrite()
523 Value beta0 = bcast(f32Cst(builder, 4.89352518554385e-03f)); in matchAndRewrite()
524 Value beta2 = bcast(f32Cst(builder, 2.26843463243900e-03f)); in matchAndRewrite()
525 Value beta4 = bcast(f32Cst(builder, 1.18534705686654e-04f)); in matchAndRewrite()
526 Value beta6 = bcast(f32Cst(builder, 1.19825839466702e-06f)); in matchAndRewrite()
529 Value x2 = builder.create<arith::MulFOp>(x, x); in matchAndRewrite()
532 Value p = builder.create<math::FmaOp>(x2, alpha13, alpha11); in matchAndRewrite()
541 Value q = builder.create<math::FmaOp>(x2, beta6, beta4); in matchAndRewrite()
546 Value res = builder.create<arith::SelectOp>( in matchAndRewrite()
586 auto bcast = [&](Value value) -> Value { in logMatchAndRewrite()
590 Value cstZero = bcast(f32Cst(builder, 0.0f)); in logMatchAndRewrite()
591 Value cstOne = bcast(f32Cst(builder, 1.0f)); in logMatchAndRewrite()
592 Value cstNegHalf = bcast(f32Cst(builder, -0.5f)); in logMatchAndRewrite()
595 Value cstMinNormPos = bcast(f32FromBits(builder, 0x00800000u)); in logMatchAndRewrite()
596 Value cstMinusInf = bcast(f32FromBits(builder, 0xff800000u)); in logMatchAndRewrite()
597 Value cstPosInf = bcast(f32FromBits(builder, 0x7f800000u)); in logMatchAndRewrite()
598 Value cstNan = bcast(f32FromBits(builder, 0x7fc00000)); in logMatchAndRewrite()
601 Value cstCephesSQRTHF = bcast(f32Cst(builder, 0.707106781186547524f)); in logMatchAndRewrite()
602 Value cstCephesLogP0 = bcast(f32Cst(builder, 7.0376836292E-2f)); in logMatchAndRewrite()
603 Value cstCephesLogP1 = bcast(f32Cst(builder, -1.1514610310E-1f)); in logMatchAndRewrite()
604 Value cstCephesLogP2 = bcast(f32Cst(builder, 1.1676998740E-1f)); in logMatchAndRewrite()
605 Value cstCephesLogP3 = bcast(f32Cst(builder, -1.2420140846E-1f)); in logMatchAndRewrite()
606 Value cstCephesLogP4 = bcast(f32Cst(builder, +1.4249322787E-1f)); in logMatchAndRewrite()
607 Value cstCephesLogP5 = bcast(f32Cst(builder, -1.6668057665E-1f)); in logMatchAndRewrite()
608 Value cstCephesLogP6 = bcast(f32Cst(builder, +2.0000714765E-1f)); in logMatchAndRewrite()
609 Value cstCephesLogP7 = bcast(f32Cst(builder, -2.4999993993E-1f)); in logMatchAndRewrite()
610 Value cstCephesLogP8 = bcast(f32Cst(builder, +3.3333331174E-1f)); in logMatchAndRewrite()
612 Value x = op.getOperand(); in logMatchAndRewrite()
618 std::pair<Value, Value> pair = frexp(builder, x, /*isPositive=*/true); in logMatchAndRewrite()
620 Value e = pair.second; in logMatchAndRewrite()
630 Value mask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x, in logMatchAndRewrite()
632 Value tmp = builder.create<arith::SelectOp>(mask, x, cstZero); in logMatchAndRewrite()
639 Value x2 = builder.create<arith::MulFOp>(x, x); in logMatchAndRewrite()
640 Value x3 = builder.create<arith::MulFOp>(x2, x); in logMatchAndRewrite()
643 Value y0, y1, y2; in logMatchAndRewrite()
658 Value cstLog2e = bcast(f32Cst(builder, static_cast<float>(LOG2E_VALUE))); in logMatchAndRewrite()
661 Value cstLn2 = bcast(f32Cst(builder, static_cast<float>(LN2_VALUE))); in logMatchAndRewrite()
665 Value invalidMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::ULT, in logMatchAndRewrite()
667 Value zeroMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, in logMatchAndRewrite()
669 Value posInfMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, in logMatchAndRewrite()
676 Value aproximation = builder.create<arith::SelectOp>( in logMatchAndRewrite()
733 auto bcast = [&](Value value) -> Value { in matchAndRewrite()
743 Value cstOne = bcast(f32Cst(builder, 1.0f)); in matchAndRewrite()
744 Value x = op.getOperand(); in matchAndRewrite()
745 Value u = builder.create<arith::AddFOp>(x, cstOne); in matchAndRewrite()
746 Value uSmall = in matchAndRewrite()
748 Value logU = builder.create<math::LogOp>(u); in matchAndRewrite()
749 Value uInf = in matchAndRewrite()
751 Value logLarge = builder.create<arith::MulFOp>( in matchAndRewrite()
754 Value approximation = builder.create<arith::SelectOp>( in matchAndRewrite()
780 auto bcast = [&](Value value) -> Value { in matchAndRewrite()
787 Value zero = bcast(f32Cst(builder, 0)); in matchAndRewrite()
788 Value one = bcast(f32Cst(builder, 1)); in matchAndRewrite()
789 Value pp[intervalsCount][polyDegree + 1]; in matchAndRewrite()
806 Value qq[intervalsCount][polyDegree + 1]; in matchAndRewrite()
823 Value offsets[intervalsCount]; in matchAndRewrite()
828 Value bounds[intervalsCount]; in matchAndRewrite()
833 Value isNegativeArg = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, in matchAndRewrite()
835 Value negArg = builder.create<arith::NegFOp>(op.getOperand()); in matchAndRewrite()
836 Value x = in matchAndRewrite()
839 Value offset = offsets[0]; in matchAndRewrite()
840 Value p[polyDegree + 1]; in matchAndRewrite()
841 Value q[polyDegree + 1]; in matchAndRewrite()
848 Value isLessThanBound[intervalsCount]; in matchAndRewrite()
864 Value pPoly = makePolynomialCalculation(builder, p, x); in matchAndRewrite()
865 Value qPoly = makePolynomialCalculation(builder, q, x); in matchAndRewrite()
866 Value rationalPoly = builder.create<arith::DivFOp>(pPoly, qPoly); in matchAndRewrite()
867 Value formula = builder.create<arith::AddFOp>(offset, rationalPoly); in matchAndRewrite()
872 Value negFormula = builder.create<arith::NegFOp>(formula); in matchAndRewrite()
873 Value res = in matchAndRewrite()
911 auto bcast = [&](Value value) -> Value { in matchAndRewrite()
914 auto fmla = [&](Value a, Value b, Value c) { in matchAndRewrite()
917 auto mul = [&](Value a, Value b) -> Value { in matchAndRewrite()
920 auto sub = [&](Value a, Value b) -> Value { in matchAndRewrite()
923 auto floor = [&](Value a) { return builder.create<math::FloorOp>(a); }; in matchAndRewrite()
925 Value cstLn2 = bcast(f32Cst(builder, static_cast<float>(LN2_VALUE))); in matchAndRewrite()
926 Value cstLog2E = bcast(f32Cst(builder, static_cast<float>(LOG2E_VALUE))); in matchAndRewrite()
929 Value cstCephesExpP0 = bcast(f32Cst(builder, 1.0)); in matchAndRewrite()
930 Value cstCephesExpP1 = bcast(f32Cst(builder, 1.0)); in matchAndRewrite()
931 Value cstCephesExpP2 = bcast(f32Cst(builder, 0.49970514590562437052f)); in matchAndRewrite()
932 Value cstCephesExpP3 = bcast(f32Cst(builder, 0.16873890085469545053f)); in matchAndRewrite()
933 Value cstCephesExpP4 = bcast(f32Cst(builder, 0.03668965196652099192f)); in matchAndRewrite()
934 Value cstCephesExpP5 = bcast(f32Cst(builder, 0.01314350012789660196f)); in matchAndRewrite()
936 Value x = op.getOperand(); in matchAndRewrite()
938 Value isNan = builder.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, x, x); in matchAndRewrite()
941 Value xL2Inv = mul(x, cstLog2E); in matchAndRewrite()
942 Value kF32 = floor(xL2Inv); in matchAndRewrite()
943 Value kLn2 = mul(kF32, cstLn2); in matchAndRewrite()
944 Value y = sub(x, kLn2); in matchAndRewrite()
948 Value y2 = mul(y, y); in matchAndRewrite()
949 Value y4 = mul(y2, y2); in matchAndRewrite()
951 Value q0 = fmla(cstCephesExpP1, y, cstCephesExpP0); in matchAndRewrite()
952 Value q1 = fmla(cstCephesExpP3, y, cstCephesExpP2); in matchAndRewrite()
953 Value q2 = fmla(cstCephesExpP5, y, cstCephesExpP4); in matchAndRewrite()
954 Value expY = fmla(q1, y2, q0); in matchAndRewrite()
960 Value k = builder.create<arith::FPToSIOp>(i32Vec, kF32); in matchAndRewrite()
961 Value exp2KValue = exp2I32(builder, k); in matchAndRewrite()
972 Value zerof32Const = bcast(f32Cst(builder, 0)); in matchAndRewrite()
979 Value kMaxConst = bcast(i32Cst(builder, 127)); in matchAndRewrite()
980 Value kMaxNegConst = bcast(i32Cst(builder, -127)); in matchAndRewrite()
981 Value rightBound = in matchAndRewrite()
983 Value leftBound = in matchAndRewrite()
986 Value isNegInfinityX = builder.create<arith::CmpFOp>( in matchAndRewrite()
988 Value isPosInfinityX = builder.create<arith::CmpFOp>( in matchAndRewrite()
990 Value isPostiveX = in matchAndRewrite()
992 Value isComputable = builder.create<arith::AndIOp>(rightBound, leftBound); in matchAndRewrite()
1034 auto bcast = [&](Value value) -> Value { in matchAndRewrite()
1041 Value cstOne = bcast(f32Cst(builder, 1.0f)); in matchAndRewrite()
1042 Value cstNegOne = bcast(f32Cst(builder, -1.0f)); in matchAndRewrite()
1043 Value x = op.getOperand(); in matchAndRewrite()
1044 Value u = builder.create<math::ExpOp>(x); in matchAndRewrite()
1045 Value uEqOneOrNaN = in matchAndRewrite()
1047 Value uMinusOne = builder.create<arith::SubFOp>(u, cstOne); in matchAndRewrite()
1048 Value uMinusOneEqNegOne = builder.create<arith::CmpFOp>( in matchAndRewrite()
1051 Value logU = builder.create<math::LogOp>(u); in matchAndRewrite()
1054 Value isInf = in matchAndRewrite()
1058 Value expm1 = builder.create<arith::MulFOp>( in matchAndRewrite()
1061 Value approximation = builder.create<arith::SelectOp>( in matchAndRewrite()
1104 auto bcast = [&](Value value) -> Value { in matchAndRewrite()
1107 auto mul = [&](Value a, Value b) -> Value { in matchAndRewrite()
1110 auto sub = [&](Value a, Value b) -> Value { in matchAndRewrite()
1113 auto floor = [&](Value a) { return builder.create<math::FloorOp>(a); }; in matchAndRewrite()
1116 auto fPToSingedInteger = [&](Value a) -> Value { in matchAndRewrite()
1120 auto modulo4 = [&](Value a) -> Value { in matchAndRewrite()
1124 auto isEqualTo = [&](Value a, Value b) -> Value { in matchAndRewrite()
1128 auto isGreaterThan = [&](Value a, Value b) -> Value { in matchAndRewrite()
1132 auto select = [&](Value cond, Value t, Value f) -> Value { in matchAndRewrite()
1136 auto fmla = [&](Value a, Value b, Value c) { in matchAndRewrite()
1140 auto bitwiseOr = [&](Value a, Value b) { in matchAndRewrite()
1144 Value twoOverPi = bcast(f32Cst(builder, (float)TWO_OVER_PI)); in matchAndRewrite()
1145 Value piOverTwo = bcast(f32Cst(builder, (float)PI_OVER_2)); in matchAndRewrite()
1147 Value x = op.getOperand(); in matchAndRewrite()
1149 Value k = floor(mul(x, twoOverPi)); in matchAndRewrite()
1151 Value y = sub(x, mul(k, piOverTwo)); in matchAndRewrite()
1153 Value cstOne = bcast(f32Cst(builder, 1.0)); in matchAndRewrite()
1154 Value cstNegativeOne = bcast(f32Cst(builder, -1.0)); in matchAndRewrite()
1156 Value cstSC2 = bcast(f32Cst(builder, -0.16666667163372039794921875f)); in matchAndRewrite()
1157 Value cstSC4 = bcast(f32Cst(builder, 8.333347737789154052734375e-3f)); in matchAndRewrite()
1158 Value cstSC6 = bcast(f32Cst(builder, -1.9842604524455964565277099609375e-4f)); in matchAndRewrite()
1159 Value cstSC8 = in matchAndRewrite()
1161 Value cstSC10 = in matchAndRewrite()
1164 Value cstCC2 = bcast(f32Cst(builder, -0.5f)); in matchAndRewrite()
1165 Value cstCC4 = bcast(f32Cst(builder, 4.166664183139801025390625e-2f)); in matchAndRewrite()
1166 Value cstCC6 = bcast(f32Cst(builder, -1.388833043165504932403564453125e-3f)); in matchAndRewrite()
1167 Value cstCC8 = bcast(f32Cst(builder, 2.47562347794882953166961669921875e-5f)); in matchAndRewrite()
1168 Value cstCC10 = in matchAndRewrite()
1171 Value kMod4 = modulo4(fPToSingedInteger(k)); in matchAndRewrite()
1173 Value kR0 = isEqualTo(kMod4, bcast(i32Cst(builder, 0))); in matchAndRewrite()
1174 Value kR1 = isEqualTo(kMod4, bcast(i32Cst(builder, 1))); in matchAndRewrite()
1175 Value kR2 = isEqualTo(kMod4, bcast(i32Cst(builder, 2))); in matchAndRewrite()
1176 Value kR3 = isEqualTo(kMod4, bcast(i32Cst(builder, 3))); in matchAndRewrite()
1178 Value sinuseCos = isSine ? bitwiseOr(kR1, kR3) : bitwiseOr(kR0, kR2); in matchAndRewrite()
1179 Value negativeRange = isSine ? isGreaterThan(kMod4, bcast(i32Cst(builder, 1))) in matchAndRewrite()
1182 Value y2 = mul(y, y); in matchAndRewrite()
1184 Value base = select(sinuseCos, cstOne, y); in matchAndRewrite()
1185 Value cstC2 = select(sinuseCos, cstCC2, cstSC2); in matchAndRewrite()
1186 Value cstC4 = select(sinuseCos, cstCC4, cstSC4); in matchAndRewrite()
1187 Value cstC6 = select(sinuseCos, cstCC6, cstSC6); in matchAndRewrite()
1188 Value cstC8 = select(sinuseCos, cstCC8, cstSC8); in matchAndRewrite()
1189 Value cstC10 = select(sinuseCos, cstCC10, cstSC10); in matchAndRewrite()
1191 Value v1 = fmla(y2, cstC10, cstC8); in matchAndRewrite()
1192 Value v2 = fmla(y2, v1, cstC6); in matchAndRewrite()
1193 Value v3 = fmla(y2, v2, cstC4); in matchAndRewrite()
1194 Value v4 = fmla(y2, v3, cstC2); in matchAndRewrite()
1195 Value v5 = fmla(y2, v4, cstOne); in matchAndRewrite()
1196 Value v6 = mul(base, v5); in matchAndRewrite()
1198 Value approximation = select(negativeRange, mul(cstNegativeOne, v6), v6); in matchAndRewrite()
1231 auto bcast = [&](Value value) -> Value { in matchAndRewrite()
1235 Value cstPosInf = bcast(f32FromBits(builder, 0x7f800000u)); in matchAndRewrite()
1236 Value cstOnePointFive = bcast(f32Cst(builder, 1.5f)); in matchAndRewrite()
1237 Value cstNegHalf = bcast(f32Cst(builder, -0.5f)); in matchAndRewrite()
1238 Value cstMinNormPos = bcast(f32FromBits(builder, 0x00800000u)); in matchAndRewrite()
1240 Value negHalf = builder.create<arith::MulFOp>(op.getOperand(), cstNegHalf); in matchAndRewrite()
1244 Value ltMinMask = builder.create<arith::CmpFOp>( in matchAndRewrite()
1246 Value infMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, in matchAndRewrite()
1248 Value notNormalFiniteMask = builder.create<arith::OrIOp>(ltMinMask, infMask); in matchAndRewrite()
1251 Value yApprox = handleMultidimensionalVectors( in matchAndRewrite()
1252 builder, op->getOperands(), 8, [&builder](ValueRange operands) -> Value { in matchAndRewrite()
1260 Value inner = builder.create<arith::MulFOp>(negHalf, yApprox); in matchAndRewrite()
1261 Value fma = builder.create<math::FmaOp>(yApprox, inner, cstOnePointFive); in matchAndRewrite()
1262 Value yNewton = builder.create<arith::MulFOp>(yApprox, fma); in matchAndRewrite()
1269 Value res = in matchAndRewrite()