Lines Matching refs:builder
62 static Value broadcast(ImplicitLocOpBuilder &builder, Value value, in broadcast() argument
66 return !shape.empty() ? builder.create<BroadcastOp>(type, value) : value; in broadcast()
89 handleMultidimensionalVectors(ImplicitLocOpBuilder &builder, in handleMultidimensionalVectors() argument
124 builder.create<vector::ShapeCastOp>(expandedType, operand); in handleMultidimensionalVectors()
144 builder.create<vector::ExtractOp>(tuple.value(), offsets); in handleMultidimensionalVectors()
152 Value result = builder.create<arith::ConstantOp>( in handleMultidimensionalVectors()
153 resultExpandedType, builder.getZeroAttr(resultExpandedType)); in handleMultidimensionalVectors()
156 result = builder.create<vector::InsertOp>(results[i], result, in handleMultidimensionalVectors()
160 return builder.create<vector::ShapeCastOp>( in handleMultidimensionalVectors()
168 static Value f32Cst(ImplicitLocOpBuilder &builder, float value) { in f32Cst() argument
169 return builder.create<arith::ConstantOp>(builder.getF32FloatAttr(value)); in f32Cst()
172 static Value i32Cst(ImplicitLocOpBuilder &builder, int32_t value) { in i32Cst() argument
173 return builder.create<arith::ConstantOp>(builder.getI32IntegerAttr(value)); in i32Cst()
176 static Value f32FromBits(ImplicitLocOpBuilder &builder, uint32_t bits) { in f32FromBits() argument
177 Value i32Value = i32Cst(builder, static_cast<int32_t>(bits)); in f32FromBits()
178 return builder.create<arith::BitcastOp>(builder.getF32Type(), i32Value); in f32FromBits()
186 static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound) { in min() argument
187 return builder.create<arith::SelectOp>( in min()
188 builder.create<arith::CmpFOp>(arith::CmpFPredicate::ULT, value, bound), in min()
193 static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound) { in max() argument
194 return builder.create<arith::SelectOp>( in max()
195 builder.create<arith::CmpFOp>(arith::CmpFPredicate::UGT, value, bound), in max()
200 static Value clamp(ImplicitLocOpBuilder &builder, Value value, Value lowerBound, in clamp() argument
202 return max(builder, min(builder, value, upperBound), lowerBound); in clamp()
207 static std::pair<Value, Value> frexp(ImplicitLocOpBuilder &builder, Value arg, in frexp() argument
213 return broadcast(builder, value, shape); in frexp()
216 auto i32 = builder.getIntegerType(32); in frexp()
218 auto f32Vec = broadcast(builder.getF32Type(), shape); in frexp()
220 Value cst126f = f32Cst(builder, 126.0f); in frexp()
221 Value cstHalf = f32Cst(builder, 0.5f); in frexp()
222 Value cstInvMantMask = f32FromBits(builder, ~0x7f800000u); in frexp()
225 Value i32Half = builder.create<arith::BitcastOp>(i32, cstHalf); in frexp()
226 Value i32InvMantMask = builder.create<arith::BitcastOp>(i32, cstInvMantMask); in frexp()
227 Value i32Arg = builder.create<arith::BitcastOp>(i32Vec, arg); in frexp()
230 Value tmp0 = builder.create<arith::AndIOp>(i32Arg, bcast(i32InvMantMask)); in frexp()
231 Value tmp1 = builder.create<arith::OrIOp>(tmp0, bcast(i32Half)); in frexp()
232 Value normalizedFraction = builder.create<arith::BitcastOp>(f32Vec, tmp1); in frexp()
235 Value arg0 = isPositive ? arg : builder.create<math::AbsOp>(arg); in frexp()
236 Value biasedExponentBits = builder.create<arith::ShRUIOp>( in frexp()
237 builder.create<arith::BitcastOp>(i32Vec, arg0), in frexp()
238 bcast(i32Cst(builder, 23))); in frexp()
240 builder.create<arith::SIToFPOp>(f32Vec, biasedExponentBits); in frexp()
242 builder.create<arith::SubFOp>(biasedExponent, bcast(cst126f)); in frexp()
248 static Value exp2I32(ImplicitLocOpBuilder &builder, Value arg) { in exp2I32() argument
253 return broadcast(builder, value, shape); in exp2I32()
256 auto f32Vec = broadcast(builder.getF32Type(), shape); in exp2I32()
258 auto exponetBitLocation = bcast(i32Cst(builder, 23)); in exp2I32()
260 auto bias = bcast(i32Cst(builder, 127)); in exp2I32()
262 Value biasedArg = builder.create<arith::AddIOp>(arg, bias); in exp2I32()
264 builder.create<arith::ShLIOp>(biasedArg, exponetBitLocation); in exp2I32()
265 Value exp2ValueF32 = builder.create<arith::BitcastOp>(f32Vec, exp2ValueInt); in exp2I32()
271 Value makePolynomialCalculation(ImplicitLocOpBuilder &builder, in makePolynomialCalculation() argument
277 return broadcast(builder, f32Cst(builder, 0.0f), shape); in makePolynomialCalculation()
282 Value res = builder.create<math::FmaOp>(x, coeffs[coeffs.size() - 1], in makePolynomialCalculation()
285 res = builder.create<math::FmaOp>(x, res, coeffs[i]); in makePolynomialCalculation()
373 ImplicitLocOpBuilder builder(op->getLoc(), rewriter); in matchAndRewrite() local
374 auto one = broadcast(builder, f32Cst(builder, 1.0f), shape); in matchAndRewrite()
378 Value abs = builder.create<math::AbsOp>(operand); in matchAndRewrite()
379 Value reciprocal = builder.create<arith::DivFOp>(one, abs); in matchAndRewrite()
381 builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, abs, reciprocal); in matchAndRewrite()
382 Value x = builder.create<arith::SelectOp>(compare, abs, reciprocal); in matchAndRewrite()
386 auto n1 = broadcast(builder, f32Cst(builder, 0.14418283f), shape); in matchAndRewrite()
387 auto n2 = broadcast(builder, f32Cst(builder, -0.34999234f), shape); in matchAndRewrite()
388 auto n3 = broadcast(builder, f32Cst(builder, -0.01067831f), shape); in matchAndRewrite()
389 auto n4 = broadcast(builder, f32Cst(builder, 1.00209986f), shape); in matchAndRewrite()
391 Value p = builder.create<math::FmaOp>(x, n1, n2); in matchAndRewrite()
392 p = builder.create<math::FmaOp>(x, p, n3); in matchAndRewrite()
393 p = builder.create<math::FmaOp>(x, p, n4); in matchAndRewrite()
394 p = builder.create<arith::MulFOp>(x, p); in matchAndRewrite()
397 auto halfPi = broadcast(builder, f32Cst(builder, 1.57079632679f), shape); in matchAndRewrite()
398 Value sub = builder.create<arith::SubFOp>(halfPi, p); in matchAndRewrite()
399 Value select = builder.create<arith::SelectOp>(compare, p, sub); in matchAndRewrite()
428 ImplicitLocOpBuilder builder(op->getLoc(), rewriter); in matchAndRewrite() local
432 auto div = builder.create<arith::DivFOp>(y, x); in matchAndRewrite()
433 auto atan = builder.create<math::AtanOp>(div); in matchAndRewrite()
436 auto zero = broadcast(builder, f32Cst(builder, 0.0f), shape); in matchAndRewrite()
437 auto pi = broadcast(builder, f32Cst(builder, 3.14159265359f), shape); in matchAndRewrite()
438 auto addPi = builder.create<arith::AddFOp>(atan, pi); in matchAndRewrite()
439 auto subPi = builder.create<arith::SubFOp>(atan, pi); in matchAndRewrite()
441 builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, atan, zero); in matchAndRewrite()
442 auto flippedAtan = builder.create<arith::SelectOp>(atanGt, subPi, addPi); in matchAndRewrite()
445 auto xGt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, x, zero); in matchAndRewrite()
446 Value result = builder.create<arith::SelectOp>(xGt, atan, flippedAtan); in matchAndRewrite()
450 builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, x, zero); in matchAndRewrite()
451 Value yGt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, y, zero); in matchAndRewrite()
452 Value isHalfPi = builder.create<arith::AndIOp>(xZero, yGt); in matchAndRewrite()
453 auto halfPi = broadcast(builder, f32Cst(builder, 1.57079632679f), shape); in matchAndRewrite()
454 result = builder.create<arith::SelectOp>(isHalfPi, halfPi, result); in matchAndRewrite()
457 Value yLt = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, y, zero); in matchAndRewrite()
458 Value isNegativeHalfPiPi = builder.create<arith::AndIOp>(xZero, yLt); in matchAndRewrite()
460 broadcast(builder, f32Cst(builder, -1.57079632679f), shape); in matchAndRewrite()
461 result = builder.create<arith::SelectOp>(isNegativeHalfPiPi, negativeHalfPiPi, in matchAndRewrite()
466 builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, y, zero); in matchAndRewrite()
467 Value isNan = builder.create<arith::AndIOp>(xZero, yZero); in matchAndRewrite()
468 Value cstNan = broadcast(builder, f32FromBits(builder, 0x7fc00000), shape); in matchAndRewrite()
469 result = builder.create<arith::SelectOp>(isNan, cstNan, result); in matchAndRewrite()
497 ImplicitLocOpBuilder builder(op->getLoc(), rewriter); in matchAndRewrite() local
499 return broadcast(builder, value, shape); in matchAndRewrite()
503 Value minusClamp = bcast(f32Cst(builder, -7.99881172180175781f)); in matchAndRewrite()
504 Value plusClamp = bcast(f32Cst(builder, 7.99881172180175781f)); in matchAndRewrite()
505 Value x = clamp(builder, op.getOperand(), minusClamp, plusClamp); in matchAndRewrite()
508 Value tiny = bcast(f32Cst(builder, 0.0004f)); in matchAndRewrite()
509 Value tinyMask = builder.create<arith::CmpFOp>( in matchAndRewrite()
510 arith::CmpFPredicate::OLT, builder.create<math::AbsOp>(op.getOperand()), in matchAndRewrite()
514 Value alpha1 = bcast(f32Cst(builder, 4.89352455891786e-03f)); in matchAndRewrite()
515 Value alpha3 = bcast(f32Cst(builder, 6.37261928875436e-04f)); in matchAndRewrite()
516 Value alpha5 = bcast(f32Cst(builder, 1.48572235717979e-05f)); in matchAndRewrite()
517 Value alpha7 = bcast(f32Cst(builder, 5.12229709037114e-08f)); in matchAndRewrite()
518 Value alpha9 = bcast(f32Cst(builder, -8.60467152213735e-11f)); in matchAndRewrite()
519 Value alpha11 = bcast(f32Cst(builder, 2.00018790482477e-13f)); in matchAndRewrite()
520 Value alpha13 = bcast(f32Cst(builder, -2.76076847742355e-16f)); in matchAndRewrite()
523 Value beta0 = bcast(f32Cst(builder, 4.89352518554385e-03f)); in matchAndRewrite()
524 Value beta2 = bcast(f32Cst(builder, 2.26843463243900e-03f)); in matchAndRewrite()
525 Value beta4 = bcast(f32Cst(builder, 1.18534705686654e-04f)); in matchAndRewrite()
526 Value beta6 = bcast(f32Cst(builder, 1.19825839466702e-06f)); in matchAndRewrite()
529 Value x2 = builder.create<arith::MulFOp>(x, x); in matchAndRewrite()
532 Value p = builder.create<math::FmaOp>(x2, alpha13, alpha11); in matchAndRewrite()
533 p = builder.create<math::FmaOp>(x2, p, alpha9); in matchAndRewrite()
534 p = builder.create<math::FmaOp>(x2, p, alpha7); in matchAndRewrite()
535 p = builder.create<math::FmaOp>(x2, p, alpha5); in matchAndRewrite()
536 p = builder.create<math::FmaOp>(x2, p, alpha3); in matchAndRewrite()
537 p = builder.create<math::FmaOp>(x2, p, alpha1); in matchAndRewrite()
538 p = builder.create<arith::MulFOp>(x, p); in matchAndRewrite()
541 Value q = builder.create<math::FmaOp>(x2, beta6, beta4); in matchAndRewrite()
542 q = builder.create<math::FmaOp>(x2, q, beta2); in matchAndRewrite()
543 q = builder.create<math::FmaOp>(x2, q, beta0); in matchAndRewrite()
546 Value res = builder.create<arith::SelectOp>( in matchAndRewrite()
547 tinyMask, x, builder.create<arith::DivFOp>(p, q)); in matchAndRewrite()
585 ImplicitLocOpBuilder builder(op->getLoc(), rewriter); in logMatchAndRewrite() local
587 return broadcast(builder, value, shape); in logMatchAndRewrite()
590 Value cstZero = bcast(f32Cst(builder, 0.0f)); in logMatchAndRewrite()
591 Value cstOne = bcast(f32Cst(builder, 1.0f)); in logMatchAndRewrite()
592 Value cstNegHalf = bcast(f32Cst(builder, -0.5f)); in logMatchAndRewrite()
595 Value cstMinNormPos = bcast(f32FromBits(builder, 0x00800000u)); in logMatchAndRewrite()
596 Value cstMinusInf = bcast(f32FromBits(builder, 0xff800000u)); in logMatchAndRewrite()
597 Value cstPosInf = bcast(f32FromBits(builder, 0x7f800000u)); in logMatchAndRewrite()
598 Value cstNan = bcast(f32FromBits(builder, 0x7fc00000)); in logMatchAndRewrite()
601 Value cstCephesSQRTHF = bcast(f32Cst(builder, 0.707106781186547524f)); in logMatchAndRewrite()
602 Value cstCephesLogP0 = bcast(f32Cst(builder, 7.0376836292E-2f)); in logMatchAndRewrite()
603 Value cstCephesLogP1 = bcast(f32Cst(builder, -1.1514610310E-1f)); in logMatchAndRewrite()
604 Value cstCephesLogP2 = bcast(f32Cst(builder, 1.1676998740E-1f)); in logMatchAndRewrite()
605 Value cstCephesLogP3 = bcast(f32Cst(builder, -1.2420140846E-1f)); in logMatchAndRewrite()
606 Value cstCephesLogP4 = bcast(f32Cst(builder, +1.4249322787E-1f)); in logMatchAndRewrite()
607 Value cstCephesLogP5 = bcast(f32Cst(builder, -1.6668057665E-1f)); in logMatchAndRewrite()
608 Value cstCephesLogP6 = bcast(f32Cst(builder, +2.0000714765E-1f)); in logMatchAndRewrite()
609 Value cstCephesLogP7 = bcast(f32Cst(builder, -2.4999993993E-1f)); in logMatchAndRewrite()
610 Value cstCephesLogP8 = bcast(f32Cst(builder, +3.3333331174E-1f)); in logMatchAndRewrite()
615 x = max(builder, x, cstMinNormPos); in logMatchAndRewrite()
618 std::pair<Value, Value> pair = frexp(builder, x, /*isPositive=*/true); in logMatchAndRewrite()
630 Value mask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x, in logMatchAndRewrite()
632 Value tmp = builder.create<arith::SelectOp>(mask, x, cstZero); in logMatchAndRewrite()
634 x = builder.create<arith::SubFOp>(x, cstOne); in logMatchAndRewrite()
635 e = builder.create<arith::SubFOp>( in logMatchAndRewrite()
636 e, builder.create<arith::SelectOp>(mask, cstOne, cstZero)); in logMatchAndRewrite()
637 x = builder.create<arith::AddFOp>(x, tmp); in logMatchAndRewrite()
639 Value x2 = builder.create<arith::MulFOp>(x, x); in logMatchAndRewrite()
640 Value x3 = builder.create<arith::MulFOp>(x2, x); in logMatchAndRewrite()
644 y0 = builder.create<math::FmaOp>(cstCephesLogP0, x, cstCephesLogP1); in logMatchAndRewrite()
645 y1 = builder.create<math::FmaOp>(cstCephesLogP3, x, cstCephesLogP4); in logMatchAndRewrite()
646 y2 = builder.create<math::FmaOp>(cstCephesLogP6, x, cstCephesLogP7); in logMatchAndRewrite()
647 y0 = builder.create<math::FmaOp>(y0, x, cstCephesLogP2); in logMatchAndRewrite()
648 y1 = builder.create<math::FmaOp>(y1, x, cstCephesLogP5); in logMatchAndRewrite()
649 y2 = builder.create<math::FmaOp>(y2, x, cstCephesLogP8); in logMatchAndRewrite()
650 y0 = builder.create<math::FmaOp>(y0, x3, y1); in logMatchAndRewrite()
651 y0 = builder.create<math::FmaOp>(y0, x3, y2); in logMatchAndRewrite()
652 y0 = builder.create<arith::MulFOp>(y0, x3); in logMatchAndRewrite()
654 y0 = builder.create<math::FmaOp>(cstNegHalf, x2, y0); in logMatchAndRewrite()
655 x = builder.create<arith::AddFOp>(x, y0); in logMatchAndRewrite()
658 Value cstLog2e = bcast(f32Cst(builder, static_cast<float>(LOG2E_VALUE))); in logMatchAndRewrite()
659 x = builder.create<math::FmaOp>(x, cstLog2e, e); in logMatchAndRewrite()
661 Value cstLn2 = bcast(f32Cst(builder, static_cast<float>(LN2_VALUE))); in logMatchAndRewrite()
662 x = builder.create<math::FmaOp>(e, cstLn2, x); in logMatchAndRewrite()
665 Value invalidMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::ULT, in logMatchAndRewrite()
667 Value zeroMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, in logMatchAndRewrite()
669 Value posInfMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, in logMatchAndRewrite()
676 Value aproximation = builder.create<arith::SelectOp>( in logMatchAndRewrite()
678 builder.create<arith::SelectOp>( in logMatchAndRewrite()
680 builder.create<arith::SelectOp>(posInfMask, cstPosInf, x))); in logMatchAndRewrite()
732 ImplicitLocOpBuilder builder(op->getLoc(), rewriter); in matchAndRewrite() local
734 return broadcast(builder, value, shape); in matchAndRewrite()
743 Value cstOne = bcast(f32Cst(builder, 1.0f)); in matchAndRewrite()
745 Value u = builder.create<arith::AddFOp>(x, cstOne); in matchAndRewrite()
747 builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, u, cstOne); in matchAndRewrite()
748 Value logU = builder.create<math::LogOp>(u); in matchAndRewrite()
750 builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, u, logU); in matchAndRewrite()
751 Value logLarge = builder.create<arith::MulFOp>( in matchAndRewrite()
752 x, builder.create<arith::DivFOp>( in matchAndRewrite()
753 logU, builder.create<arith::SubFOp>(u, cstOne))); in matchAndRewrite()
754 Value approximation = builder.create<arith::SelectOp>( in matchAndRewrite()
755 builder.create<arith::OrIOp>(uSmall, uInf), x, logLarge); in matchAndRewrite()
779 ImplicitLocOpBuilder builder(op->getLoc(), rewriter); in matchAndRewrite() local
781 return broadcast(builder, value, shape); in matchAndRewrite()
787 Value zero = bcast(f32Cst(builder, 0)); in matchAndRewrite()
788 Value one = bcast(f32Cst(builder, 1)); in matchAndRewrite()
790 pp[0][0] = bcast(f32Cst(builder, +0.00000000000000000e+00f)); in matchAndRewrite()
791 pp[0][1] = bcast(f32Cst(builder, +1.12837916222975858e+00f)); in matchAndRewrite()
792 pp[0][2] = bcast(f32Cst(builder, -5.23018562988006470e-01f)); in matchAndRewrite()
793 pp[0][3] = bcast(f32Cst(builder, +2.09741709609267072e-01f)); in matchAndRewrite()
794 pp[0][4] = bcast(f32Cst(builder, +2.58146801602987875e-02f)); in matchAndRewrite()
795 pp[1][0] = bcast(f32Cst(builder, +0.00000000000000000e+00f)); in matchAndRewrite()
796 pp[1][1] = bcast(f32Cst(builder, +1.12750687816789140e+00f)); in matchAndRewrite()
797 pp[1][2] = bcast(f32Cst(builder, -3.64721408487825775e-01f)); in matchAndRewrite()
798 pp[1][3] = bcast(f32Cst(builder, +1.18407396425136952e-01f)); in matchAndRewrite()
799 pp[1][4] = bcast(f32Cst(builder, +3.70645533056476558e-02f)); in matchAndRewrite()
800 pp[2][0] = bcast(f32Cst(builder, -3.30093071049483172e-03f)); in matchAndRewrite()
801 pp[2][1] = bcast(f32Cst(builder, +3.51961938357697011e-03f)); in matchAndRewrite()
802 pp[2][2] = bcast(f32Cst(builder, -1.41373622814988039e-03f)); in matchAndRewrite()
803 pp[2][3] = bcast(f32Cst(builder, +2.53447094961941348e-04f)); in matchAndRewrite()
804 pp[2][4] = bcast(f32Cst(builder, -1.71048029455037401e-05f)); in matchAndRewrite()
807 qq[0][0] = bcast(f32Cst(builder, +1.000000000000000000e+00f)); in matchAndRewrite()
808 qq[0][1] = bcast(f32Cst(builder, -4.635138185962547255e-01f)); in matchAndRewrite()
809 qq[0][2] = bcast(f32Cst(builder, +5.192301327279782447e-01f)); in matchAndRewrite()
810 qq[0][3] = bcast(f32Cst(builder, -1.318089722204810087e-01f)); in matchAndRewrite()
811 qq[0][4] = bcast(f32Cst(builder, +7.397964654672315005e-02f)); in matchAndRewrite()
812 qq[1][0] = bcast(f32Cst(builder, +1.00000000000000000e+00f)); in matchAndRewrite()
813 qq[1][1] = bcast(f32Cst(builder, -3.27607011824493086e-01f)); in matchAndRewrite()
814 qq[1][2] = bcast(f32Cst(builder, +4.48369090658821977e-01f)); in matchAndRewrite()
815 qq[1][3] = bcast(f32Cst(builder, -8.83462621207857930e-02f)); in matchAndRewrite()
816 qq[1][4] = bcast(f32Cst(builder, +5.72442770283176093e-02f)); in matchAndRewrite()
817 qq[2][0] = bcast(f32Cst(builder, +1.00000000000000000e+00f)); in matchAndRewrite()
818 qq[2][1] = bcast(f32Cst(builder, -2.06069165953913769e+00f)); in matchAndRewrite()
819 qq[2][2] = bcast(f32Cst(builder, +1.62705939945477759e+00f)); in matchAndRewrite()
820 qq[2][3] = bcast(f32Cst(builder, -5.83389859211130017e-01f)); in matchAndRewrite()
821 qq[2][4] = bcast(f32Cst(builder, +8.21908939856640930e-02f)); in matchAndRewrite()
824 offsets[0] = bcast(f32Cst(builder, 0.0f)); in matchAndRewrite()
825 offsets[1] = bcast(f32Cst(builder, 0.0f)); in matchAndRewrite()
826 offsets[2] = bcast(f32Cst(builder, 1.0f)); in matchAndRewrite()
829 bounds[0] = bcast(f32Cst(builder, 0.8f)); in matchAndRewrite()
830 bounds[1] = bcast(f32Cst(builder, 2.0f)); in matchAndRewrite()
831 bounds[2] = bcast(f32Cst(builder, 3.75f)); in matchAndRewrite()
833 Value isNegativeArg = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, in matchAndRewrite()
835 Value negArg = builder.create<arith::NegFOp>(op.getOperand()); in matchAndRewrite()
837 builder.create<arith::SelectOp>(isNegativeArg, negArg, op.getOperand()); in matchAndRewrite()
851 builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, x, bounds[j]); in matchAndRewrite()
853 p[i] = builder.create<arith::SelectOp>(isLessThanBound[j], p[i], in matchAndRewrite()
855 q[i] = builder.create<arith::SelectOp>(isLessThanBound[j], q[i], in matchAndRewrite()
858 offset = builder.create<arith::SelectOp>(isLessThanBound[j], offset, in matchAndRewrite()
861 isLessThanBound[intervalsCount - 1] = builder.create<arith::CmpFOp>( in matchAndRewrite()
864 Value pPoly = makePolynomialCalculation(builder, p, x); in matchAndRewrite()
865 Value qPoly = makePolynomialCalculation(builder, q, x); in matchAndRewrite()
866 Value rationalPoly = builder.create<arith::DivFOp>(pPoly, qPoly); in matchAndRewrite()
867 Value formula = builder.create<arith::AddFOp>(offset, rationalPoly); in matchAndRewrite()
868 formula = builder.create<arith::SelectOp>(isLessThanBound[intervalsCount - 1], in matchAndRewrite()
872 Value negFormula = builder.create<arith::NegFOp>(formula); in matchAndRewrite()
874 builder.create<arith::SelectOp>(isNegativeArg, negFormula, formula); in matchAndRewrite()
907 ImplicitLocOpBuilder builder(op->getLoc(), rewriter); in matchAndRewrite() local
912 return broadcast(builder, value, shape); in matchAndRewrite()
915 return builder.create<math::FmaOp>(a, b, c); in matchAndRewrite()
918 return builder.create<arith::MulFOp>(a, b); in matchAndRewrite()
921 return builder.create<arith::SubFOp>(a, b); in matchAndRewrite()
923 auto floor = [&](Value a) { return builder.create<math::FloorOp>(a); }; in matchAndRewrite()
925 Value cstLn2 = bcast(f32Cst(builder, static_cast<float>(LN2_VALUE))); in matchAndRewrite()
926 Value cstLog2E = bcast(f32Cst(builder, static_cast<float>(LOG2E_VALUE))); in matchAndRewrite()
929 Value cstCephesExpP0 = bcast(f32Cst(builder, 1.0)); in matchAndRewrite()
930 Value cstCephesExpP1 = bcast(f32Cst(builder, 1.0)); in matchAndRewrite()
931 Value cstCephesExpP2 = bcast(f32Cst(builder, 0.49970514590562437052f)); in matchAndRewrite()
932 Value cstCephesExpP3 = bcast(f32Cst(builder, 0.16873890085469545053f)); in matchAndRewrite()
933 Value cstCephesExpP4 = bcast(f32Cst(builder, 0.03668965196652099192f)); in matchAndRewrite()
934 Value cstCephesExpP5 = bcast(f32Cst(builder, 0.01314350012789660196f)); in matchAndRewrite()
938 Value isNan = builder.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, x, x); in matchAndRewrite()
957 auto i32Vec = broadcast(builder.getI32Type(), shape); in matchAndRewrite()
960 Value k = builder.create<arith::FPToSIOp>(i32Vec, kF32); in matchAndRewrite()
961 Value exp2KValue = exp2I32(builder, k); in matchAndRewrite()
972 Value zerof32Const = bcast(f32Cst(builder, 0)); in matchAndRewrite()
974 bcast(f32Cst(builder, std::numeric_limits<float>::infinity())); in matchAndRewrite()
976 bcast(f32Cst(builder, -std::numeric_limits<float>::infinity())); in matchAndRewrite()
977 auto underflow = bcast(f32Cst(builder, std::numeric_limits<float>::min())); in matchAndRewrite()
979 Value kMaxConst = bcast(i32Cst(builder, 127)); in matchAndRewrite()
980 Value kMaxNegConst = bcast(i32Cst(builder, -127)); in matchAndRewrite()
982 builder.create<arith::CmpIOp>(arith::CmpIPredicate::sle, k, kMaxConst); in matchAndRewrite()
984 builder.create<arith::CmpIOp>(arith::CmpIPredicate::sge, k, kMaxNegConst); in matchAndRewrite()
986 Value isNegInfinityX = builder.create<arith::CmpFOp>( in matchAndRewrite()
988 Value isPosInfinityX = builder.create<arith::CmpFOp>( in matchAndRewrite()
991 builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, x, zerof32Const); in matchAndRewrite()
992 Value isComputable = builder.create<arith::AndIOp>(rightBound, leftBound); in matchAndRewrite()
994 expY = builder.create<arith::SelectOp>( in matchAndRewrite()
996 builder.create<arith::SelectOp>( in matchAndRewrite()
998 builder.create<arith::SelectOp>( in matchAndRewrite()
1000 builder.create<arith::SelectOp>( in matchAndRewrite()
1002 builder.create<arith::SelectOp>(isPostiveX, constPosInfinity, in matchAndRewrite()
1033 ImplicitLocOpBuilder builder(op->getLoc(), rewriter); in matchAndRewrite() local
1035 return broadcast(builder, value, shape); in matchAndRewrite()
1041 Value cstOne = bcast(f32Cst(builder, 1.0f)); in matchAndRewrite()
1042 Value cstNegOne = bcast(f32Cst(builder, -1.0f)); in matchAndRewrite()
1044 Value u = builder.create<math::ExpOp>(x); in matchAndRewrite()
1046 builder.create<arith::CmpFOp>(arith::CmpFPredicate::UEQ, u, cstOne); in matchAndRewrite()
1047 Value uMinusOne = builder.create<arith::SubFOp>(u, cstOne); in matchAndRewrite()
1048 Value uMinusOneEqNegOne = builder.create<arith::CmpFOp>( in matchAndRewrite()
1051 Value logU = builder.create<math::LogOp>(u); in matchAndRewrite()
1055 builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, logU, u); in matchAndRewrite()
1058 Value expm1 = builder.create<arith::MulFOp>( in matchAndRewrite()
1059 uMinusOne, builder.create<arith::DivFOp>(x, logU)); in matchAndRewrite()
1060 expm1 = builder.create<arith::SelectOp>(isInf, u, expm1); in matchAndRewrite()
1061 Value approximation = builder.create<arith::SelectOp>( in matchAndRewrite()
1063 builder.create<arith::SelectOp>(uMinusOneEqNegOne, cstNegOne, expm1)); in matchAndRewrite()
1103 ImplicitLocOpBuilder builder(op->getLoc(), rewriter); in matchAndRewrite() local
1105 return broadcast(builder, value, shape); in matchAndRewrite()
1108 return builder.create<arith::MulFOp>(a, b); in matchAndRewrite()
1111 return builder.create<arith::SubFOp>(a, b); in matchAndRewrite()
1113 auto floor = [&](Value a) { return builder.create<math::FloorOp>(a); }; in matchAndRewrite()
1115 auto i32Vec = broadcast(builder.getI32Type(), shape); in matchAndRewrite()
1117 return builder.create<arith::FPToSIOp>(i32Vec, a); in matchAndRewrite()
1121 return builder.create<arith::AndIOp>(a, bcast(i32Cst(builder, 3))); in matchAndRewrite()
1125 return builder.create<arith::CmpIOp>(arith::CmpIPredicate::eq, a, b); in matchAndRewrite()
1129 return builder.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, a, b); in matchAndRewrite()
1133 return builder.create<arith::SelectOp>(cond, t, f); in matchAndRewrite()
1137 return builder.create<math::FmaOp>(a, b, c); in matchAndRewrite()
1141 return builder.create<arith::OrIOp>(a, b); in matchAndRewrite()
1144 Value twoOverPi = bcast(f32Cst(builder, (float)TWO_OVER_PI)); in matchAndRewrite()
1145 Value piOverTwo = bcast(f32Cst(builder, (float)PI_OVER_2)); in matchAndRewrite()
1153 Value cstOne = bcast(f32Cst(builder, 1.0)); in matchAndRewrite()
1154 Value cstNegativeOne = bcast(f32Cst(builder, -1.0)); in matchAndRewrite()
1156 Value cstSC2 = bcast(f32Cst(builder, -0.16666667163372039794921875f)); in matchAndRewrite()
1157 Value cstSC4 = bcast(f32Cst(builder, 8.333347737789154052734375e-3f)); in matchAndRewrite()
1158 Value cstSC6 = bcast(f32Cst(builder, -1.9842604524455964565277099609375e-4f)); in matchAndRewrite()
1160 bcast(f32Cst(builder, 2.760012648650445044040679931640625e-6f)); in matchAndRewrite()
1162 bcast(f32Cst(builder, -2.50293279435709337121807038784027099609375e-8f)); in matchAndRewrite()
1164 Value cstCC2 = bcast(f32Cst(builder, -0.5f)); in matchAndRewrite()
1165 Value cstCC4 = bcast(f32Cst(builder, 4.166664183139801025390625e-2f)); in matchAndRewrite()
1166 Value cstCC6 = bcast(f32Cst(builder, -1.388833043165504932403564453125e-3f)); in matchAndRewrite()
1167 Value cstCC8 = bcast(f32Cst(builder, 2.47562347794882953166961669921875e-5f)); in matchAndRewrite()
1169 bcast(f32Cst(builder, -2.59630184018533327616751194000244140625e-7f)); in matchAndRewrite()
1173 Value kR0 = isEqualTo(kMod4, bcast(i32Cst(builder, 0))); in matchAndRewrite()
1174 Value kR1 = isEqualTo(kMod4, bcast(i32Cst(builder, 1))); in matchAndRewrite()
1175 Value kR2 = isEqualTo(kMod4, bcast(i32Cst(builder, 2))); in matchAndRewrite()
1176 Value kR3 = isEqualTo(kMod4, bcast(i32Cst(builder, 3))); in matchAndRewrite()
1179 Value negativeRange = isSine ? isGreaterThan(kMod4, bcast(i32Cst(builder, 1))) in matchAndRewrite()
1230 ImplicitLocOpBuilder builder(op->getLoc(), rewriter); in matchAndRewrite() local
1232 return broadcast(builder, value, shape); in matchAndRewrite()
1235 Value cstPosInf = bcast(f32FromBits(builder, 0x7f800000u)); in matchAndRewrite()
1236 Value cstOnePointFive = bcast(f32Cst(builder, 1.5f)); in matchAndRewrite()
1237 Value cstNegHalf = bcast(f32Cst(builder, -0.5f)); in matchAndRewrite()
1238 Value cstMinNormPos = bcast(f32FromBits(builder, 0x00800000u)); in matchAndRewrite()
1240 Value negHalf = builder.create<arith::MulFOp>(op.getOperand(), cstNegHalf); in matchAndRewrite()
1244 Value ltMinMask = builder.create<arith::CmpFOp>( in matchAndRewrite()
1246 Value infMask = builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, in matchAndRewrite()
1248 Value notNormalFiniteMask = builder.create<arith::OrIOp>(ltMinMask, infMask); in matchAndRewrite()
1252 builder, op->getOperands(), 8, [&builder](ValueRange operands) -> Value { in matchAndRewrite()
1253 return builder.create<x86vector::RsqrtOp>(operands); in matchAndRewrite()
1260 Value inner = builder.create<arith::MulFOp>(negHalf, yApprox); in matchAndRewrite()
1261 Value fma = builder.create<math::FmaOp>(yApprox, inner, cstOnePointFive); in matchAndRewrite()
1262 Value yNewton = builder.create<arith::MulFOp>(yApprox, fma); in matchAndRewrite()
1270 builder.create<arith::SelectOp>(notNormalFiniteMask, yApprox, yNewton); in matchAndRewrite()