Lines Matching refs:MatMul

1282                                CallInst *MatMul) {  in getNonAliasingPointer()  argument
1294 BasicBlock *Check0 = MatMul->getParent(); in getNonAliasingPointer()
1303 SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI, in getNonAliasingPointer()
1306 SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI, in getNonAliasingPointer()
1309 SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI, in getNonAliasingPointer()
1315 IRBuilder<> Builder(MatMul); in getNonAliasingPointer()
1367 bool isFusionProfitable(CallInst *MatMul) { in isFusionProfitable() argument
1371 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in isFusionProfitable()
1372 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in isFusionProfitable()
1377 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in isFusionProfitable()
1411 void createTiledLoops(CallInst *MatMul, Value *LPtr, ShapeInfo LShape, in createTiledLoops() argument
1413 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in createTiledLoops()
1418 Instruction *InsertI = cast<Instruction>(MatMul); in createTiledLoops()
1422 IRBuilder<> Builder(MatMul); in createTiledLoops()
1426 FixedVectorType::get(MatMul->getType()->getScalarType(), TileSize); in createTiledLoops()
1451 getFastMathFlags(MatMul)); in createTiledLoops()
1470 void emitSIMDTiling(CallInst *MatMul, LoadInst *LoadOp0, LoadInst *LoadOp1, in emitSIMDTiling() argument
1475 if (!isFusionProfitable(MatMul)) in emitSIMDTiling()
1478 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in emitSIMDTiling()
1479 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in emitSIMDTiling()
1484 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in emitSIMDTiling()
1486 Value *APtr = getNonAliasingPointer(LoadOp0, Store, MatMul); in emitSIMDTiling()
1487 Value *BPtr = getNonAliasingPointer(LoadOp1, Store, MatMul); in emitSIMDTiling()
1491 createTiledLoops(MatMul, APtr, LShape, BPtr, RShape, Store); in emitSIMDTiling()
1511 getFastMathFlags(MatMul)); in emitSIMDTiling()
1521 FusedInsts.insert(MatMul); in emitSIMDTiling()
1523 MatMul->eraseFromParent(); in emitSIMDTiling()
1538 void LowerMatrixMultiplyFused(CallInst *MatMul, in LowerMatrixMultiplyFused() argument
1545 Value *A = MatMul->getArgOperand(0); in LowerMatrixMultiplyFused()
1546 Value *B = MatMul->getArgOperand(1); in LowerMatrixMultiplyFused()
1553 IRBuilder<> Builder(MatMul); in LowerMatrixMultiplyFused()
1554 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in LowerMatrixMultiplyFused()
1555 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in LowerMatrixMultiplyFused()
1556 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in LowerMatrixMultiplyFused()
1579 getFastMathFlags(MatMul)); in LowerMatrixMultiplyFused()
1581 FusedInsts.insert(MatMul); in LowerMatrixMultiplyFused()
1589 finalizeLowering(MatMul, Result, Builder); in LowerMatrixMultiplyFused()
1593 if (!MatMul->hasOneUse() || MatrixLayout != MatrixLayoutTy::ColumnMajor) in LowerMatrixMultiplyFused()
1600 auto *Store = dyn_cast<StoreInst>(*MatMul->user_begin()); in LowerMatrixMultiplyFused()
1614 if (DT->dominates(CurrI, MatMul)) in LowerMatrixMultiplyFused()
1626 I->moveBefore(MatMul); in LowerMatrixMultiplyFused()
1628 emitSIMDTiling(MatMul, LoadOp0, LoadOp1, Store, FusedInsts); in LowerMatrixMultiplyFused()
1634 void LowerMultiply(CallInst *MatMul) { in LowerMultiply() argument
1635 IRBuilder<> Builder(MatMul); in LowerMultiply()
1636 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in LowerMultiply()
1637 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in LowerMultiply()
1638 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in LowerMultiply()
1640 const MatrixTy &Lhs = getMatrix(MatMul->getArgOperand(0), LShape, Builder); in LowerMultiply()
1641 const MatrixTy &Rhs = getMatrix(MatMul->getArgOperand(1), RShape, Builder); in LowerMultiply()
1655 getFastMathFlags(MatMul)); in LowerMultiply()
1656 finalizeLowering(MatMul, Result, Builder); in LowerMultiply()