Lines Matching refs:MatMul

1282                                CallInst *MatMul) {  in getNonAliasingPointer()  argument
1294 BasicBlock *Check0 = MatMul->getParent(); in getNonAliasingPointer()
1303 SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI, in getNonAliasingPointer()
1306 SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI, in getNonAliasingPointer()
1309 SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI, in getNonAliasingPointer()
1315 IRBuilder<> Builder(MatMul); in getNonAliasingPointer()
1362 bool isFusionProfitable(CallInst *MatMul) { in isFusionProfitable() argument
1366 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in isFusionProfitable()
1367 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in isFusionProfitable()
1372 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in isFusionProfitable()
1405 void createTiledLoops(CallInst *MatMul, Value *LPtr, ShapeInfo LShape, in createTiledLoops() argument
1407 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in createTiledLoops()
1412 Instruction *InsertI = cast<Instruction>(MatMul); in createTiledLoops()
1416 IRBuilder<> Builder(MatMul); in createTiledLoops()
1420 FixedVectorType::get(MatMul->getType()->getScalarType(), TileSize); in createTiledLoops()
1443 getFastMathFlags(MatMul)); in createTiledLoops()
1462 void emitSIMDTiling(CallInst *MatMul, LoadInst *LoadOp0, LoadInst *LoadOp1, in emitSIMDTiling() argument
1467 if (!isFusionProfitable(MatMul)) in emitSIMDTiling()
1470 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in emitSIMDTiling()
1471 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in emitSIMDTiling()
1476 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in emitSIMDTiling()
1478 Value *APtr = getNonAliasingPointer(LoadOp0, Store, MatMul); in emitSIMDTiling()
1479 Value *BPtr = getNonAliasingPointer(LoadOp1, Store, MatMul); in emitSIMDTiling()
1483 createTiledLoops(MatMul, APtr, LShape, BPtr, RShape, Store); in emitSIMDTiling()
1503 getFastMathFlags(MatMul)); in emitSIMDTiling()
1513 FusedInsts.insert(MatMul); in emitSIMDTiling()
1515 MatMul->eraseFromParent(); in emitSIMDTiling()
1530 void LowerMatrixMultiplyFused(CallInst *MatMul, in LowerMatrixMultiplyFused() argument
1537 Value *A = MatMul->getArgOperand(0); in LowerMatrixMultiplyFused()
1538 Value *B = MatMul->getArgOperand(1); in LowerMatrixMultiplyFused()
1545 IRBuilder<> Builder(MatMul); in LowerMatrixMultiplyFused()
1546 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in LowerMatrixMultiplyFused()
1547 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in LowerMatrixMultiplyFused()
1548 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in LowerMatrixMultiplyFused()
1571 getFastMathFlags(MatMul)); in LowerMatrixMultiplyFused()
1573 FusedInsts.insert(MatMul); in LowerMatrixMultiplyFused()
1581 finalizeLowering(MatMul, Result, Builder); in LowerMatrixMultiplyFused()
1585 if (!MatMul->hasOneUse() || MatrixLayout != MatrixLayoutTy::ColumnMajor) in LowerMatrixMultiplyFused()
1592 auto *Store = dyn_cast<StoreInst>(*MatMul->user_begin()); in LowerMatrixMultiplyFused()
1606 if (DT->dominates(CurrI, MatMul)) in LowerMatrixMultiplyFused()
1618 I->moveBefore(MatMul); in LowerMatrixMultiplyFused()
1620 emitSIMDTiling(MatMul, LoadOp0, LoadOp1, Store, FusedInsts); in LowerMatrixMultiplyFused()
1626 void LowerMultiply(CallInst *MatMul) { in LowerMultiply() argument
1627 IRBuilder<> Builder(MatMul); in LowerMultiply()
1628 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in LowerMultiply()
1629 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in LowerMultiply()
1630 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in LowerMultiply()
1632 const MatrixTy &Lhs = getMatrix(MatMul->getArgOperand(0), LShape, Builder); in LowerMultiply()
1633 const MatrixTy &Rhs = getMatrix(MatMul->getArgOperand(1), RShape, Builder); in LowerMultiply()
1647 getFastMathFlags(MatMul)); in LowerMultiply()
1648 finalizeLowering(MatMul, Result, Builder); in LowerMultiply()