Lines Matching refs:MatMul
1313 void lowerDotProduct(CallInst *MatMul, in lowerDotProduct() argument
1316 if (FusedInsts.contains(MatMul) || in lowerDotProduct()
1319 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in lowerDotProduct()
1320 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in lowerDotProduct()
1325 Value *LHS = MatMul->getArgOperand(0); in lowerDotProduct()
1326 Value *RHS = MatMul->getArgOperand(1); in lowerDotProduct()
1412 FusedInsts.insert(MatMul); in lowerDotProduct()
1413 IRBuilder<> Builder(MatMul); in lowerDotProduct()
1462 Result = Builder.CreateInsertElement(PoisonValue::get(MatMul->getType()), in lowerDotProduct()
1464 MatMul->replaceAllUsesWith(Result); in lowerDotProduct()
1465 FusedInsts.insert(MatMul); in lowerDotProduct()
1466 ToRemove.push_back(MatMul); in lowerDotProduct()
1561 CallInst *MatMul) { in getNonAliasingPointer() argument
1573 BasicBlock *Check0 = MatMul->getParent(); in getNonAliasingPointer()
1582 SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI, in getNonAliasingPointer()
1585 SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI, in getNonAliasingPointer()
1588 SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI, in getNonAliasingPointer()
1594 IRBuilder<> Builder(MatMul); in getNonAliasingPointer()
1645 bool isFusionProfitable(CallInst *MatMul) { in isFusionProfitable() argument
1649 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in isFusionProfitable()
1650 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in isFusionProfitable()
1655 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in isFusionProfitable()
1689 void createTiledLoops(CallInst *MatMul, Value *LPtr, ShapeInfo LShape, in createTiledLoops() argument
1691 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in createTiledLoops()
1696 Instruction *InsertI = cast<Instruction>(MatMul); in createTiledLoops()
1700 IRBuilder<> Builder(MatMul); in createTiledLoops()
1704 FixedVectorType::get(MatMul->getType()->getScalarType(), TileSize); in createTiledLoops()
1729 getFastMathFlags(MatMul)); in createTiledLoops()
1748 void emitSIMDTiling(CallInst *MatMul, LoadInst *LoadOp0, LoadInst *LoadOp1, in emitSIMDTiling() argument
1753 if (!isFusionProfitable(MatMul)) in emitSIMDTiling()
1756 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in emitSIMDTiling()
1757 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in emitSIMDTiling()
1762 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in emitSIMDTiling()
1764 Value *APtr = getNonAliasingPointer(LoadOp0, Store, MatMul); in emitSIMDTiling()
1765 Value *BPtr = getNonAliasingPointer(LoadOp1, Store, MatMul); in emitSIMDTiling()
1769 createTiledLoops(MatMul, APtr, LShape, BPtr, RShape, Store); in emitSIMDTiling()
1789 getFastMathFlags(MatMul)); in emitSIMDTiling()
1799 FusedInsts.insert(MatMul); in emitSIMDTiling()
1801 MatMul->eraseFromParent(); in emitSIMDTiling()
1816 void LowerMatrixMultiplyFused(CallInst *MatMul, in LowerMatrixMultiplyFused() argument
1823 Value *A = MatMul->getArgOperand(0); in LowerMatrixMultiplyFused()
1824 Value *B = MatMul->getArgOperand(1); in LowerMatrixMultiplyFused()
1831 IRBuilder<> Builder(MatMul); in LowerMatrixMultiplyFused()
1832 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in LowerMatrixMultiplyFused()
1833 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in LowerMatrixMultiplyFused()
1834 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in LowerMatrixMultiplyFused()
1857 getFastMathFlags(MatMul)); in LowerMatrixMultiplyFused()
1859 FusedInsts.insert(MatMul); in LowerMatrixMultiplyFused()
1867 finalizeLowering(MatMul, Result, Builder); in LowerMatrixMultiplyFused()
1871 if (!MatMul->hasOneUse() || MatrixLayout != MatrixLayoutTy::ColumnMajor) in LowerMatrixMultiplyFused()
1878 auto *Store = dyn_cast<StoreInst>(*MatMul->user_begin()); in LowerMatrixMultiplyFused()
1892 if (DT->dominates(CurrI, MatMul)) in LowerMatrixMultiplyFused()
1904 I->moveBefore(MatMul); in LowerMatrixMultiplyFused()
1906 emitSIMDTiling(MatMul, LoadOp0, LoadOp1, Store, FusedInsts); in LowerMatrixMultiplyFused()
1912 void LowerMultiply(CallInst *MatMul) { in LowerMultiply() argument
1913 IRBuilder<> Builder(MatMul); in LowerMultiply()
1914 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in LowerMultiply()
1915 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in LowerMultiply()
1916 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in LowerMultiply()
1918 const MatrixTy &Lhs = getMatrix(MatMul->getArgOperand(0), LShape, Builder); in LowerMultiply()
1919 const MatrixTy &Rhs = getMatrix(MatMul->getArgOperand(1), RShape, Builder); in LowerMultiply()
1933 getFastMathFlags(MatMul)); in LowerMultiply()
1934 finalizeLowering(MatMul, Result, Builder); in LowerMultiply()