63 Value *BlockGenerator::trySynthesizeNewValue(ScopStmt &Stmt, Value *Old,  in trySynthesizeNewValue()  argument
82   Scop &S = *Stmt.getParent();  in trySynthesizeNewValue()
96 Value *BlockGenerator::getNewValue(ScopStmt &Stmt, Value *Old, ValueMapT &BBMap,  in getNewValue()  argument
126   auto VUse = VirtualUse::create(&Stmt, L, Old, true);  in getNewValue()
191     New = trySynthesizeNewValue(Stmt, Old, BBMap, LTS, L);  in getNewValue()
213 void BlockGenerator::copyInstScalar(ScopStmt &Stmt, Instruction *Inst,  in copyInstScalar()  argument
226         getNewValue(Stmt, OldOperand, BBMap, LTS, getLoopForStmt(Stmt));  in copyInstScalar()
255 BlockGenerator::generateLocationAccessed(ScopStmt &Stmt, MemAccInst Inst,  in generateLocationAccessed()  argument
258   const MemoryAccess &MA = Stmt.getArrayAccessFor(Inst);  in generateLocationAccessed()
260       Stmt, getLoopForStmt(Stmt),  in generateLocationAccessed()
266     ScopStmt &Stmt, Loop *L, Value *Pointer, ValueMapT &BBMap,  in generateLocationAccessed()  argument
290   return getNewValue(Stmt, Pointer, BBMap, LTS, L);  in generateLocationAccessed()
305 Loop *BlockGenerator::getLoopForStmt(const ScopStmt &Stmt) const {  in getLoopForStmt()
306   auto *StmtBB = Stmt.getEntryBlock();  in getLoopForStmt()
310 Value *BlockGenerator::generateArrayLoad(ScopStmt &Stmt, LoadInst *Load,  in generateArrayLoad()  argument
317       generateLocationAccessed(Stmt, Load, BBMap, LTS, NewAccesses);  in generateArrayLoad()
329 void BlockGenerator::generateArrayStore(ScopStmt &Stmt, StoreInst *Store,  in generateArrayStore()  argument
332   MemoryAccess &MA = Stmt.getArrayAccessFor(Store);  in generateArrayStore()
336   generateConditionalExecution(Stmt, AccDom, Subject.c_str(), [&, this]() {  in generateArrayStore()
338         generateLocationAccessed(Stmt, Store, BBMap, LTS, NewAccesses);  in generateArrayStore()
339     Value *ValueOperand = getNewValue(Stmt, Store->getValueOperand(), BBMap,  in generateArrayStore()
340                                       LTS, getLoopForStmt(Stmt));  in generateArrayStore()
350 bool BlockGenerator::canSyntheziseInStmt(ScopStmt &Stmt, Instruction *Inst) {  in canSyntheziseInStmt()  argument
351   Loop *L = getLoopForStmt(Stmt);  in canSyntheziseInStmt()
352   return (Stmt.isBlockStmt() || !Stmt.getRegion()->contains(L)) &&  in canSyntheziseInStmt()
353          canSynthesize(Inst, *Stmt.getParent(), &SE, L);  in canSyntheziseInStmt()
356 void BlockGenerator::copyInstruction(ScopStmt &Stmt, Instruction *Inst,  in copyInstruction()  argument
365   if (canSyntheziseInStmt(Stmt, Inst))  in copyInstruction()
369     Value *NewLoad = generateArrayLoad(Stmt, Load, BBMap, LTS, NewAccesses);  in copyInstruction()
378     if (!Stmt.getArrayAccessOrNULLFor(Store))  in copyInstruction()
381     generateArrayStore(Stmt, Store, BBMap, LTS, NewAccesses);  in copyInstruction()
386     copyPHIInstruction(Stmt, PHI, BBMap, LTS);  in copyInstruction()
395   copyInstScalar(Stmt, Inst, BBMap, LTS);  in copyInstruction()
416 void BlockGenerator::copyStmt(ScopStmt &Stmt, LoopToScevMapT &LTS,  in copyStmt()  argument
418   assert(Stmt.isBlockStmt() &&  in copyStmt()
423   BasicBlock *BB = Stmt.getBasicBlock();  in copyStmt()
424   copyBB(Stmt, BB, BBMap, LTS, NewAccesses);  in copyStmt()
435 BasicBlock *BlockGenerator::copyBB(ScopStmt &Stmt, BasicBlock *BB,  in copyBB()  argument
440   generateScalarLoads(Stmt, LTS, BBMap, NewAccesses);  in copyBB()
441   generateBeginStmtTrace(Stmt, LTS, BBMap);  in copyBB()
443   copyBB(Stmt, BB, CopyBB, BBMap, LTS, NewAccesses);  in copyBB()
447   generateScalarStores(Stmt, LTS, BBMap, NewAccesses);  in copyBB()
451 void BlockGenerator::copyBB(ScopStmt &Stmt, BasicBlock *BB, BasicBlock *CopyBB,  in copyBB()  argument
461   if (Stmt.isBlockStmt() || (Stmt.isRegionStmt() && Stmt.getEntryBlock() == BB))  in copyBB()
462     for (Instruction *Inst : Stmt.getInstructions())  in copyBB()
463       copyInstruction(Stmt, Inst, BBMap, LTS, NewAccesses);  in copyBB()
466       copyInstruction(Stmt, &Inst, BBMap, LTS, NewAccesses);  in copyBB()
558     ScopStmt &Stmt, LoopToScevMapT &LTS, ValueMapT &BBMap,  in generateScalarLoads()  argument
560   for (MemoryAccess *MA : Stmt) {  in generateScalarLoads()
566         Stmt.getDomain().intersect_params(Stmt.getParent()->getContext());  in generateScalarLoads()
573         getImplicitAddress(*MA, getLoopForStmt(Stmt), LTS, BBMap, NewAccesses);  in generateScalarLoads()
583 Value *BlockGenerator::buildContainsCondition(ScopStmt &Stmt,  in buildContainsCondition()  argument
585   isl::ast_build AstBuild = Stmt.getAstBuild();  in buildContainsCondition()
586   isl::set Domain = Stmt.getDomain();  in buildContainsCondition()
608     ScopStmt &Stmt, const isl::set &Subdomain, StringRef Subject,  in generateConditionalExecution()  argument
610   isl::set StmtDom = Stmt.getDomain();  in generateConditionalExecution()
615       !StmtDom.intersect_params(Stmt.getParent()->getContext())  in generateConditionalExecution()
623   Value *Cond = buildContainsCondition(Stmt, Subdomain);  in generateConditionalExecution()
661 void BlockGenerator::generateBeginStmtTrace(ScopStmt &Stmt, LoopToScevMapT &LTS,  in generateBeginStmtTrace()  argument
666   Scop *S = Stmt.getParent();  in generateBeginStmtTrace()
667   const char *BaseName = Stmt.getBaseName();  in generateBeginStmtTrace()
669   isl::ast_build AstBuild = Stmt.getAstBuild();  in generateBeginStmtTrace()
670   isl::set Domain = Stmt.getDomain();  in generateBeginStmtTrace()
705     for (Instruction *Inst : Stmt.insts()) {  in generateBeginStmtTrace()
714         Values.push_back(getNewValue(Stmt, Inst, BBMap, LTS,  in generateBeginStmtTrace()
737           Values.push_back(getNewValue(Stmt, OpInst, BBMap, LTS,  in generateBeginStmtTrace()
755     ScopStmt &Stmt, LoopToScevMapT &LTS, ValueMapT &BBMap,  in generateScalarStores()  argument
757   Loop *L = LI.getLoopFor(Stmt.getBasicBlock());  in generateScalarStores()
759   assert(Stmt.isBlockStmt() &&  in generateScalarStores()
763   for (MemoryAccess *MA : Stmt) {  in generateScalarStores()
771         Stmt, AccDom, Subject.c_str(), [&, this, MA]() {  in generateScalarStores()
781                                  return p.first == Stmt.getBasicBlock();  in generateScalarStores()
786           auto Address = getImplicitAddress(*MA, getLoopForStmt(Stmt), LTS,  in generateScalarStores()
789           Val = getNewValue(Stmt, Val, BBMap, LTS, L);  in generateScalarStores()
978   for (auto &Stmt : S)  in invalidateScalarEvolution()  local
979     if (Stmt.isCopyStmt())  in invalidateScalarEvolution()
981     else if (Stmt.isBlockStmt())  in invalidateScalarEvolution()
982       for (auto &Inst : *Stmt.getBasicBlock())  in invalidateScalarEvolution()
984     else if (Stmt.isRegionStmt())  in invalidateScalarEvolution()
985       for (auto *BB : Stmt.getRegion()->blocks())  in invalidateScalarEvolution()
1019 Value *VectorBlockGenerator::getVectorValue(ScopStmt &Stmt, Value *Old,  in getVectorValue()  argument
1032         Vector, getNewValue(Stmt, Old, ScalarMaps[Lane], VLTS[Lane], L),  in getVectorValue()
1041     ScopStmt &Stmt, LoadInst *Load, VectorValueMapT &ScalarMaps,  in generateStrideOneLoad()  argument
1049   Value *NewPointer = generateLocationAccessed(Stmt, Load, ScalarMaps[Offset],  in generateStrideOneLoad()
1072     ScopStmt &Stmt, LoadInst *Load, ValueMapT &BBMap,  in generateStrideZeroLoad()  argument
1078       generateLocationAccessed(Stmt, Load, BBMap, VLTS[0], NewAccesses);  in generateStrideZeroLoad()
1096     ScopStmt &Stmt, LoadInst *Load, VectorValueMapT &ScalarMaps,  in generateUnknownStrideLoad()  argument
1105     Value *NewPointer = generateLocationAccessed(Stmt, Load, ScalarMaps[i],  in generateUnknownStrideLoad()
1117     ScopStmt &Stmt, LoadInst *Load, ValueMapT &VectorMap,  in generateLoad()  argument
1128           generateArrayLoad(Stmt, Load, ScalarMaps[i], VLTS[i], NewAccesses);  in generateLoad()
1132   const MemoryAccess &Access = Stmt.getArrayAccessFor(Load);  in generateLoad()
1140     NewLoad = generateStrideZeroLoad(Stmt, Load, ScalarMaps[0], NewAccesses);  in generateLoad()
1142     NewLoad = generateStrideOneLoad(Stmt, Load, ScalarMaps, NewAccesses);  in generateLoad()
1144     NewLoad = generateStrideOneLoad(Stmt, Load, ScalarMaps, NewAccesses, true);  in generateLoad()
1146     NewLoad = generateUnknownStrideLoad(Stmt, Load, ScalarMaps, NewAccesses);  in generateLoad()
1151 void VectorBlockGenerator::copyUnaryInst(ScopStmt &Stmt, UnaryInstruction *Inst,  in copyUnaryInst()  argument
1155   Value *NewOperand = getVectorValue(Stmt, Inst->getOperand(0), VectorMap,  in copyUnaryInst()
1156                                      ScalarMaps, getLoopForStmt(Stmt));  in copyUnaryInst()
1165 void VectorBlockGenerator::copyBinaryInst(ScopStmt &Stmt, BinaryOperator *Inst,  in copyBinaryInst()  argument
1168   Loop *L = getLoopForStmt(Stmt);  in copyBinaryInst()
1173   NewOpZero = getVectorValue(Stmt, OpZero, VectorMap, ScalarMaps, L);  in copyBinaryInst()
1174   NewOpOne = getVectorValue(Stmt, OpOne, VectorMap, ScalarMaps, L);  in copyBinaryInst()
1182     ScopStmt &Stmt, StoreInst *Store, ValueMapT &VectorMap,  in copyStore()  argument
1184   const MemoryAccess &Access = Stmt.getArrayAccessFor(Store);  in copyStore()
1186   Value *Vector = getVectorValue(Stmt, Store->getValueOperand(), VectorMap,  in copyStore()
1187                                  ScalarMaps, getLoopForStmt(Stmt));  in copyStore()
1198     Value *NewPointer = generateLocationAccessed(Stmt, Store, ScalarMaps[0],  in copyStore()
1210       Value *NewPointer = generateLocationAccessed(Stmt, Store, ScalarMaps[i],  in copyStore()
1258     ScopStmt &Stmt, Instruction *Inst, ValueMapT &VectorMap,  in copyInstScalarized()  argument
1266     BlockGenerator::copyInstruction(Stmt, Inst, ScalarMaps[VectorLane],  in copyInstScalarized()
1286     ScopStmt &Stmt, Instruction *Inst, ValueMapT &VectorMap,  in copyInstruction()  argument
1293   if (canSyntheziseInStmt(Stmt, Inst))  in copyInstruction()
1297     generateLoad(Stmt, Load, VectorMap, ScalarMaps, NewAccesses);  in copyInstruction()
1304       if (!Stmt.getArrayAccessOrNULLFor(Store))  in copyInstruction()
1307       copyStore(Stmt, Store, VectorMap, ScalarMaps, NewAccesses);  in copyInstruction()
1312       copyUnaryInst(Stmt, Unary, VectorMap, ScalarMaps);  in copyInstruction()
1317       copyBinaryInst(Stmt, Binary, VectorMap, ScalarMaps);  in copyInstruction()
1325   copyInstScalarized(Stmt, Inst, VectorMap, ScalarMaps, NewAccesses);  in copyInstruction()
1329     ScopStmt &Stmt, ValueMapT &VectorBlockMap) {  in generateScalarVectorLoads()  argument
1330   for (MemoryAccess *MA : Stmt) {  in generateScalarVectorLoads()
1351 void VectorBlockGenerator::verifyNoScalarStores(ScopStmt &Stmt) {  in verifyNoScalarStores()  argument
1352   for (MemoryAccess *MA : Stmt) {  in verifyNoScalarStores()
1361     ScopStmt &Stmt, __isl_keep isl_id_to_ast_expr *NewAccesses) {  in copyStmt()  argument
1362   assert(Stmt.isBlockStmt() &&  in copyStmt()
1366   BasicBlock *BB = Stmt.getBasicBlock();  in copyStmt()
1389   generateScalarVectorLoads(Stmt, VectorBlockMap);  in copyStmt()
1391   for (Instruction *Inst : Stmt.getInstructions())  in copyStmt()
1392     copyInstruction(Stmt, Inst, VectorBlockMap, ScalarBlockMap, NewAccesses);  in copyStmt()
1394   verifyNoScalarStores(Stmt);  in copyStmt()
1453 void RegionGenerator::copyStmt(ScopStmt &Stmt, LoopToScevMapT &LTS,  in copyStmt()  argument
1455   assert(Stmt.isRegionStmt() &&  in copyStmt()
1468   Region *R = Stmt.getRegion();  in copyStmt()
1479   generateScalarLoads(Stmt, LTS, EntryBBMap, IdToAstExp);  in copyStmt()
1480   generateBeginStmtTrace(Stmt, LTS, EntryBBMap);  in copyStmt()
1517     copyBB(Stmt, BB, BBCopy, RegionMap, LTS, IdToAstExp);  in copyStmt()
1525       addOperandToPHI(Stmt, PHINodePair.first, PHINodePair.second, BB, LTS);  in copyStmt()
1571     copyInstScalar(Stmt, TI, RegionMap, LTS);  in copyStmt()
1611   generateScalarStores(Stmt, LTS, ValueMap, IdToAstExp);  in copyStmt()
1620   ScopStmt *Stmt = MA->getStatement();  in buildExitPHI()  local
1621   Region *SubR = Stmt->getRegion();  in buildExitPHI()
1652         getNewValue(*Stmt, OrigIncomingValue, *LocalBBMap, LTS, L);  in buildExitPHI()
1661   ScopStmt *Stmt = MA->getStatement();  in getExitScalar()  local
1664   Loop *L = LI.getLoopFor(Stmt->getRegion()->getExit());  in getExitScalar()
1674       return getNewValue(*Stmt, OldVal, BBMap, LTS, L);  in getExitScalar()
1683   return getNewValue(*Stmt, OldVal, BBMap, LTS, L);  in getExitScalar()
1687     ScopStmt &Stmt, LoopToScevMapT &LTS, ValueMapT &BBMap,  in generateScalarStores()  argument
1689   assert(Stmt.getRegion() &&  in generateScalarStores()
1701   for (MemoryAccess *MA : Stmt) {  in generateScalarStores()
1709   for (MemoryAccess *MA : Stmt) {  in generateScalarStores()
1716         Stmt, AccDom, Subject.c_str(), [&, this, MA]() {  in generateScalarStores()
1719           Value *Address = getImplicitAddress(*MA, getLoopForStmt(Stmt), LTS,  in generateScalarStores()
1734 void RegionGenerator::addOperandToPHI(ScopStmt &Stmt, PHINode *PHI,  in addOperandToPHI()  argument
1743     assert(Stmt.represents(IncomingBB) &&  in addOperandToPHI()
1755   if (Stmt.represents(IncomingBB)) {  in addOperandToPHI()
1763     OpCopy = getNewValue(Stmt, Op, BBCopyMap, LTS, getLoopForStmt(Stmt));  in addOperandToPHI()
1775     OpCopy = getNewValue(Stmt, PHI, BBCopyMap, LTS, getLoopForStmt(Stmt));  in addOperandToPHI()
1782 void RegionGenerator::copyPHIInstruction(ScopStmt &Stmt, PHINode *PHI,  in copyPHIInstruction()  argument
1792     addOperandToPHI(Stmt, PHI, PHICopy, IncomingBB, LTS);  in copyPHIInstruction()