387   void scalarizeInstruction(Instruction *Instr, const VPIteration &Instance,
429   void vectorizeInterleaveGroup(Instruction *Instr,
434   void vectorizeMemoryInstruction(Instruction *Instr,
1125   bool isAccessInterleaved(Instruction *Instr) {  in isAccessInterleaved()  argument
1126     return InterleaveInfo.isInterleaved(Instr);  in isAccessInterleaved()
1131   getInterleavedAccessGroup(Instruction *Instr) {  in getInterleavedAccessGroup()  argument
1132     return InterleaveInfo.getInterleaveGroup(Instr);  in getInterleavedAccessGroup()
1495   Instruction *Instr = dyn_cast<Instruction>(V);  in getBroadcastInstrs()  local
1497                      (!Instr ||  in getBroadcastInstrs()
1498                       DT->dominates(Instr->getParent(), LoopVectorPreHeader));  in getBroadcastInstrs()
1994 void InnerLoopVectorizer::vectorizeInterleaveGroup(Instruction *Instr,  in vectorizeInterleaveGroup()  argument
1997       Cost->getInterleavedAccessGroup(Instr);  in vectorizeInterleaveGroup()
2001   if (Instr != Group->getInsertPos())  in vectorizeInterleaveGroup()
2004   const DataLayout &DL = Instr->getModule()->getDataLayout();  in vectorizeInterleaveGroup()
2005   Value *Ptr = getLoadStorePointerOperand(Instr);  in vectorizeInterleaveGroup()
2008   Type *ScalarTy = getMemInstValueType(Instr);  in vectorizeInterleaveGroup()
2011   Type *PtrTy = VecTy->getPointerTo(getLoadStoreAddressSpace(Instr));  in vectorizeInterleaveGroup()
2016   unsigned Index = Group->getIndex(Instr);  in vectorizeInterleaveGroup()
2062   setDebugLocFromInst(Builder, Instr);  in vectorizeInterleaveGroup()
2072   if (isa<LoadInst>(Instr)) {  in vectorizeInterleaveGroup()
2181 void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr,  in vectorizeMemoryInstruction()  argument
2184   LoadInst *LI = dyn_cast<LoadInst>(Instr);  in vectorizeMemoryInstruction()
2185   StoreInst *SI = dyn_cast<StoreInst>(Instr);  in vectorizeMemoryInstruction()
2190       Cost->getWideningDecision(Instr, VF);  in vectorizeMemoryInstruction()
2194     return vectorizeInterleaveGroup(Instr);  in vectorizeMemoryInstruction()
2196   Type *ScalarDataTy = getMemInstValueType(Instr);  in vectorizeMemoryInstruction()
2198   Value *Ptr = getLoadStorePointerOperand(Instr);  in vectorizeMemoryInstruction()
2199   unsigned Alignment = getLoadStoreAlignment(Instr);  in vectorizeMemoryInstruction()
2202   const DataLayout &DL = Instr->getModule()->getDataLayout();  in vectorizeMemoryInstruction()
2205   unsigned AddressSpace = getLoadStoreAddressSpace(Instr);  in vectorizeMemoryInstruction()
2231           getLoadStorePointerOperand(Instr)->stripPointerCasts()))  in vectorizeMemoryInstruction()
2315     VectorLoopValueMap.setVectorValue(Instr, Part, NewLI);  in vectorizeMemoryInstruction()
2319 void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr,  in scalarizeInstruction()  argument
2322   assert(!Instr->getType()->isAggregateType() && "Can't handle vectors");  in scalarizeInstruction()
2324   setDebugLocFromInst(Builder, Instr);  in scalarizeInstruction()
2327   bool IsVoidRetTy = Instr->getType()->isVoidTy();  in scalarizeInstruction()
2329   Instruction *Cloned = Instr->clone();  in scalarizeInstruction()
2331     Cloned->setName(Instr->getName() + ".cloned");  in scalarizeInstruction()
2335   for (unsigned op = 0, e = Instr->getNumOperands(); op != e; ++op) {  in scalarizeInstruction()
2336     auto *NewOp = getOrCreateScalarValue(Instr->getOperand(op), Instance);  in scalarizeInstruction()
2339   addNewMetadata(Cloned, Instr);  in scalarizeInstruction()
2345   VectorLoopValueMap.setScalarValue(Instr, Instance, Cloned);  in scalarizeInstruction()
5105         auto *Instr = dyn_cast<Instruction>(U);  in calculateRegisterUsage()  local
5108         if (!Instr)  in calculateRegisterUsage()
5112         if (!TheLoop->contains(Instr)) {  in calculateRegisterUsage()
5113           LoopInvariants.insert(Instr);  in calculateRegisterUsage()
5118         EndPoint[Instr] = IdxToInstr.size();  in calculateRegisterUsage()
5119         Ends.insert(Instr);  in calculateRegisterUsage()
6653 VPRegionBlock *VPRecipeBuilder::createReplicateRegion(Instruction *Instr,  in createReplicateRegion()  argument
6660   VPValue *BlockInMask = createBlockInMask(Instr->getParent(), Plan);  in createReplicateRegion()
6663   std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str();  in createReplicateRegion()
6664   assert(Instr->getParent() && "Predicated instruction not in any basic block");  in createReplicateRegion()
6668       Instr->getType()->isVoidTy() ? nullptr : new VPPredInstPHIRecipe(Instr);  in createReplicateRegion()
6681 bool VPRecipeBuilder::tryToCreateRecipe(Instruction *Instr, VFRange &Range,  in tryToCreateRecipe()  argument
6686   if ((Recipe = tryToInterleaveMemory(Instr, Range, Plan))) {  in tryToCreateRecipe()
6692   if ((Recipe = tryToWidenMemory(Instr, Range, Plan))) {  in tryToCreateRecipe()
6698   if ((Recipe = tryToOptimizeInduction(Instr, Range))) {  in tryToCreateRecipe()
6702   if ((Recipe = tryToBlend(Instr, Plan))) {  in tryToCreateRecipe()
6706   if (PHINode *Phi = dyn_cast<PHINode>(Instr)) {  in tryToCreateRecipe()
6714   if (tryToWiden(Instr, VPBB, Range))  in tryToCreateRecipe()
6799       Instruction *Instr = &I;  in buildVPlanWithVPRecipes()  local
6803       if (isa<BranchInst>(Instr) ||  in buildVPlanWithVPRecipes()
6804           DeadInstructions.find(Instr) != DeadInstructions.end())  in buildVPlanWithVPRecipes()
6810           CM.getInterleavedAccessGroup(Instr);  in buildVPlanWithVPRecipes()
6811       if (IG && Instr != IG->getInsertPos() &&  in buildVPlanWithVPRecipes()
6813           CM.getWideningDecision(Instr, Range.Start) ==  in buildVPlanWithVPRecipes()
6815         auto SinkCandidate = SinkAfterInverse.find(Instr);  in buildVPlanWithVPRecipes()
6824       auto SAIt = SinkAfter.find(Instr);  in buildVPlanWithVPRecipes()
6829         SinkAfterInverse[SAIt->second] = Instr;  in buildVPlanWithVPRecipes()
6833       Ingredients.push_back(Instr);  in buildVPlanWithVPRecipes()
6837       auto SAInvIt = SinkAfterInverse.find(Instr);  in buildVPlanWithVPRecipes()
6843     for (Instruction *Instr : Ingredients) {  in buildVPlanWithVPRecipes()
6844       if (RecipeBuilder.tryToCreateRecipe(Instr, Range, Plan, VPBB))  in buildVPlanWithVPRecipes()
6850           Instr, Range, VPBB, PredInst2Recipe, Plan);  in buildVPlanWithVPRecipes()
6931   for (auto &Instr : make_range(Begin, End))  in execute()  local
6932     State.ILV->widenInstruction(Instr);  in execute()
7081     return State.ILV->vectorizeMemoryInstruction(&Instr);  in execute()
7088   State.ILV->vectorizeMemoryInstruction(&Instr, &MaskValues);  in execute()