1c00cb762STyker //===- AssumeBundleBuilder.cpp - tools to preserve informations -*- C++ -*-===//
2c00cb762STyker //
3c00cb762STyker // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4c00cb762STyker // See https://llvm.org/LICENSE.txt for license information.
5c00cb762STyker // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6c00cb762STyker //
7c00cb762STyker //===----------------------------------------------------------------------===//
8c00cb762STyker 
9c00cb762STyker #include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
10da100de0SOCHyams #include "llvm/ADT/DepthFirstIterator.h"
115957e058STyker #include "llvm/ADT/MapVector.h"
1216f777f4STyker #include "llvm/ADT/Statistic.h"
13c00cb762STyker #include "llvm/Analysis/AssumeBundleQueries.h"
14813f438bSTyker #include "llvm/Analysis/AssumptionCache.h"
15821a0f23STyker #include "llvm/Analysis/ValueTracking.h"
16821a0f23STyker #include "llvm/IR/Dominators.h"
17c00cb762STyker #include "llvm/IR/Function.h"
18c00cb762STyker #include "llvm/IR/InstIterator.h"
19c00cb762STyker #include "llvm/IR/IntrinsicInst.h"
20c00cb762STyker #include "llvm/IR/Module.h"
21*a494ae43Sserge-sans-paille #include "llvm/IR/Operator.h"
22da100de0SOCHyams #include "llvm/InitializePasses.h"
23c00cb762STyker #include "llvm/Support/CommandLine.h"
2416f777f4STyker #include "llvm/Support/DebugCounter.h"
25da100de0SOCHyams #include "llvm/Transforms/Utils/Local.h"
26c00cb762STyker 
27c00cb762STyker using namespace llvm;
28c00cb762STyker 
29d8aba75aSFangrui Song namespace llvm {
30c00cb762STyker cl::opt<bool> ShouldPreserveAllAttributes(
31c00cb762STyker     "assume-preserve-all", cl::init(false), cl::Hidden,
32c00cb762STyker     cl::desc("enable preservation of all attrbitues. even those that are "
33c00cb762STyker              "unlikely to be usefull"));
34c00cb762STyker 
35c00cb762STyker cl::opt<bool> EnableKnowledgeRetention(
36c00cb762STyker     "enable-knowledge-retention", cl::init(false), cl::Hidden,
37c00cb762STyker     cl::desc(
38c00cb762STyker         "enable preservation of attributes throughout code transformation"));
39d8aba75aSFangrui Song } // namespace llvm
40c00cb762STyker 
41642e9225STyker #define DEBUG_TYPE "assume-builder"
42642e9225STyker 
4316f777f4STyker STATISTIC(NumAssumeBuilt, "Number of assume built by the assume builder");
4416f777f4STyker STATISTIC(NumBundlesInAssumes, "Total number of Bundles in the assume built");
4516f777f4STyker STATISTIC(NumAssumesMerged,
4616f777f4STyker           "Number of assume merged by the assume simplify pass");
4716f777f4STyker STATISTIC(NumAssumesRemoved,
4816f777f4STyker           "Number of assume removed by the assume simplify pass");
4916f777f4STyker 
5016f777f4STyker DEBUG_COUNTER(BuildAssumeCounter, "assume-builder-counter",
5116f777f4STyker               "Controls which assumes gets created");
5216f777f4STyker 
53c00cb762STyker namespace {
54c00cb762STyker 
isUsefullToPreserve(Attribute::AttrKind Kind)55c00cb762STyker bool isUsefullToPreserve(Attribute::AttrKind Kind) {
56c00cb762STyker   switch (Kind) {
57c00cb762STyker     case Attribute::NonNull:
589b3c2a72SJuneyoung Lee     case Attribute::NoUndef:
59c00cb762STyker     case Attribute::Alignment:
60c00cb762STyker     case Attribute::Dereferenceable:
61c00cb762STyker     case Attribute::DereferenceableOrNull:
62c00cb762STyker     case Attribute::Cold:
63c00cb762STyker       return true;
64c00cb762STyker     default:
65c00cb762STyker       return false;
66c00cb762STyker   }
67c00cb762STyker }
68c00cb762STyker 
69b7338fb1STyker /// This function will try to transform the given knowledge into a more
70b7338fb1STyker /// canonical one. the canonical knowledge maybe the given one.
canonicalizedKnowledge(RetainedKnowledge RK,const DataLayout & DL)715f2c53bdSSimon Pilgrim RetainedKnowledge canonicalizedKnowledge(RetainedKnowledge RK,
725f2c53bdSSimon Pilgrim                                          const DataLayout &DL) {
73b7338fb1STyker   switch (RK.AttrKind) {
74b7338fb1STyker   default:
75b7338fb1STyker     return RK;
76b7338fb1STyker   case Attribute::NonNull:
77b0eb40caSVitaly Buka     RK.WasOn = getUnderlyingObject(RK.WasOn);
78b7338fb1STyker     return RK;
79b7338fb1STyker   case Attribute::Alignment: {
80b7338fb1STyker     Value *V = RK.WasOn->stripInBoundsOffsets([&](const Value *Strip) {
81b7338fb1STyker       if (auto *GEP = dyn_cast<GEPOperator>(Strip))
82b7338fb1STyker         RK.ArgValue =
83642e9225STyker             MinAlign(RK.ArgValue, GEP->getMaxPreservedAlignment(DL).value());
84b7338fb1STyker     });
85b7338fb1STyker     RK.WasOn = V;
86b7338fb1STyker     return RK;
87b7338fb1STyker   }
88b7338fb1STyker   case Attribute::Dereferenceable:
89b7338fb1STyker   case Attribute::DereferenceableOrNull: {
90b7338fb1STyker     int64_t Offset = 0;
91642e9225STyker     Value *V = GetPointerBaseWithConstantOffset(RK.WasOn, Offset, DL,
92642e9225STyker                                                 /*AllowNonInBounds*/ false);
93b7338fb1STyker     if (Offset < 0)
94b7338fb1STyker       return RK;
95b7338fb1STyker     RK.ArgValue = RK.ArgValue + Offset;
96b7338fb1STyker     RK.WasOn = V;
97b7338fb1STyker   }
98b7338fb1STyker   }
99b7338fb1STyker   return RK;
100b7338fb1STyker }
101b7338fb1STyker 
102c00cb762STyker /// This class contain all knowledge that have been gather while building an
103c00cb762STyker /// llvm.assume and the function to manipulate it.
104c00cb762STyker struct AssumeBuilderState {
105c00cb762STyker   Module *M;
106c00cb762STyker 
107e5f8a77cSTyker   using MapKey = std::pair<Value *, Attribute::AttrKind>;
1083628bb74SArthur Eubanks   SmallMapVector<MapKey, uint64_t, 8> AssumedKnowledgeMap;
109642e9225STyker   Instruction *InstBeingModified = nullptr;
110821a0f23STyker   AssumptionCache* AC = nullptr;
111821a0f23STyker   DominatorTree* DT = nullptr;
112c00cb762STyker 
AssumeBuilderState__anonda79dfad0111::AssumeBuilderState113821a0f23STyker   AssumeBuilderState(Module *M, Instruction *I = nullptr,
114821a0f23STyker                      AssumptionCache *AC = nullptr, DominatorTree *DT = nullptr)
115642e9225STyker       : M(M), InstBeingModified(I), AC(AC), DT(DT) {}
116821a0f23STyker 
tryToPreserveWithoutAddingAssume__anonda79dfad0111::AssumeBuilderState117821a0f23STyker   bool tryToPreserveWithoutAddingAssume(RetainedKnowledge RK) {
118642e9225STyker     if (!InstBeingModified || !RK.WasOn)
119821a0f23STyker       return false;
120821a0f23STyker     bool HasBeenPreserved = false;
121821a0f23STyker     Use* ToUpdate = nullptr;
122821a0f23STyker     getKnowledgeForValue(
123821a0f23STyker         RK.WasOn, {RK.AttrKind}, AC,
124821a0f23STyker         [&](RetainedKnowledge RKOther, Instruction *Assume,
125821a0f23STyker             const CallInst::BundleOpInfo *Bundle) {
126642e9225STyker           if (!isValidAssumeForContext(Assume, InstBeingModified, DT))
127821a0f23STyker             return false;
128821a0f23STyker           if (RKOther.ArgValue >= RK.ArgValue) {
129821a0f23STyker             HasBeenPreserved = true;
130821a0f23STyker             return true;
131642e9225STyker           } else if (isValidAssumeForContext(InstBeingModified, Assume, DT)) {
132821a0f23STyker             HasBeenPreserved = true;
133821a0f23STyker             IntrinsicInst *Intr = cast<IntrinsicInst>(Assume);
134821a0f23STyker             ToUpdate = &Intr->op_begin()[Bundle->Begin + ABA_Argument];
135821a0f23STyker             return true;
136821a0f23STyker           }
137821a0f23STyker           return false;
138821a0f23STyker         });
139821a0f23STyker     if (ToUpdate)
140821a0f23STyker       ToUpdate->set(
141821a0f23STyker           ConstantInt::get(Type::getInt64Ty(M->getContext()), RK.ArgValue));
142821a0f23STyker     return HasBeenPreserved;
143821a0f23STyker   }
144c00cb762STyker 
isKnowledgeWorthPreserving__anonda79dfad0111::AssumeBuilderState145b7338fb1STyker   bool isKnowledgeWorthPreserving(RetainedKnowledge RK) {
146b7338fb1STyker     if (!RK)
147b7338fb1STyker       return false;
148b7338fb1STyker     if (!RK.WasOn)
149b7338fb1STyker       return true;
150b7338fb1STyker     if (RK.WasOn->getType()->isPointerTy()) {
151b0eb40caSVitaly Buka       Value *UnderlyingPtr = getUnderlyingObject(RK.WasOn);
152b7338fb1STyker       if (isa<AllocaInst>(UnderlyingPtr) || isa<GlobalValue>(UnderlyingPtr))
153b7338fb1STyker         return false;
154b7338fb1STyker     }
155b7338fb1STyker     if (auto *Arg = dyn_cast<Argument>(RK.WasOn)) {
156b7338fb1STyker       if (Arg->hasAttribute(RK.AttrKind) &&
1576ac32872SNikita Popov           (!Attribute::isIntAttrKind(RK.AttrKind) ||
158b7338fb1STyker            Arg->getAttribute(RK.AttrKind).getValueAsInt() >= RK.ArgValue))
159b7338fb1STyker         return false;
160b7338fb1STyker       return true;
161b7338fb1STyker     }
162b7338fb1STyker     if (auto *Inst = dyn_cast<Instruction>(RK.WasOn))
163b7338fb1STyker       if (wouldInstructionBeTriviallyDead(Inst)) {
164b7338fb1STyker         if (RK.WasOn->use_empty())
165b7338fb1STyker           return false;
166b6cb03e6SAnna Thomas         Use *SingleUse = RK.WasOn->getSingleUndroppableUse();
167b6cb03e6SAnna Thomas         if (SingleUse && SingleUse->getUser() == InstBeingModified)
168b7338fb1STyker           return false;
169b7338fb1STyker       }
170b7338fb1STyker     return true;
171b7338fb1STyker   }
172b7338fb1STyker 
addKnowledge__anonda79dfad0111::AssumeBuilderState173e5f8a77cSTyker   void addKnowledge(RetainedKnowledge RK) {
174642e9225STyker     RK = canonicalizedKnowledge(RK, M->getDataLayout());
175b7338fb1STyker 
176b7338fb1STyker     if (!isKnowledgeWorthPreserving(RK))
177b7338fb1STyker       return;
178b7338fb1STyker 
179b7338fb1STyker     if (tryToPreserveWithoutAddingAssume(RK))
180821a0f23STyker       return;
181e5f8a77cSTyker     MapKey Key{RK.WasOn, RK.AttrKind};
182e5f8a77cSTyker     auto Lookup = AssumedKnowledgeMap.find(Key);
183e5f8a77cSTyker     if (Lookup == AssumedKnowledgeMap.end()) {
184e5f8a77cSTyker       AssumedKnowledgeMap[Key] = RK.ArgValue;
185e5f8a77cSTyker       return;
186e5f8a77cSTyker     }
187e5f8a77cSTyker     assert(((Lookup->second == 0 && RK.ArgValue == 0) ||
188e5f8a77cSTyker             (Lookup->second != 0 && RK.ArgValue != 0)) &&
189e5f8a77cSTyker            "inconsistent argument value");
190e5f8a77cSTyker 
191e5f8a77cSTyker     /// This is only desirable because for all attributes taking an argument
192e5f8a77cSTyker     /// higher is better.
193e5f8a77cSTyker     Lookup->second = std::max(Lookup->second, RK.ArgValue);
194e5f8a77cSTyker   }
195e5f8a77cSTyker 
addAttribute__anonda79dfad0111::AssumeBuilderState196c00cb762STyker   void addAttribute(Attribute Attr, Value *WasOn) {
197e5f8a77cSTyker     if (Attr.isTypeAttribute() || Attr.isStringAttribute() ||
198e5f8a77cSTyker         (!ShouldPreserveAllAttributes &&
199c00cb762STyker          !isUsefullToPreserve(Attr.getKindAsEnum())))
200c00cb762STyker       return;
2013628bb74SArthur Eubanks     uint64_t AttrArg = 0;
202c00cb762STyker     if (Attr.isIntAttribute())
203e5f8a77cSTyker       AttrArg = Attr.getValueAsInt();
204e5f8a77cSTyker     addKnowledge({Attr.getKindAsEnum(), AttrArg, WasOn});
205c00cb762STyker   }
206c00cb762STyker 
addCall__anonda79dfad0111::AssumeBuilderState207c00cb762STyker   void addCall(const CallBase *Call) {
208a9831cceSArthur Eubanks     auto addAttrList = [&](AttributeList AttrList, unsigned NumArgs) {
209a9831cceSArthur Eubanks       for (unsigned Idx = 0; Idx < NumArgs; Idx++)
210a9831cceSArthur Eubanks         for (Attribute Attr : AttrList.getParamAttrs(Idx)) {
211edf634ebSJuneyoung Lee           bool IsPoisonAttr = Attr.hasAttribute(Attribute::NonNull) ||
212edf634ebSJuneyoung Lee                               Attr.hasAttribute(Attribute::Alignment);
213a9831cceSArthur Eubanks           if (!IsPoisonAttr || Call->isPassingUndefUB(Idx))
214a9831cceSArthur Eubanks             addAttribute(Attr, Call->getArgOperand(Idx));
215edf634ebSJuneyoung Lee         }
21680ea2bb5SArthur Eubanks       for (Attribute Attr : AttrList.getFnAttrs())
217c00cb762STyker         addAttribute(Attr, nullptr);
218c00cb762STyker     };
219a9831cceSArthur Eubanks     addAttrList(Call->getAttributes(), Call->arg_size());
220c00cb762STyker     if (Function *Fn = Call->getCalledFunction())
221a9831cceSArthur Eubanks       addAttrList(Fn->getAttributes(), Fn->arg_size());
222c00cb762STyker   }
223c00cb762STyker 
build__anonda79dfad0111::AssumeBuilderState2249ef6aa02SPhilip Reames   AssumeInst *build() {
225e5f8a77cSTyker     if (AssumedKnowledgeMap.empty())
226c00cb762STyker       return nullptr;
22716f777f4STyker     if (!DebugCounter::shouldExecute(BuildAssumeCounter))
22816f777f4STyker       return nullptr;
229c00cb762STyker     Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
230c00cb762STyker     LLVMContext &C = M->getContext();
231c00cb762STyker     SmallVector<OperandBundleDef, 8> OpBundle;
232e5f8a77cSTyker     for (auto &MapElem : AssumedKnowledgeMap) {
233c00cb762STyker       SmallVector<Value *, 2> Args;
234e5f8a77cSTyker       if (MapElem.first.first)
235e5f8a77cSTyker         Args.push_back(MapElem.first.first);
236e5f8a77cSTyker 
237e5f8a77cSTyker       /// This is only valid because for all attribute that currently exist a
238e5f8a77cSTyker       /// value of 0 is useless. and should not be preserved.
239e5f8a77cSTyker       if (MapElem.second)
240e5f8a77cSTyker         Args.push_back(ConstantInt::get(Type::getInt64Ty(M->getContext()),
241e5f8a77cSTyker                                         MapElem.second));
242e5f8a77cSTyker       OpBundle.push_back(OperandBundleDefT<Value *>(
243e5f8a77cSTyker           std::string(Attribute::getNameFromAttrKind(MapElem.first.second)),
244e5f8a77cSTyker           Args));
24516f777f4STyker       NumBundlesInAssumes++;
246c00cb762STyker     }
24716f777f4STyker     NumAssumeBuilt++;
2489ef6aa02SPhilip Reames     return cast<AssumeInst>(CallInst::Create(
249c00cb762STyker         FnAssume, ArrayRef<Value *>({ConstantInt::getTrue(C)}), OpBundle));
250c00cb762STyker   }
251c00cb762STyker 
addAccessedPtr__anonda79dfad0111::AssumeBuilderState252c00cb762STyker   void addAccessedPtr(Instruction *MemInst, Value *Pointer, Type *AccType,
253c00cb762STyker                       MaybeAlign MA) {
254e5f8a77cSTyker     unsigned DerefSize = MemInst->getModule()
255c00cb762STyker                              ->getDataLayout()
256c00cb762STyker                              .getTypeStoreSize(AccType)
257c00cb762STyker                              .getKnownMinSize();
258c00cb762STyker     if (DerefSize != 0) {
259e5f8a77cSTyker       addKnowledge({Attribute::Dereferenceable, DerefSize, Pointer});
260c00cb762STyker       if (!NullPointerIsDefined(MemInst->getFunction(),
261c00cb762STyker                                 Pointer->getType()->getPointerAddressSpace()))
262e5f8a77cSTyker         addKnowledge({Attribute::NonNull, 0u, Pointer});
263c00cb762STyker     }
264c00cb762STyker     if (MA.valueOrOne() > 1)
2653628bb74SArthur Eubanks       addKnowledge({Attribute::Alignment, MA.valueOrOne().value(), Pointer});
266c00cb762STyker   }
267c00cb762STyker 
addInstruction__anonda79dfad0111::AssumeBuilderState268c00cb762STyker   void addInstruction(Instruction *I) {
269c00cb762STyker     if (auto *Call = dyn_cast<CallBase>(I))
270c00cb762STyker       return addCall(Call);
271c00cb762STyker     if (auto *Load = dyn_cast<LoadInst>(I))
272c00cb762STyker       return addAccessedPtr(I, Load->getPointerOperand(), Load->getType(),
273c00cb762STyker                             Load->getAlign());
274c00cb762STyker     if (auto *Store = dyn_cast<StoreInst>(I))
275c00cb762STyker       return addAccessedPtr(I, Store->getPointerOperand(),
276c00cb762STyker                             Store->getValueOperand()->getType(),
277c00cb762STyker                             Store->getAlign());
278c00cb762STyker     // TODO: Add support for the other Instructions.
279c00cb762STyker     // TODO: Maybe we should look around and merge with other llvm.assume.
280c00cb762STyker   }
281c00cb762STyker };
282c00cb762STyker 
283c00cb762STyker } // namespace
284c00cb762STyker 
buildAssumeFromInst(Instruction * I)2859ef6aa02SPhilip Reames AssumeInst *llvm::buildAssumeFromInst(Instruction *I) {
286c00cb762STyker   if (!EnableKnowledgeRetention)
287c00cb762STyker     return nullptr;
288c00cb762STyker   AssumeBuilderState Builder(I->getModule());
289c00cb762STyker   Builder.addInstruction(I);
290c00cb762STyker   return Builder.build();
291c00cb762STyker }
292c00cb762STyker 
salvageKnowledge(Instruction * I,AssumptionCache * AC,DominatorTree * DT)29378d85c20STyker void llvm::salvageKnowledge(Instruction *I, AssumptionCache *AC,
29478d85c20STyker                             DominatorTree *DT) {
29578d85c20STyker   if (!EnableKnowledgeRetention || I->isTerminator())
296e5f8a77cSTyker     return;
297821a0f23STyker   AssumeBuilderState Builder(I->getModule(), I, AC, DT);
298e5f8a77cSTyker   Builder.addInstruction(I);
2999ef6aa02SPhilip Reames   if (auto *Intr = Builder.build()) {
300c00cb762STyker     Intr->insertBefore(I);
301813f438bSTyker     if (AC)
3029ef6aa02SPhilip Reames       AC->registerAssumption(Intr);
303813f438bSTyker   }
304c00cb762STyker }
305c00cb762STyker 
3069ef6aa02SPhilip Reames AssumeInst *
buildAssumeFromKnowledge(ArrayRef<RetainedKnowledge> Knowledge,Instruction * CtxI,AssumptionCache * AC,DominatorTree * DT)307642e9225STyker llvm::buildAssumeFromKnowledge(ArrayRef<RetainedKnowledge> Knowledge,
308642e9225STyker                                Instruction *CtxI, AssumptionCache *AC,
309642e9225STyker                                DominatorTree *DT) {
310642e9225STyker   AssumeBuilderState Builder(CtxI->getModule(), CtxI, AC, DT);
311642e9225STyker   for (const RetainedKnowledge &RK : Knowledge)
312642e9225STyker     Builder.addKnowledge(RK);
313642e9225STyker   return Builder.build();
314642e9225STyker }
315642e9225STyker 
simplifyRetainedKnowledge(AssumeInst * Assume,RetainedKnowledge RK,AssumptionCache * AC,DominatorTree * DT)3169ef6aa02SPhilip Reames RetainedKnowledge llvm::simplifyRetainedKnowledge(AssumeInst *Assume,
317642e9225STyker                                                   RetainedKnowledge RK,
318642e9225STyker                                                   AssumptionCache *AC,
319642e9225STyker                                                   DominatorTree *DT) {
320642e9225STyker   AssumeBuilderState Builder(Assume->getModule(), Assume, AC, DT);
321642e9225STyker   RK = canonicalizedKnowledge(RK, Assume->getModule()->getDataLayout());
322642e9225STyker 
323642e9225STyker   if (!Builder.isKnowledgeWorthPreserving(RK))
324642e9225STyker     return RetainedKnowledge::none();
325642e9225STyker 
326642e9225STyker   if (Builder.tryToPreserveWithoutAddingAssume(RK))
327642e9225STyker     return RetainedKnowledge::none();
328642e9225STyker   return RK;
329642e9225STyker }
330642e9225STyker 
331da100de0SOCHyams namespace {
332da100de0SOCHyams 
333da100de0SOCHyams struct AssumeSimplify {
334da100de0SOCHyams   Function &F;
335da100de0SOCHyams   AssumptionCache &AC;
336da100de0SOCHyams   DominatorTree *DT;
337da100de0SOCHyams   LLVMContext &C;
338da100de0SOCHyams   SmallDenseSet<IntrinsicInst *> CleanupToDo;
339da100de0SOCHyams   StringMapEntry<uint32_t> *IgnoreTag;
340da100de0SOCHyams   SmallDenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 4>, 8> BBToAssume;
341da100de0SOCHyams   bool MadeChange = false;
342da100de0SOCHyams 
AssumeSimplify__anonda79dfad0511::AssumeSimplify343da100de0SOCHyams   AssumeSimplify(Function &F, AssumptionCache &AC, DominatorTree *DT,
344da100de0SOCHyams                  LLVMContext &C)
345da100de0SOCHyams       : F(F), AC(AC), DT(DT), C(C),
346da100de0SOCHyams         IgnoreTag(C.getOrInsertBundleTag(IgnoreBundleTag)) {}
347da100de0SOCHyams 
buildMapping__anonda79dfad0511::AssumeSimplify348da100de0SOCHyams   void buildMapping(bool FilterBooleanArgument) {
349da100de0SOCHyams     BBToAssume.clear();
350da100de0SOCHyams     for (Value *V : AC.assumptions()) {
351da100de0SOCHyams       if (!V)
352da100de0SOCHyams         continue;
353da100de0SOCHyams       IntrinsicInst *Assume = cast<IntrinsicInst>(V);
354da100de0SOCHyams       if (FilterBooleanArgument) {
355da100de0SOCHyams         auto *Arg = dyn_cast<ConstantInt>(Assume->getOperand(0));
356da100de0SOCHyams         if (!Arg || Arg->isZero())
357da100de0SOCHyams           continue;
358da100de0SOCHyams       }
359da100de0SOCHyams       BBToAssume[Assume->getParent()].push_back(Assume);
360da100de0SOCHyams     }
361da100de0SOCHyams 
362da100de0SOCHyams     for (auto &Elem : BBToAssume) {
363da100de0SOCHyams       llvm::sort(Elem.second,
364da100de0SOCHyams                  [](const IntrinsicInst *LHS, const IntrinsicInst *RHS) {
365da100de0SOCHyams                    return LHS->comesBefore(RHS);
366da100de0SOCHyams                  });
367da100de0SOCHyams     }
368da100de0SOCHyams   }
369da100de0SOCHyams 
370da100de0SOCHyams   /// Remove all asumes in CleanupToDo if there boolean argument is true and
371da100de0SOCHyams   /// ForceCleanup is set or the assume doesn't hold valuable knowledge.
RunCleanup__anonda79dfad0511::AssumeSimplify372da100de0SOCHyams   void RunCleanup(bool ForceCleanup) {
373da100de0SOCHyams     for (IntrinsicInst *Assume : CleanupToDo) {
374da100de0SOCHyams       auto *Arg = dyn_cast<ConstantInt>(Assume->getOperand(0));
375da100de0SOCHyams       if (!Arg || Arg->isZero() ||
3769ef6aa02SPhilip Reames           (!ForceCleanup &&
3779ef6aa02SPhilip Reames            !isAssumeWithEmptyBundle(cast<AssumeInst>(*Assume))))
378da100de0SOCHyams         continue;
379da100de0SOCHyams       MadeChange = true;
38016f777f4STyker       if (ForceCleanup)
38116f777f4STyker         NumAssumesMerged++;
38216f777f4STyker       else
38316f777f4STyker         NumAssumesRemoved++;
384da100de0SOCHyams       Assume->eraseFromParent();
385da100de0SOCHyams     }
386da100de0SOCHyams     CleanupToDo.clear();
387da100de0SOCHyams   }
388da100de0SOCHyams 
389da100de0SOCHyams   /// Remove knowledge stored in assume when it is already know by an attribute
390da100de0SOCHyams   /// or an other assume. This can when valid update an existing knowledge in an
391da100de0SOCHyams   /// attribute or an other assume.
dropRedundantKnowledge__anonda79dfad0511::AssumeSimplify392da100de0SOCHyams   void dropRedundantKnowledge() {
393da100de0SOCHyams     struct MapValue {
394da100de0SOCHyams       IntrinsicInst *Assume;
3953628bb74SArthur Eubanks       uint64_t ArgValue;
396da100de0SOCHyams       CallInst::BundleOpInfo *BOI;
397da100de0SOCHyams     };
398da100de0SOCHyams     buildMapping(false);
399da100de0SOCHyams     SmallDenseMap<std::pair<Value *, Attribute::AttrKind>,
400da100de0SOCHyams                   SmallVector<MapValue, 2>, 16>
401da100de0SOCHyams         Knowledge;
402da100de0SOCHyams     for (BasicBlock *BB : depth_first(&F))
403da100de0SOCHyams       for (Value *V : BBToAssume[BB]) {
404da100de0SOCHyams         if (!V)
405da100de0SOCHyams           continue;
406da100de0SOCHyams         IntrinsicInst *Assume = cast<IntrinsicInst>(V);
407da100de0SOCHyams         for (CallInst::BundleOpInfo &BOI : Assume->bundle_op_infos()) {
408da100de0SOCHyams           auto RemoveFromAssume = [&]() {
409da100de0SOCHyams             CleanupToDo.insert(Assume);
410da100de0SOCHyams             if (BOI.Begin != BOI.End) {
411da100de0SOCHyams               Use *U = &Assume->op_begin()[BOI.Begin + ABA_WasOn];
412da100de0SOCHyams               U->set(UndefValue::get(U->get()->getType()));
413da100de0SOCHyams             }
414da100de0SOCHyams             BOI.Tag = IgnoreTag;
415da100de0SOCHyams           };
416da100de0SOCHyams           if (BOI.Tag == IgnoreTag) {
417da100de0SOCHyams             CleanupToDo.insert(Assume);
418da100de0SOCHyams             continue;
419da100de0SOCHyams           }
4209ef6aa02SPhilip Reames           RetainedKnowledge RK =
4219ef6aa02SPhilip Reames             getKnowledgeFromBundle(cast<AssumeInst>(*Assume), BOI);
422da100de0SOCHyams           if (auto *Arg = dyn_cast_or_null<Argument>(RK.WasOn)) {
423da100de0SOCHyams             bool HasSameKindAttr = Arg->hasAttribute(RK.AttrKind);
424da100de0SOCHyams             if (HasSameKindAttr)
4256ac32872SNikita Popov               if (!Attribute::isIntAttrKind(RK.AttrKind) ||
426da100de0SOCHyams                   Arg->getAttribute(RK.AttrKind).getValueAsInt() >=
427da100de0SOCHyams                       RK.ArgValue) {
428da100de0SOCHyams                 RemoveFromAssume();
429da100de0SOCHyams                 continue;
430da100de0SOCHyams               }
431da100de0SOCHyams             if (isValidAssumeForContext(
432da100de0SOCHyams                     Assume, &*F.getEntryBlock().getFirstInsertionPt()) ||
433da100de0SOCHyams                 Assume == &*F.getEntryBlock().getFirstInsertionPt()) {
434da100de0SOCHyams               if (HasSameKindAttr)
435da100de0SOCHyams                 Arg->removeAttr(RK.AttrKind);
436da100de0SOCHyams               Arg->addAttr(Attribute::get(C, RK.AttrKind, RK.ArgValue));
437da100de0SOCHyams               MadeChange = true;
438da100de0SOCHyams               RemoveFromAssume();
439da100de0SOCHyams               continue;
440da100de0SOCHyams             }
441da100de0SOCHyams           }
442da100de0SOCHyams           auto &Lookup = Knowledge[{RK.WasOn, RK.AttrKind}];
443da100de0SOCHyams           for (MapValue &Elem : Lookup) {
444da100de0SOCHyams             if (!isValidAssumeForContext(Elem.Assume, Assume, DT))
445da100de0SOCHyams               continue;
446da100de0SOCHyams             if (Elem.ArgValue >= RK.ArgValue) {
447da100de0SOCHyams               RemoveFromAssume();
448da100de0SOCHyams               continue;
449da100de0SOCHyams             } else if (isValidAssumeForContext(Assume, Elem.Assume, DT)) {
450da100de0SOCHyams               Elem.Assume->op_begin()[Elem.BOI->Begin + ABA_Argument].set(
451da100de0SOCHyams                   ConstantInt::get(Type::getInt64Ty(C), RK.ArgValue));
452da100de0SOCHyams               MadeChange = true;
453da100de0SOCHyams               RemoveFromAssume();
454da100de0SOCHyams               continue;
455da100de0SOCHyams             }
456da100de0SOCHyams           }
457da100de0SOCHyams           Lookup.push_back({Assume, RK.ArgValue, &BOI});
458da100de0SOCHyams         }
459da100de0SOCHyams       }
460da100de0SOCHyams   }
461da100de0SOCHyams 
462da100de0SOCHyams   using MergeIterator = SmallVectorImpl<IntrinsicInst *>::iterator;
463da100de0SOCHyams 
464da100de0SOCHyams   /// Merge all Assumes from Begin to End in and insert the resulting assume as
465da100de0SOCHyams   /// high as possible in the basicblock.
mergeRange__anonda79dfad0511::AssumeSimplify466da100de0SOCHyams   void mergeRange(BasicBlock *BB, MergeIterator Begin, MergeIterator End) {
467da100de0SOCHyams     if (Begin == End || std::next(Begin) == End)
468da100de0SOCHyams       return;
469da100de0SOCHyams     /// Provide no additional information so that AssumeBuilderState doesn't
470da100de0SOCHyams     /// try to do any punning since it already has been done better.
471da100de0SOCHyams     AssumeBuilderState Builder(F.getParent());
472da100de0SOCHyams 
473da100de0SOCHyams     /// For now it is initialized to the best value it could have
474da100de0SOCHyams     Instruction *InsertPt = BB->getFirstNonPHI();
475da100de0SOCHyams     if (isa<LandingPadInst>(InsertPt))
476da100de0SOCHyams       InsertPt = InsertPt->getNextNode();
477da100de0SOCHyams     for (IntrinsicInst *I : make_range(Begin, End)) {
478da100de0SOCHyams       CleanupToDo.insert(I);
479da100de0SOCHyams       for (CallInst::BundleOpInfo &BOI : I->bundle_op_infos()) {
4809ef6aa02SPhilip Reames         RetainedKnowledge RK =
4819ef6aa02SPhilip Reames           getKnowledgeFromBundle(cast<AssumeInst>(*I), BOI);
482da100de0SOCHyams         if (!RK)
483da100de0SOCHyams           continue;
484da100de0SOCHyams         Builder.addKnowledge(RK);
485da100de0SOCHyams         if (auto *I = dyn_cast_or_null<Instruction>(RK.WasOn))
486da100de0SOCHyams           if (I->getParent() == InsertPt->getParent() &&
487da100de0SOCHyams               (InsertPt->comesBefore(I) || InsertPt == I))
488da100de0SOCHyams             InsertPt = I->getNextNode();
489da100de0SOCHyams       }
490da100de0SOCHyams     }
491da100de0SOCHyams 
492da100de0SOCHyams     /// Adjust InsertPt if it is before Begin, since mergeAssumes only
493da100de0SOCHyams     /// guarantees we can place the resulting assume between Begin and End.
494da100de0SOCHyams     if (InsertPt->comesBefore(*Begin))
495da100de0SOCHyams       for (auto It = (*Begin)->getIterator(), E = InsertPt->getIterator();
496da100de0SOCHyams            It != E; --It)
497da100de0SOCHyams         if (!isGuaranteedToTransferExecutionToSuccessor(&*It)) {
498da100de0SOCHyams           InsertPt = It->getNextNode();
499da100de0SOCHyams           break;
500da100de0SOCHyams         }
5019ef6aa02SPhilip Reames     auto *MergedAssume = Builder.build();
502da100de0SOCHyams     if (!MergedAssume)
503da100de0SOCHyams       return;
504da100de0SOCHyams     MadeChange = true;
505da100de0SOCHyams     MergedAssume->insertBefore(InsertPt);
5069ef6aa02SPhilip Reames     AC.registerAssumption(MergedAssume);
507da100de0SOCHyams   }
508da100de0SOCHyams 
509da100de0SOCHyams   /// Merge assume when they are in the same BasicBlock and for all instruction
510da100de0SOCHyams   /// between them isGuaranteedToTransferExecutionToSuccessor returns true.
mergeAssumes__anonda79dfad0511::AssumeSimplify511da100de0SOCHyams   void mergeAssumes() {
512da100de0SOCHyams     buildMapping(true);
513da100de0SOCHyams 
514da100de0SOCHyams     SmallVector<MergeIterator, 4> SplitPoints;
515da100de0SOCHyams     for (auto &Elem : BBToAssume) {
516da100de0SOCHyams       SmallVectorImpl<IntrinsicInst *> &AssumesInBB = Elem.second;
517da100de0SOCHyams       if (AssumesInBB.size() < 2)
518da100de0SOCHyams         continue;
519da100de0SOCHyams       /// AssumesInBB is already sorted by order in the block.
520da100de0SOCHyams 
521da100de0SOCHyams       BasicBlock::iterator It = AssumesInBB.front()->getIterator();
522da100de0SOCHyams       BasicBlock::iterator E = AssumesInBB.back()->getIterator();
523da100de0SOCHyams       SplitPoints.push_back(AssumesInBB.begin());
524da100de0SOCHyams       MergeIterator LastSplit = AssumesInBB.begin();
525da100de0SOCHyams       for (; It != E; ++It)
526da100de0SOCHyams         if (!isGuaranteedToTransferExecutionToSuccessor(&*It)) {
527da100de0SOCHyams           for (; (*LastSplit)->comesBefore(&*It); ++LastSplit)
528da100de0SOCHyams             ;
529da100de0SOCHyams           if (SplitPoints.back() != LastSplit)
530da100de0SOCHyams             SplitPoints.push_back(LastSplit);
531da100de0SOCHyams         }
532da100de0SOCHyams       SplitPoints.push_back(AssumesInBB.end());
533da100de0SOCHyams       for (auto SplitIt = SplitPoints.begin();
534da100de0SOCHyams            SplitIt != std::prev(SplitPoints.end()); SplitIt++) {
535da100de0SOCHyams         mergeRange(Elem.first, *SplitIt, *(SplitIt + 1));
536da100de0SOCHyams       }
537da100de0SOCHyams       SplitPoints.clear();
538da100de0SOCHyams     }
539da100de0SOCHyams   }
540da100de0SOCHyams };
541da100de0SOCHyams 
simplifyAssumes(Function & F,AssumptionCache * AC,DominatorTree * DT)542da100de0SOCHyams bool simplifyAssumes(Function &F, AssumptionCache *AC, DominatorTree *DT) {
543da100de0SOCHyams   AssumeSimplify AS(F, *AC, DT, F.getContext());
544da100de0SOCHyams 
545da100de0SOCHyams   /// Remove knowledge that is already known by a dominating other assume or an
546da100de0SOCHyams   /// attribute.
547da100de0SOCHyams   AS.dropRedundantKnowledge();
548da100de0SOCHyams 
549da100de0SOCHyams   /// Remove assume that are empty.
550da100de0SOCHyams   AS.RunCleanup(false);
551da100de0SOCHyams 
552da100de0SOCHyams   /// Merge assume in the same basicblock when possible.
553da100de0SOCHyams   AS.mergeAssumes();
554da100de0SOCHyams 
555da100de0SOCHyams   /// Remove assume that were merged.
556da100de0SOCHyams   AS.RunCleanup(true);
557da100de0SOCHyams   return AS.MadeChange;
558da100de0SOCHyams }
559da100de0SOCHyams 
560da100de0SOCHyams } // namespace
561da100de0SOCHyams 
run(Function & F,FunctionAnalysisManager & AM)562da100de0SOCHyams PreservedAnalyses AssumeSimplifyPass::run(Function &F,
563da100de0SOCHyams                                           FunctionAnalysisManager &AM) {
564da100de0SOCHyams   if (!EnableKnowledgeRetention)
565da100de0SOCHyams     return PreservedAnalyses::all();
566da100de0SOCHyams   simplifyAssumes(F, &AM.getResult<AssumptionAnalysis>(F),
567da100de0SOCHyams                   AM.getCachedResult<DominatorTreeAnalysis>(F));
568da100de0SOCHyams   return PreservedAnalyses::all();
569da100de0SOCHyams }
570da100de0SOCHyams 
571350dadaaSBenjamin Kramer namespace {
572da100de0SOCHyams class AssumeSimplifyPassLegacyPass : public FunctionPass {
573da100de0SOCHyams public:
574da100de0SOCHyams   static char ID;
575da100de0SOCHyams 
AssumeSimplifyPassLegacyPass()576da100de0SOCHyams   AssumeSimplifyPassLegacyPass() : FunctionPass(ID) {
577da100de0SOCHyams     initializeAssumeSimplifyPassLegacyPassPass(
578da100de0SOCHyams         *PassRegistry::getPassRegistry());
579da100de0SOCHyams   }
runOnFunction(Function & F)580da100de0SOCHyams   bool runOnFunction(Function &F) override {
581da100de0SOCHyams     if (skipFunction(F) || !EnableKnowledgeRetention)
582da100de0SOCHyams       return false;
583da100de0SOCHyams     AssumptionCache &AC =
584da100de0SOCHyams         getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
58511a3f040SRoman Lebedev     DominatorTreeWrapperPass *DTWP =
58611a3f040SRoman Lebedev         getAnalysisIfAvailable<DominatorTreeWrapperPass>();
58711a3f040SRoman Lebedev     return simplifyAssumes(F, &AC, DTWP ? &DTWP->getDomTree() : nullptr);
588da100de0SOCHyams   }
589da100de0SOCHyams 
getAnalysisUsage(AnalysisUsage & AU) const590da100de0SOCHyams   void getAnalysisUsage(AnalysisUsage &AU) const override {
59111a3f040SRoman Lebedev     AU.addRequired<AssumptionCacheTracker>();
59211a3f040SRoman Lebedev 
593da100de0SOCHyams     AU.setPreservesAll();
594da100de0SOCHyams   }
595da100de0SOCHyams };
596350dadaaSBenjamin Kramer } // namespace
597da100de0SOCHyams 
598da100de0SOCHyams char AssumeSimplifyPassLegacyPass::ID = 0;
599da100de0SOCHyams 
600da100de0SOCHyams INITIALIZE_PASS_BEGIN(AssumeSimplifyPassLegacyPass, "assume-simplify",
601da100de0SOCHyams                       "Assume Simplify", false, false)
INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)602da100de0SOCHyams INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
603da100de0SOCHyams INITIALIZE_PASS_END(AssumeSimplifyPassLegacyPass, "assume-simplify",
604da100de0SOCHyams                     "Assume Simplify", false, false)
605da100de0SOCHyams 
606da100de0SOCHyams FunctionPass *llvm::createAssumeSimplifyPass() {
607da100de0SOCHyams   return new AssumeSimplifyPassLegacyPass();
608da100de0SOCHyams }
609da100de0SOCHyams 
run(Function & F,FunctionAnalysisManager & AM)610c00cb762STyker PreservedAnalyses AssumeBuilderPass::run(Function &F,
611c00cb762STyker                                          FunctionAnalysisManager &AM) {
61211a3f040SRoman Lebedev   AssumptionCache *AC = &AM.getResult<AssumptionAnalysis>(F);
613821a0f23STyker   DominatorTree* DT = AM.getCachedResult<DominatorTreeAnalysis>(F);
614c00cb762STyker   for (Instruction &I : instructions(F))
615821a0f23STyker     salvageKnowledge(&I, AC, DT);
616c00cb762STyker   return PreservedAnalyses::all();
617c00cb762STyker }
61811a3f040SRoman Lebedev 
61911a3f040SRoman Lebedev namespace {
62011a3f040SRoman Lebedev class AssumeBuilderPassLegacyPass : public FunctionPass {
62111a3f040SRoman Lebedev public:
62211a3f040SRoman Lebedev   static char ID;
62311a3f040SRoman Lebedev 
AssumeBuilderPassLegacyPass()62411a3f040SRoman Lebedev   AssumeBuilderPassLegacyPass() : FunctionPass(ID) {
62511a3f040SRoman Lebedev     initializeAssumeBuilderPassLegacyPassPass(*PassRegistry::getPassRegistry());
62611a3f040SRoman Lebedev   }
runOnFunction(Function & F)62711a3f040SRoman Lebedev   bool runOnFunction(Function &F) override {
62811a3f040SRoman Lebedev     AssumptionCache &AC =
62911a3f040SRoman Lebedev         getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
63011a3f040SRoman Lebedev     DominatorTreeWrapperPass *DTWP =
63111a3f040SRoman Lebedev         getAnalysisIfAvailable<DominatorTreeWrapperPass>();
63211a3f040SRoman Lebedev     for (Instruction &I : instructions(F))
63311a3f040SRoman Lebedev       salvageKnowledge(&I, &AC, DTWP ? &DTWP->getDomTree() : nullptr);
63411a3f040SRoman Lebedev     return true;
63511a3f040SRoman Lebedev   }
63611a3f040SRoman Lebedev 
getAnalysisUsage(AnalysisUsage & AU) const63711a3f040SRoman Lebedev   void getAnalysisUsage(AnalysisUsage &AU) const override {
63811a3f040SRoman Lebedev     AU.addRequired<AssumptionCacheTracker>();
63911a3f040SRoman Lebedev 
64011a3f040SRoman Lebedev     AU.setPreservesAll();
64111a3f040SRoman Lebedev   }
64211a3f040SRoman Lebedev };
64311a3f040SRoman Lebedev } // namespace
64411a3f040SRoman Lebedev 
64511a3f040SRoman Lebedev char AssumeBuilderPassLegacyPass::ID = 0;
64611a3f040SRoman Lebedev 
64711a3f040SRoman Lebedev INITIALIZE_PASS_BEGIN(AssumeBuilderPassLegacyPass, "assume-builder",
64811a3f040SRoman Lebedev                       "Assume Builder", false, false)
64911a3f040SRoman Lebedev INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
65011a3f040SRoman Lebedev INITIALIZE_PASS_END(AssumeBuilderPassLegacyPass, "assume-builder",
65111a3f040SRoman Lebedev                     "Assume Builder", false, false)
652