1c00cb762STyker //===- AssumeBundleBuilder.cpp - tools to preserve informations -*- C++ -*-===//
2c00cb762STyker //
3c00cb762STyker // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4c00cb762STyker // See https://llvm.org/LICENSE.txt for license information.
5c00cb762STyker // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6c00cb762STyker //
7c00cb762STyker //===----------------------------------------------------------------------===//
8c00cb762STyker
9c00cb762STyker #include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
10da100de0SOCHyams #include "llvm/ADT/DepthFirstIterator.h"
115957e058STyker #include "llvm/ADT/MapVector.h"
1216f777f4STyker #include "llvm/ADT/Statistic.h"
13c00cb762STyker #include "llvm/Analysis/AssumeBundleQueries.h"
14813f438bSTyker #include "llvm/Analysis/AssumptionCache.h"
15821a0f23STyker #include "llvm/Analysis/ValueTracking.h"
16821a0f23STyker #include "llvm/IR/Dominators.h"
17c00cb762STyker #include "llvm/IR/Function.h"
18c00cb762STyker #include "llvm/IR/InstIterator.h"
19c00cb762STyker #include "llvm/IR/IntrinsicInst.h"
20c00cb762STyker #include "llvm/IR/Module.h"
21*a494ae43Sserge-sans-paille #include "llvm/IR/Operator.h"
22da100de0SOCHyams #include "llvm/InitializePasses.h"
23c00cb762STyker #include "llvm/Support/CommandLine.h"
2416f777f4STyker #include "llvm/Support/DebugCounter.h"
25da100de0SOCHyams #include "llvm/Transforms/Utils/Local.h"
26c00cb762STyker
27c00cb762STyker using namespace llvm;
28c00cb762STyker
29d8aba75aSFangrui Song namespace llvm {
30c00cb762STyker cl::opt<bool> ShouldPreserveAllAttributes(
31c00cb762STyker "assume-preserve-all", cl::init(false), cl::Hidden,
32c00cb762STyker cl::desc("enable preservation of all attrbitues. even those that are "
33c00cb762STyker "unlikely to be usefull"));
34c00cb762STyker
35c00cb762STyker cl::opt<bool> EnableKnowledgeRetention(
36c00cb762STyker "enable-knowledge-retention", cl::init(false), cl::Hidden,
37c00cb762STyker cl::desc(
38c00cb762STyker "enable preservation of attributes throughout code transformation"));
39d8aba75aSFangrui Song } // namespace llvm
40c00cb762STyker
41642e9225STyker #define DEBUG_TYPE "assume-builder"
42642e9225STyker
4316f777f4STyker STATISTIC(NumAssumeBuilt, "Number of assume built by the assume builder");
4416f777f4STyker STATISTIC(NumBundlesInAssumes, "Total number of Bundles in the assume built");
4516f777f4STyker STATISTIC(NumAssumesMerged,
4616f777f4STyker "Number of assume merged by the assume simplify pass");
4716f777f4STyker STATISTIC(NumAssumesRemoved,
4816f777f4STyker "Number of assume removed by the assume simplify pass");
4916f777f4STyker
5016f777f4STyker DEBUG_COUNTER(BuildAssumeCounter, "assume-builder-counter",
5116f777f4STyker "Controls which assumes gets created");
5216f777f4STyker
53c00cb762STyker namespace {
54c00cb762STyker
isUsefullToPreserve(Attribute::AttrKind Kind)55c00cb762STyker bool isUsefullToPreserve(Attribute::AttrKind Kind) {
56c00cb762STyker switch (Kind) {
57c00cb762STyker case Attribute::NonNull:
589b3c2a72SJuneyoung Lee case Attribute::NoUndef:
59c00cb762STyker case Attribute::Alignment:
60c00cb762STyker case Attribute::Dereferenceable:
61c00cb762STyker case Attribute::DereferenceableOrNull:
62c00cb762STyker case Attribute::Cold:
63c00cb762STyker return true;
64c00cb762STyker default:
65c00cb762STyker return false;
66c00cb762STyker }
67c00cb762STyker }
68c00cb762STyker
69b7338fb1STyker /// This function will try to transform the given knowledge into a more
70b7338fb1STyker /// canonical one. the canonical knowledge maybe the given one.
canonicalizedKnowledge(RetainedKnowledge RK,const DataLayout & DL)715f2c53bdSSimon Pilgrim RetainedKnowledge canonicalizedKnowledge(RetainedKnowledge RK,
725f2c53bdSSimon Pilgrim const DataLayout &DL) {
73b7338fb1STyker switch (RK.AttrKind) {
74b7338fb1STyker default:
75b7338fb1STyker return RK;
76b7338fb1STyker case Attribute::NonNull:
77b0eb40caSVitaly Buka RK.WasOn = getUnderlyingObject(RK.WasOn);
78b7338fb1STyker return RK;
79b7338fb1STyker case Attribute::Alignment: {
80b7338fb1STyker Value *V = RK.WasOn->stripInBoundsOffsets([&](const Value *Strip) {
81b7338fb1STyker if (auto *GEP = dyn_cast<GEPOperator>(Strip))
82b7338fb1STyker RK.ArgValue =
83642e9225STyker MinAlign(RK.ArgValue, GEP->getMaxPreservedAlignment(DL).value());
84b7338fb1STyker });
85b7338fb1STyker RK.WasOn = V;
86b7338fb1STyker return RK;
87b7338fb1STyker }
88b7338fb1STyker case Attribute::Dereferenceable:
89b7338fb1STyker case Attribute::DereferenceableOrNull: {
90b7338fb1STyker int64_t Offset = 0;
91642e9225STyker Value *V = GetPointerBaseWithConstantOffset(RK.WasOn, Offset, DL,
92642e9225STyker /*AllowNonInBounds*/ false);
93b7338fb1STyker if (Offset < 0)
94b7338fb1STyker return RK;
95b7338fb1STyker RK.ArgValue = RK.ArgValue + Offset;
96b7338fb1STyker RK.WasOn = V;
97b7338fb1STyker }
98b7338fb1STyker }
99b7338fb1STyker return RK;
100b7338fb1STyker }
101b7338fb1STyker
102c00cb762STyker /// This class contain all knowledge that have been gather while building an
103c00cb762STyker /// llvm.assume and the function to manipulate it.
104c00cb762STyker struct AssumeBuilderState {
105c00cb762STyker Module *M;
106c00cb762STyker
107e5f8a77cSTyker using MapKey = std::pair<Value *, Attribute::AttrKind>;
1083628bb74SArthur Eubanks SmallMapVector<MapKey, uint64_t, 8> AssumedKnowledgeMap;
109642e9225STyker Instruction *InstBeingModified = nullptr;
110821a0f23STyker AssumptionCache* AC = nullptr;
111821a0f23STyker DominatorTree* DT = nullptr;
112c00cb762STyker
AssumeBuilderState__anonda79dfad0111::AssumeBuilderState113821a0f23STyker AssumeBuilderState(Module *M, Instruction *I = nullptr,
114821a0f23STyker AssumptionCache *AC = nullptr, DominatorTree *DT = nullptr)
115642e9225STyker : M(M), InstBeingModified(I), AC(AC), DT(DT) {}
116821a0f23STyker
tryToPreserveWithoutAddingAssume__anonda79dfad0111::AssumeBuilderState117821a0f23STyker bool tryToPreserveWithoutAddingAssume(RetainedKnowledge RK) {
118642e9225STyker if (!InstBeingModified || !RK.WasOn)
119821a0f23STyker return false;
120821a0f23STyker bool HasBeenPreserved = false;
121821a0f23STyker Use* ToUpdate = nullptr;
122821a0f23STyker getKnowledgeForValue(
123821a0f23STyker RK.WasOn, {RK.AttrKind}, AC,
124821a0f23STyker [&](RetainedKnowledge RKOther, Instruction *Assume,
125821a0f23STyker const CallInst::BundleOpInfo *Bundle) {
126642e9225STyker if (!isValidAssumeForContext(Assume, InstBeingModified, DT))
127821a0f23STyker return false;
128821a0f23STyker if (RKOther.ArgValue >= RK.ArgValue) {
129821a0f23STyker HasBeenPreserved = true;
130821a0f23STyker return true;
131642e9225STyker } else if (isValidAssumeForContext(InstBeingModified, Assume, DT)) {
132821a0f23STyker HasBeenPreserved = true;
133821a0f23STyker IntrinsicInst *Intr = cast<IntrinsicInst>(Assume);
134821a0f23STyker ToUpdate = &Intr->op_begin()[Bundle->Begin + ABA_Argument];
135821a0f23STyker return true;
136821a0f23STyker }
137821a0f23STyker return false;
138821a0f23STyker });
139821a0f23STyker if (ToUpdate)
140821a0f23STyker ToUpdate->set(
141821a0f23STyker ConstantInt::get(Type::getInt64Ty(M->getContext()), RK.ArgValue));
142821a0f23STyker return HasBeenPreserved;
143821a0f23STyker }
144c00cb762STyker
isKnowledgeWorthPreserving__anonda79dfad0111::AssumeBuilderState145b7338fb1STyker bool isKnowledgeWorthPreserving(RetainedKnowledge RK) {
146b7338fb1STyker if (!RK)
147b7338fb1STyker return false;
148b7338fb1STyker if (!RK.WasOn)
149b7338fb1STyker return true;
150b7338fb1STyker if (RK.WasOn->getType()->isPointerTy()) {
151b0eb40caSVitaly Buka Value *UnderlyingPtr = getUnderlyingObject(RK.WasOn);
152b7338fb1STyker if (isa<AllocaInst>(UnderlyingPtr) || isa<GlobalValue>(UnderlyingPtr))
153b7338fb1STyker return false;
154b7338fb1STyker }
155b7338fb1STyker if (auto *Arg = dyn_cast<Argument>(RK.WasOn)) {
156b7338fb1STyker if (Arg->hasAttribute(RK.AttrKind) &&
1576ac32872SNikita Popov (!Attribute::isIntAttrKind(RK.AttrKind) ||
158b7338fb1STyker Arg->getAttribute(RK.AttrKind).getValueAsInt() >= RK.ArgValue))
159b7338fb1STyker return false;
160b7338fb1STyker return true;
161b7338fb1STyker }
162b7338fb1STyker if (auto *Inst = dyn_cast<Instruction>(RK.WasOn))
163b7338fb1STyker if (wouldInstructionBeTriviallyDead(Inst)) {
164b7338fb1STyker if (RK.WasOn->use_empty())
165b7338fb1STyker return false;
166b6cb03e6SAnna Thomas Use *SingleUse = RK.WasOn->getSingleUndroppableUse();
167b6cb03e6SAnna Thomas if (SingleUse && SingleUse->getUser() == InstBeingModified)
168b7338fb1STyker return false;
169b7338fb1STyker }
170b7338fb1STyker return true;
171b7338fb1STyker }
172b7338fb1STyker
addKnowledge__anonda79dfad0111::AssumeBuilderState173e5f8a77cSTyker void addKnowledge(RetainedKnowledge RK) {
174642e9225STyker RK = canonicalizedKnowledge(RK, M->getDataLayout());
175b7338fb1STyker
176b7338fb1STyker if (!isKnowledgeWorthPreserving(RK))
177b7338fb1STyker return;
178b7338fb1STyker
179b7338fb1STyker if (tryToPreserveWithoutAddingAssume(RK))
180821a0f23STyker return;
181e5f8a77cSTyker MapKey Key{RK.WasOn, RK.AttrKind};
182e5f8a77cSTyker auto Lookup = AssumedKnowledgeMap.find(Key);
183e5f8a77cSTyker if (Lookup == AssumedKnowledgeMap.end()) {
184e5f8a77cSTyker AssumedKnowledgeMap[Key] = RK.ArgValue;
185e5f8a77cSTyker return;
186e5f8a77cSTyker }
187e5f8a77cSTyker assert(((Lookup->second == 0 && RK.ArgValue == 0) ||
188e5f8a77cSTyker (Lookup->second != 0 && RK.ArgValue != 0)) &&
189e5f8a77cSTyker "inconsistent argument value");
190e5f8a77cSTyker
191e5f8a77cSTyker /// This is only desirable because for all attributes taking an argument
192e5f8a77cSTyker /// higher is better.
193e5f8a77cSTyker Lookup->second = std::max(Lookup->second, RK.ArgValue);
194e5f8a77cSTyker }
195e5f8a77cSTyker
addAttribute__anonda79dfad0111::AssumeBuilderState196c00cb762STyker void addAttribute(Attribute Attr, Value *WasOn) {
197e5f8a77cSTyker if (Attr.isTypeAttribute() || Attr.isStringAttribute() ||
198e5f8a77cSTyker (!ShouldPreserveAllAttributes &&
199c00cb762STyker !isUsefullToPreserve(Attr.getKindAsEnum())))
200c00cb762STyker return;
2013628bb74SArthur Eubanks uint64_t AttrArg = 0;
202c00cb762STyker if (Attr.isIntAttribute())
203e5f8a77cSTyker AttrArg = Attr.getValueAsInt();
204e5f8a77cSTyker addKnowledge({Attr.getKindAsEnum(), AttrArg, WasOn});
205c00cb762STyker }
206c00cb762STyker
addCall__anonda79dfad0111::AssumeBuilderState207c00cb762STyker void addCall(const CallBase *Call) {
208a9831cceSArthur Eubanks auto addAttrList = [&](AttributeList AttrList, unsigned NumArgs) {
209a9831cceSArthur Eubanks for (unsigned Idx = 0; Idx < NumArgs; Idx++)
210a9831cceSArthur Eubanks for (Attribute Attr : AttrList.getParamAttrs(Idx)) {
211edf634ebSJuneyoung Lee bool IsPoisonAttr = Attr.hasAttribute(Attribute::NonNull) ||
212edf634ebSJuneyoung Lee Attr.hasAttribute(Attribute::Alignment);
213a9831cceSArthur Eubanks if (!IsPoisonAttr || Call->isPassingUndefUB(Idx))
214a9831cceSArthur Eubanks addAttribute(Attr, Call->getArgOperand(Idx));
215edf634ebSJuneyoung Lee }
21680ea2bb5SArthur Eubanks for (Attribute Attr : AttrList.getFnAttrs())
217c00cb762STyker addAttribute(Attr, nullptr);
218c00cb762STyker };
219a9831cceSArthur Eubanks addAttrList(Call->getAttributes(), Call->arg_size());
220c00cb762STyker if (Function *Fn = Call->getCalledFunction())
221a9831cceSArthur Eubanks addAttrList(Fn->getAttributes(), Fn->arg_size());
222c00cb762STyker }
223c00cb762STyker
build__anonda79dfad0111::AssumeBuilderState2249ef6aa02SPhilip Reames AssumeInst *build() {
225e5f8a77cSTyker if (AssumedKnowledgeMap.empty())
226c00cb762STyker return nullptr;
22716f777f4STyker if (!DebugCounter::shouldExecute(BuildAssumeCounter))
22816f777f4STyker return nullptr;
229c00cb762STyker Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
230c00cb762STyker LLVMContext &C = M->getContext();
231c00cb762STyker SmallVector<OperandBundleDef, 8> OpBundle;
232e5f8a77cSTyker for (auto &MapElem : AssumedKnowledgeMap) {
233c00cb762STyker SmallVector<Value *, 2> Args;
234e5f8a77cSTyker if (MapElem.first.first)
235e5f8a77cSTyker Args.push_back(MapElem.first.first);
236e5f8a77cSTyker
237e5f8a77cSTyker /// This is only valid because for all attribute that currently exist a
238e5f8a77cSTyker /// value of 0 is useless. and should not be preserved.
239e5f8a77cSTyker if (MapElem.second)
240e5f8a77cSTyker Args.push_back(ConstantInt::get(Type::getInt64Ty(M->getContext()),
241e5f8a77cSTyker MapElem.second));
242e5f8a77cSTyker OpBundle.push_back(OperandBundleDefT<Value *>(
243e5f8a77cSTyker std::string(Attribute::getNameFromAttrKind(MapElem.first.second)),
244e5f8a77cSTyker Args));
24516f777f4STyker NumBundlesInAssumes++;
246c00cb762STyker }
24716f777f4STyker NumAssumeBuilt++;
2489ef6aa02SPhilip Reames return cast<AssumeInst>(CallInst::Create(
249c00cb762STyker FnAssume, ArrayRef<Value *>({ConstantInt::getTrue(C)}), OpBundle));
250c00cb762STyker }
251c00cb762STyker
addAccessedPtr__anonda79dfad0111::AssumeBuilderState252c00cb762STyker void addAccessedPtr(Instruction *MemInst, Value *Pointer, Type *AccType,
253c00cb762STyker MaybeAlign MA) {
254e5f8a77cSTyker unsigned DerefSize = MemInst->getModule()
255c00cb762STyker ->getDataLayout()
256c00cb762STyker .getTypeStoreSize(AccType)
257c00cb762STyker .getKnownMinSize();
258c00cb762STyker if (DerefSize != 0) {
259e5f8a77cSTyker addKnowledge({Attribute::Dereferenceable, DerefSize, Pointer});
260c00cb762STyker if (!NullPointerIsDefined(MemInst->getFunction(),
261c00cb762STyker Pointer->getType()->getPointerAddressSpace()))
262e5f8a77cSTyker addKnowledge({Attribute::NonNull, 0u, Pointer});
263c00cb762STyker }
264c00cb762STyker if (MA.valueOrOne() > 1)
2653628bb74SArthur Eubanks addKnowledge({Attribute::Alignment, MA.valueOrOne().value(), Pointer});
266c00cb762STyker }
267c00cb762STyker
addInstruction__anonda79dfad0111::AssumeBuilderState268c00cb762STyker void addInstruction(Instruction *I) {
269c00cb762STyker if (auto *Call = dyn_cast<CallBase>(I))
270c00cb762STyker return addCall(Call);
271c00cb762STyker if (auto *Load = dyn_cast<LoadInst>(I))
272c00cb762STyker return addAccessedPtr(I, Load->getPointerOperand(), Load->getType(),
273c00cb762STyker Load->getAlign());
274c00cb762STyker if (auto *Store = dyn_cast<StoreInst>(I))
275c00cb762STyker return addAccessedPtr(I, Store->getPointerOperand(),
276c00cb762STyker Store->getValueOperand()->getType(),
277c00cb762STyker Store->getAlign());
278c00cb762STyker // TODO: Add support for the other Instructions.
279c00cb762STyker // TODO: Maybe we should look around and merge with other llvm.assume.
280c00cb762STyker }
281c00cb762STyker };
282c00cb762STyker
283c00cb762STyker } // namespace
284c00cb762STyker
buildAssumeFromInst(Instruction * I)2859ef6aa02SPhilip Reames AssumeInst *llvm::buildAssumeFromInst(Instruction *I) {
286c00cb762STyker if (!EnableKnowledgeRetention)
287c00cb762STyker return nullptr;
288c00cb762STyker AssumeBuilderState Builder(I->getModule());
289c00cb762STyker Builder.addInstruction(I);
290c00cb762STyker return Builder.build();
291c00cb762STyker }
292c00cb762STyker
salvageKnowledge(Instruction * I,AssumptionCache * AC,DominatorTree * DT)29378d85c20STyker void llvm::salvageKnowledge(Instruction *I, AssumptionCache *AC,
29478d85c20STyker DominatorTree *DT) {
29578d85c20STyker if (!EnableKnowledgeRetention || I->isTerminator())
296e5f8a77cSTyker return;
297821a0f23STyker AssumeBuilderState Builder(I->getModule(), I, AC, DT);
298e5f8a77cSTyker Builder.addInstruction(I);
2999ef6aa02SPhilip Reames if (auto *Intr = Builder.build()) {
300c00cb762STyker Intr->insertBefore(I);
301813f438bSTyker if (AC)
3029ef6aa02SPhilip Reames AC->registerAssumption(Intr);
303813f438bSTyker }
304c00cb762STyker }
305c00cb762STyker
3069ef6aa02SPhilip Reames AssumeInst *
buildAssumeFromKnowledge(ArrayRef<RetainedKnowledge> Knowledge,Instruction * CtxI,AssumptionCache * AC,DominatorTree * DT)307642e9225STyker llvm::buildAssumeFromKnowledge(ArrayRef<RetainedKnowledge> Knowledge,
308642e9225STyker Instruction *CtxI, AssumptionCache *AC,
309642e9225STyker DominatorTree *DT) {
310642e9225STyker AssumeBuilderState Builder(CtxI->getModule(), CtxI, AC, DT);
311642e9225STyker for (const RetainedKnowledge &RK : Knowledge)
312642e9225STyker Builder.addKnowledge(RK);
313642e9225STyker return Builder.build();
314642e9225STyker }
315642e9225STyker
simplifyRetainedKnowledge(AssumeInst * Assume,RetainedKnowledge RK,AssumptionCache * AC,DominatorTree * DT)3169ef6aa02SPhilip Reames RetainedKnowledge llvm::simplifyRetainedKnowledge(AssumeInst *Assume,
317642e9225STyker RetainedKnowledge RK,
318642e9225STyker AssumptionCache *AC,
319642e9225STyker DominatorTree *DT) {
320642e9225STyker AssumeBuilderState Builder(Assume->getModule(), Assume, AC, DT);
321642e9225STyker RK = canonicalizedKnowledge(RK, Assume->getModule()->getDataLayout());
322642e9225STyker
323642e9225STyker if (!Builder.isKnowledgeWorthPreserving(RK))
324642e9225STyker return RetainedKnowledge::none();
325642e9225STyker
326642e9225STyker if (Builder.tryToPreserveWithoutAddingAssume(RK))
327642e9225STyker return RetainedKnowledge::none();
328642e9225STyker return RK;
329642e9225STyker }
330642e9225STyker
331da100de0SOCHyams namespace {
332da100de0SOCHyams
333da100de0SOCHyams struct AssumeSimplify {
334da100de0SOCHyams Function &F;
335da100de0SOCHyams AssumptionCache &AC;
336da100de0SOCHyams DominatorTree *DT;
337da100de0SOCHyams LLVMContext &C;
338da100de0SOCHyams SmallDenseSet<IntrinsicInst *> CleanupToDo;
339da100de0SOCHyams StringMapEntry<uint32_t> *IgnoreTag;
340da100de0SOCHyams SmallDenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 4>, 8> BBToAssume;
341da100de0SOCHyams bool MadeChange = false;
342da100de0SOCHyams
AssumeSimplify__anonda79dfad0511::AssumeSimplify343da100de0SOCHyams AssumeSimplify(Function &F, AssumptionCache &AC, DominatorTree *DT,
344da100de0SOCHyams LLVMContext &C)
345da100de0SOCHyams : F(F), AC(AC), DT(DT), C(C),
346da100de0SOCHyams IgnoreTag(C.getOrInsertBundleTag(IgnoreBundleTag)) {}
347da100de0SOCHyams
buildMapping__anonda79dfad0511::AssumeSimplify348da100de0SOCHyams void buildMapping(bool FilterBooleanArgument) {
349da100de0SOCHyams BBToAssume.clear();
350da100de0SOCHyams for (Value *V : AC.assumptions()) {
351da100de0SOCHyams if (!V)
352da100de0SOCHyams continue;
353da100de0SOCHyams IntrinsicInst *Assume = cast<IntrinsicInst>(V);
354da100de0SOCHyams if (FilterBooleanArgument) {
355da100de0SOCHyams auto *Arg = dyn_cast<ConstantInt>(Assume->getOperand(0));
356da100de0SOCHyams if (!Arg || Arg->isZero())
357da100de0SOCHyams continue;
358da100de0SOCHyams }
359da100de0SOCHyams BBToAssume[Assume->getParent()].push_back(Assume);
360da100de0SOCHyams }
361da100de0SOCHyams
362da100de0SOCHyams for (auto &Elem : BBToAssume) {
363da100de0SOCHyams llvm::sort(Elem.second,
364da100de0SOCHyams [](const IntrinsicInst *LHS, const IntrinsicInst *RHS) {
365da100de0SOCHyams return LHS->comesBefore(RHS);
366da100de0SOCHyams });
367da100de0SOCHyams }
368da100de0SOCHyams }
369da100de0SOCHyams
370da100de0SOCHyams /// Remove all asumes in CleanupToDo if there boolean argument is true and
371da100de0SOCHyams /// ForceCleanup is set or the assume doesn't hold valuable knowledge.
RunCleanup__anonda79dfad0511::AssumeSimplify372da100de0SOCHyams void RunCleanup(bool ForceCleanup) {
373da100de0SOCHyams for (IntrinsicInst *Assume : CleanupToDo) {
374da100de0SOCHyams auto *Arg = dyn_cast<ConstantInt>(Assume->getOperand(0));
375da100de0SOCHyams if (!Arg || Arg->isZero() ||
3769ef6aa02SPhilip Reames (!ForceCleanup &&
3779ef6aa02SPhilip Reames !isAssumeWithEmptyBundle(cast<AssumeInst>(*Assume))))
378da100de0SOCHyams continue;
379da100de0SOCHyams MadeChange = true;
38016f777f4STyker if (ForceCleanup)
38116f777f4STyker NumAssumesMerged++;
38216f777f4STyker else
38316f777f4STyker NumAssumesRemoved++;
384da100de0SOCHyams Assume->eraseFromParent();
385da100de0SOCHyams }
386da100de0SOCHyams CleanupToDo.clear();
387da100de0SOCHyams }
388da100de0SOCHyams
389da100de0SOCHyams /// Remove knowledge stored in assume when it is already know by an attribute
390da100de0SOCHyams /// or an other assume. This can when valid update an existing knowledge in an
391da100de0SOCHyams /// attribute or an other assume.
dropRedundantKnowledge__anonda79dfad0511::AssumeSimplify392da100de0SOCHyams void dropRedundantKnowledge() {
393da100de0SOCHyams struct MapValue {
394da100de0SOCHyams IntrinsicInst *Assume;
3953628bb74SArthur Eubanks uint64_t ArgValue;
396da100de0SOCHyams CallInst::BundleOpInfo *BOI;
397da100de0SOCHyams };
398da100de0SOCHyams buildMapping(false);
399da100de0SOCHyams SmallDenseMap<std::pair<Value *, Attribute::AttrKind>,
400da100de0SOCHyams SmallVector<MapValue, 2>, 16>
401da100de0SOCHyams Knowledge;
402da100de0SOCHyams for (BasicBlock *BB : depth_first(&F))
403da100de0SOCHyams for (Value *V : BBToAssume[BB]) {
404da100de0SOCHyams if (!V)
405da100de0SOCHyams continue;
406da100de0SOCHyams IntrinsicInst *Assume = cast<IntrinsicInst>(V);
407da100de0SOCHyams for (CallInst::BundleOpInfo &BOI : Assume->bundle_op_infos()) {
408da100de0SOCHyams auto RemoveFromAssume = [&]() {
409da100de0SOCHyams CleanupToDo.insert(Assume);
410da100de0SOCHyams if (BOI.Begin != BOI.End) {
411da100de0SOCHyams Use *U = &Assume->op_begin()[BOI.Begin + ABA_WasOn];
412da100de0SOCHyams U->set(UndefValue::get(U->get()->getType()));
413da100de0SOCHyams }
414da100de0SOCHyams BOI.Tag = IgnoreTag;
415da100de0SOCHyams };
416da100de0SOCHyams if (BOI.Tag == IgnoreTag) {
417da100de0SOCHyams CleanupToDo.insert(Assume);
418da100de0SOCHyams continue;
419da100de0SOCHyams }
4209ef6aa02SPhilip Reames RetainedKnowledge RK =
4219ef6aa02SPhilip Reames getKnowledgeFromBundle(cast<AssumeInst>(*Assume), BOI);
422da100de0SOCHyams if (auto *Arg = dyn_cast_or_null<Argument>(RK.WasOn)) {
423da100de0SOCHyams bool HasSameKindAttr = Arg->hasAttribute(RK.AttrKind);
424da100de0SOCHyams if (HasSameKindAttr)
4256ac32872SNikita Popov if (!Attribute::isIntAttrKind(RK.AttrKind) ||
426da100de0SOCHyams Arg->getAttribute(RK.AttrKind).getValueAsInt() >=
427da100de0SOCHyams RK.ArgValue) {
428da100de0SOCHyams RemoveFromAssume();
429da100de0SOCHyams continue;
430da100de0SOCHyams }
431da100de0SOCHyams if (isValidAssumeForContext(
432da100de0SOCHyams Assume, &*F.getEntryBlock().getFirstInsertionPt()) ||
433da100de0SOCHyams Assume == &*F.getEntryBlock().getFirstInsertionPt()) {
434da100de0SOCHyams if (HasSameKindAttr)
435da100de0SOCHyams Arg->removeAttr(RK.AttrKind);
436da100de0SOCHyams Arg->addAttr(Attribute::get(C, RK.AttrKind, RK.ArgValue));
437da100de0SOCHyams MadeChange = true;
438da100de0SOCHyams RemoveFromAssume();
439da100de0SOCHyams continue;
440da100de0SOCHyams }
441da100de0SOCHyams }
442da100de0SOCHyams auto &Lookup = Knowledge[{RK.WasOn, RK.AttrKind}];
443da100de0SOCHyams for (MapValue &Elem : Lookup) {
444da100de0SOCHyams if (!isValidAssumeForContext(Elem.Assume, Assume, DT))
445da100de0SOCHyams continue;
446da100de0SOCHyams if (Elem.ArgValue >= RK.ArgValue) {
447da100de0SOCHyams RemoveFromAssume();
448da100de0SOCHyams continue;
449da100de0SOCHyams } else if (isValidAssumeForContext(Assume, Elem.Assume, DT)) {
450da100de0SOCHyams Elem.Assume->op_begin()[Elem.BOI->Begin + ABA_Argument].set(
451da100de0SOCHyams ConstantInt::get(Type::getInt64Ty(C), RK.ArgValue));
452da100de0SOCHyams MadeChange = true;
453da100de0SOCHyams RemoveFromAssume();
454da100de0SOCHyams continue;
455da100de0SOCHyams }
456da100de0SOCHyams }
457da100de0SOCHyams Lookup.push_back({Assume, RK.ArgValue, &BOI});
458da100de0SOCHyams }
459da100de0SOCHyams }
460da100de0SOCHyams }
461da100de0SOCHyams
462da100de0SOCHyams using MergeIterator = SmallVectorImpl<IntrinsicInst *>::iterator;
463da100de0SOCHyams
464da100de0SOCHyams /// Merge all Assumes from Begin to End in and insert the resulting assume as
465da100de0SOCHyams /// high as possible in the basicblock.
mergeRange__anonda79dfad0511::AssumeSimplify466da100de0SOCHyams void mergeRange(BasicBlock *BB, MergeIterator Begin, MergeIterator End) {
467da100de0SOCHyams if (Begin == End || std::next(Begin) == End)
468da100de0SOCHyams return;
469da100de0SOCHyams /// Provide no additional information so that AssumeBuilderState doesn't
470da100de0SOCHyams /// try to do any punning since it already has been done better.
471da100de0SOCHyams AssumeBuilderState Builder(F.getParent());
472da100de0SOCHyams
473da100de0SOCHyams /// For now it is initialized to the best value it could have
474da100de0SOCHyams Instruction *InsertPt = BB->getFirstNonPHI();
475da100de0SOCHyams if (isa<LandingPadInst>(InsertPt))
476da100de0SOCHyams InsertPt = InsertPt->getNextNode();
477da100de0SOCHyams for (IntrinsicInst *I : make_range(Begin, End)) {
478da100de0SOCHyams CleanupToDo.insert(I);
479da100de0SOCHyams for (CallInst::BundleOpInfo &BOI : I->bundle_op_infos()) {
4809ef6aa02SPhilip Reames RetainedKnowledge RK =
4819ef6aa02SPhilip Reames getKnowledgeFromBundle(cast<AssumeInst>(*I), BOI);
482da100de0SOCHyams if (!RK)
483da100de0SOCHyams continue;
484da100de0SOCHyams Builder.addKnowledge(RK);
485da100de0SOCHyams if (auto *I = dyn_cast_or_null<Instruction>(RK.WasOn))
486da100de0SOCHyams if (I->getParent() == InsertPt->getParent() &&
487da100de0SOCHyams (InsertPt->comesBefore(I) || InsertPt == I))
488da100de0SOCHyams InsertPt = I->getNextNode();
489da100de0SOCHyams }
490da100de0SOCHyams }
491da100de0SOCHyams
492da100de0SOCHyams /// Adjust InsertPt if it is before Begin, since mergeAssumes only
493da100de0SOCHyams /// guarantees we can place the resulting assume between Begin and End.
494da100de0SOCHyams if (InsertPt->comesBefore(*Begin))
495da100de0SOCHyams for (auto It = (*Begin)->getIterator(), E = InsertPt->getIterator();
496da100de0SOCHyams It != E; --It)
497da100de0SOCHyams if (!isGuaranteedToTransferExecutionToSuccessor(&*It)) {
498da100de0SOCHyams InsertPt = It->getNextNode();
499da100de0SOCHyams break;
500da100de0SOCHyams }
5019ef6aa02SPhilip Reames auto *MergedAssume = Builder.build();
502da100de0SOCHyams if (!MergedAssume)
503da100de0SOCHyams return;
504da100de0SOCHyams MadeChange = true;
505da100de0SOCHyams MergedAssume->insertBefore(InsertPt);
5069ef6aa02SPhilip Reames AC.registerAssumption(MergedAssume);
507da100de0SOCHyams }
508da100de0SOCHyams
509da100de0SOCHyams /// Merge assume when they are in the same BasicBlock and for all instruction
510da100de0SOCHyams /// between them isGuaranteedToTransferExecutionToSuccessor returns true.
mergeAssumes__anonda79dfad0511::AssumeSimplify511da100de0SOCHyams void mergeAssumes() {
512da100de0SOCHyams buildMapping(true);
513da100de0SOCHyams
514da100de0SOCHyams SmallVector<MergeIterator, 4> SplitPoints;
515da100de0SOCHyams for (auto &Elem : BBToAssume) {
516da100de0SOCHyams SmallVectorImpl<IntrinsicInst *> &AssumesInBB = Elem.second;
517da100de0SOCHyams if (AssumesInBB.size() < 2)
518da100de0SOCHyams continue;
519da100de0SOCHyams /// AssumesInBB is already sorted by order in the block.
520da100de0SOCHyams
521da100de0SOCHyams BasicBlock::iterator It = AssumesInBB.front()->getIterator();
522da100de0SOCHyams BasicBlock::iterator E = AssumesInBB.back()->getIterator();
523da100de0SOCHyams SplitPoints.push_back(AssumesInBB.begin());
524da100de0SOCHyams MergeIterator LastSplit = AssumesInBB.begin();
525da100de0SOCHyams for (; It != E; ++It)
526da100de0SOCHyams if (!isGuaranteedToTransferExecutionToSuccessor(&*It)) {
527da100de0SOCHyams for (; (*LastSplit)->comesBefore(&*It); ++LastSplit)
528da100de0SOCHyams ;
529da100de0SOCHyams if (SplitPoints.back() != LastSplit)
530da100de0SOCHyams SplitPoints.push_back(LastSplit);
531da100de0SOCHyams }
532da100de0SOCHyams SplitPoints.push_back(AssumesInBB.end());
533da100de0SOCHyams for (auto SplitIt = SplitPoints.begin();
534da100de0SOCHyams SplitIt != std::prev(SplitPoints.end()); SplitIt++) {
535da100de0SOCHyams mergeRange(Elem.first, *SplitIt, *(SplitIt + 1));
536da100de0SOCHyams }
537da100de0SOCHyams SplitPoints.clear();
538da100de0SOCHyams }
539da100de0SOCHyams }
540da100de0SOCHyams };
541da100de0SOCHyams
simplifyAssumes(Function & F,AssumptionCache * AC,DominatorTree * DT)542da100de0SOCHyams bool simplifyAssumes(Function &F, AssumptionCache *AC, DominatorTree *DT) {
543da100de0SOCHyams AssumeSimplify AS(F, *AC, DT, F.getContext());
544da100de0SOCHyams
545da100de0SOCHyams /// Remove knowledge that is already known by a dominating other assume or an
546da100de0SOCHyams /// attribute.
547da100de0SOCHyams AS.dropRedundantKnowledge();
548da100de0SOCHyams
549da100de0SOCHyams /// Remove assume that are empty.
550da100de0SOCHyams AS.RunCleanup(false);
551da100de0SOCHyams
552da100de0SOCHyams /// Merge assume in the same basicblock when possible.
553da100de0SOCHyams AS.mergeAssumes();
554da100de0SOCHyams
555da100de0SOCHyams /// Remove assume that were merged.
556da100de0SOCHyams AS.RunCleanup(true);
557da100de0SOCHyams return AS.MadeChange;
558da100de0SOCHyams }
559da100de0SOCHyams
560da100de0SOCHyams } // namespace
561da100de0SOCHyams
run(Function & F,FunctionAnalysisManager & AM)562da100de0SOCHyams PreservedAnalyses AssumeSimplifyPass::run(Function &F,
563da100de0SOCHyams FunctionAnalysisManager &AM) {
564da100de0SOCHyams if (!EnableKnowledgeRetention)
565da100de0SOCHyams return PreservedAnalyses::all();
566da100de0SOCHyams simplifyAssumes(F, &AM.getResult<AssumptionAnalysis>(F),
567da100de0SOCHyams AM.getCachedResult<DominatorTreeAnalysis>(F));
568da100de0SOCHyams return PreservedAnalyses::all();
569da100de0SOCHyams }
570da100de0SOCHyams
571350dadaaSBenjamin Kramer namespace {
572da100de0SOCHyams class AssumeSimplifyPassLegacyPass : public FunctionPass {
573da100de0SOCHyams public:
574da100de0SOCHyams static char ID;
575da100de0SOCHyams
AssumeSimplifyPassLegacyPass()576da100de0SOCHyams AssumeSimplifyPassLegacyPass() : FunctionPass(ID) {
577da100de0SOCHyams initializeAssumeSimplifyPassLegacyPassPass(
578da100de0SOCHyams *PassRegistry::getPassRegistry());
579da100de0SOCHyams }
runOnFunction(Function & F)580da100de0SOCHyams bool runOnFunction(Function &F) override {
581da100de0SOCHyams if (skipFunction(F) || !EnableKnowledgeRetention)
582da100de0SOCHyams return false;
583da100de0SOCHyams AssumptionCache &AC =
584da100de0SOCHyams getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
58511a3f040SRoman Lebedev DominatorTreeWrapperPass *DTWP =
58611a3f040SRoman Lebedev getAnalysisIfAvailable<DominatorTreeWrapperPass>();
58711a3f040SRoman Lebedev return simplifyAssumes(F, &AC, DTWP ? &DTWP->getDomTree() : nullptr);
588da100de0SOCHyams }
589da100de0SOCHyams
getAnalysisUsage(AnalysisUsage & AU) const590da100de0SOCHyams void getAnalysisUsage(AnalysisUsage &AU) const override {
59111a3f040SRoman Lebedev AU.addRequired<AssumptionCacheTracker>();
59211a3f040SRoman Lebedev
593da100de0SOCHyams AU.setPreservesAll();
594da100de0SOCHyams }
595da100de0SOCHyams };
596350dadaaSBenjamin Kramer } // namespace
597da100de0SOCHyams
598da100de0SOCHyams char AssumeSimplifyPassLegacyPass::ID = 0;
599da100de0SOCHyams
600da100de0SOCHyams INITIALIZE_PASS_BEGIN(AssumeSimplifyPassLegacyPass, "assume-simplify",
601da100de0SOCHyams "Assume Simplify", false, false)
INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)602da100de0SOCHyams INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
603da100de0SOCHyams INITIALIZE_PASS_END(AssumeSimplifyPassLegacyPass, "assume-simplify",
604da100de0SOCHyams "Assume Simplify", false, false)
605da100de0SOCHyams
606da100de0SOCHyams FunctionPass *llvm::createAssumeSimplifyPass() {
607da100de0SOCHyams return new AssumeSimplifyPassLegacyPass();
608da100de0SOCHyams }
609da100de0SOCHyams
run(Function & F,FunctionAnalysisManager & AM)610c00cb762STyker PreservedAnalyses AssumeBuilderPass::run(Function &F,
611c00cb762STyker FunctionAnalysisManager &AM) {
61211a3f040SRoman Lebedev AssumptionCache *AC = &AM.getResult<AssumptionAnalysis>(F);
613821a0f23STyker DominatorTree* DT = AM.getCachedResult<DominatorTreeAnalysis>(F);
614c00cb762STyker for (Instruction &I : instructions(F))
615821a0f23STyker salvageKnowledge(&I, AC, DT);
616c00cb762STyker return PreservedAnalyses::all();
617c00cb762STyker }
61811a3f040SRoman Lebedev
61911a3f040SRoman Lebedev namespace {
62011a3f040SRoman Lebedev class AssumeBuilderPassLegacyPass : public FunctionPass {
62111a3f040SRoman Lebedev public:
62211a3f040SRoman Lebedev static char ID;
62311a3f040SRoman Lebedev
AssumeBuilderPassLegacyPass()62411a3f040SRoman Lebedev AssumeBuilderPassLegacyPass() : FunctionPass(ID) {
62511a3f040SRoman Lebedev initializeAssumeBuilderPassLegacyPassPass(*PassRegistry::getPassRegistry());
62611a3f040SRoman Lebedev }
runOnFunction(Function & F)62711a3f040SRoman Lebedev bool runOnFunction(Function &F) override {
62811a3f040SRoman Lebedev AssumptionCache &AC =
62911a3f040SRoman Lebedev getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
63011a3f040SRoman Lebedev DominatorTreeWrapperPass *DTWP =
63111a3f040SRoman Lebedev getAnalysisIfAvailable<DominatorTreeWrapperPass>();
63211a3f040SRoman Lebedev for (Instruction &I : instructions(F))
63311a3f040SRoman Lebedev salvageKnowledge(&I, &AC, DTWP ? &DTWP->getDomTree() : nullptr);
63411a3f040SRoman Lebedev return true;
63511a3f040SRoman Lebedev }
63611a3f040SRoman Lebedev
getAnalysisUsage(AnalysisUsage & AU) const63711a3f040SRoman Lebedev void getAnalysisUsage(AnalysisUsage &AU) const override {
63811a3f040SRoman Lebedev AU.addRequired<AssumptionCacheTracker>();
63911a3f040SRoman Lebedev
64011a3f040SRoman Lebedev AU.setPreservesAll();
64111a3f040SRoman Lebedev }
64211a3f040SRoman Lebedev };
64311a3f040SRoman Lebedev } // namespace
64411a3f040SRoman Lebedev
64511a3f040SRoman Lebedev char AssumeBuilderPassLegacyPass::ID = 0;
64611a3f040SRoman Lebedev
64711a3f040SRoman Lebedev INITIALIZE_PASS_BEGIN(AssumeBuilderPassLegacyPass, "assume-builder",
64811a3f040SRoman Lebedev "Assume Builder", false, false)
64911a3f040SRoman Lebedev INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
65011a3f040SRoman Lebedev INITIALIZE_PASS_END(AssumeBuilderPassLegacyPass, "assume-builder",
65111a3f040SRoman Lebedev "Assume Builder", false, false)
652