1 //===- AssumptionCache.cpp - Cache finding @llvm.assume calls -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a pass that keeps track of @llvm.assume intrinsics in
10 // the functions of a module.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Analysis/AssumeBundleQueries.h"
15 #include "llvm/Analysis/AssumptionCache.h"
16 #include "llvm/ADT/STLExtras.h"
17 #include "llvm/ADT/SmallPtrSet.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/IR/BasicBlock.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/InstrTypes.h"
22 #include "llvm/IR/Instruction.h"
23 #include "llvm/IR/Instructions.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/IR/PassManager.h"
26 #include "llvm/IR/PatternMatch.h"
27 #include "llvm/InitializePasses.h"
28 #include "llvm/Pass.h"
29 #include "llvm/Support/Casting.h"
30 #include "llvm/Support/CommandLine.h"
31 #include "llvm/Support/ErrorHandling.h"
32 #include "llvm/Support/raw_ostream.h"
33 #include <algorithm>
34 #include <cassert>
35 #include <utility>
36 
37 using namespace llvm;
38 using namespace llvm::PatternMatch;
39 
40 static cl::opt<bool>
41     VerifyAssumptionCache("verify-assumption-cache", cl::Hidden,
42                           cl::desc("Enable verification of assumption cache"),
43                           cl::init(false));
44 
45 SmallVector<AssumptionCache::ResultElem, 1> &
46 AssumptionCache::getOrInsertAffectedValues(Value *V) {
47   // Try using find_as first to avoid creating extra value handles just for the
48   // purpose of doing the lookup.
49   auto AVI = AffectedValues.find_as(V);
50   if (AVI != AffectedValues.end())
51     return AVI->second;
52 
53   auto AVIP = AffectedValues.insert(
54       {AffectedValueCallbackVH(V, this), SmallVector<ResultElem, 1>()});
55   return AVIP.first->second;
56 }
57 
58 static void
59 findAffectedValues(CallInst *CI,
60                    SmallVectorImpl<AssumptionCache::ResultElem> &Affected) {
61   // Note: This code must be kept in-sync with the code in
62   // computeKnownBitsFromAssume in ValueTracking.
63 
64   auto AddAffected = [&Affected](Value *V, unsigned Idx =
65                                                AssumptionCache::ExprResultIdx) {
66     if (isa<Argument>(V)) {
67       Affected.push_back({V, Idx});
68     } else if (auto *I = dyn_cast<Instruction>(V)) {
69       Affected.push_back({I, Idx});
70 
71       // Peek through unary operators to find the source of the condition.
72       Value *Op;
73       if (match(I, m_BitCast(m_Value(Op))) ||
74           match(I, m_PtrToInt(m_Value(Op))) || match(I, m_Not(m_Value(Op)))) {
75         if (isa<Instruction>(Op) || isa<Argument>(Op))
76           Affected.push_back({Op, Idx});
77       }
78     }
79   };
80 
81   for (unsigned Idx = 0; Idx != CI->getNumOperandBundles(); Idx++) {
82     if (CI->getOperandBundleAt(Idx).Inputs.size() > ABA_WasOn &&
83         CI->getOperandBundleAt(Idx).getTagName() != IgnoreBundleTag)
84       AddAffected(CI->getOperandBundleAt(Idx).Inputs[ABA_WasOn], Idx);
85   }
86 
87   Value *Cond = CI->getArgOperand(0), *A, *B;
88   AddAffected(Cond);
89 
90   CmpInst::Predicate Pred;
91   if (match(Cond, m_ICmp(Pred, m_Value(A), m_Value(B)))) {
92     AddAffected(A);
93     AddAffected(B);
94 
95     if (Pred == ICmpInst::ICMP_EQ) {
96       // For equality comparisons, we handle the case of bit inversion.
97       auto AddAffectedFromEq = [&AddAffected](Value *V) {
98         Value *A;
99         if (match(V, m_Not(m_Value(A)))) {
100           AddAffected(A);
101           V = A;
102         }
103 
104         Value *B;
105         // (A & B) or (A | B) or (A ^ B).
106         if (match(V, m_BitwiseLogic(m_Value(A), m_Value(B)))) {
107           AddAffected(A);
108           AddAffected(B);
109         // (A << C) or (A >>_s C) or (A >>_u C) where C is some constant.
110         } else if (match(V, m_Shift(m_Value(A), m_ConstantInt()))) {
111           AddAffected(A);
112         }
113       };
114 
115       AddAffectedFromEq(A);
116       AddAffectedFromEq(B);
117     }
118   }
119 }
120 
121 void AssumptionCache::updateAffectedValues(CallInst *CI) {
122   SmallVector<AssumptionCache::ResultElem, 16> Affected;
123   findAffectedValues(CI, Affected);
124 
125   for (auto &AV : Affected) {
126     auto &AVV = getOrInsertAffectedValues(AV.Assume);
127     if (std::find_if(AVV.begin(), AVV.end(), [&](ResultElem &Elem) {
128           return Elem.Assume == CI && Elem.Index == AV.Index;
129         }) == AVV.end())
130       AVV.push_back({CI, AV.Index});
131   }
132 }
133 
134 void AssumptionCache::unregisterAssumption(CallInst *CI) {
135   SmallVector<AssumptionCache::ResultElem, 16> Affected;
136   findAffectedValues(CI, Affected);
137 
138   for (auto &AV : Affected) {
139     auto AVI = AffectedValues.find_as(AV.Assume);
140     if (AVI == AffectedValues.end())
141       continue;
142     bool Found = false;
143     bool HasNonnull = false;
144     for (ResultElem &Elem : AVI->second) {
145       if (Elem.Assume == CI) {
146         Found = true;
147         Elem.Assume = nullptr;
148       }
149       HasNonnull |= !!Elem.Assume;
150       if (HasNonnull && Found)
151         break;
152     }
153     assert(Found && "already unregistered or incorrect cache state");
154     if (!HasNonnull)
155       AffectedValues.erase(AVI);
156   }
157 
158   AssumeHandles.erase(
159       remove_if(AssumeHandles, [CI](ResultElem &RE) { return CI == RE; }),
160       AssumeHandles.end());
161 }
162 
163 void AssumptionCache::AffectedValueCallbackVH::deleted() {
164   auto AVI = AC->AffectedValues.find(getValPtr());
165   if (AVI != AC->AffectedValues.end())
166     AC->AffectedValues.erase(AVI);
167   // 'this' now dangles!
168 }
169 
170 void AssumptionCache::transferAffectedValuesInCache(Value *OV, Value *NV) {
171   auto &NAVV = getOrInsertAffectedValues(NV);
172   auto AVI = AffectedValues.find(OV);
173   if (AVI == AffectedValues.end())
174     return;
175 
176   for (auto &A : AVI->second)
177     if (!llvm::is_contained(NAVV, A))
178       NAVV.push_back(A);
179   AffectedValues.erase(OV);
180 }
181 
182 void AssumptionCache::AffectedValueCallbackVH::allUsesReplacedWith(Value *NV) {
183   if (!isa<Instruction>(NV) && !isa<Argument>(NV))
184     return;
185 
186   // Any assumptions that affected this value now affect the new value.
187 
188   AC->transferAffectedValuesInCache(getValPtr(), NV);
189   // 'this' now might dangle! If the AffectedValues map was resized to add an
190   // entry for NV then this object might have been destroyed in favor of some
191   // copy in the grown map.
192 }
193 
194 void AssumptionCache::scanFunction() {
195   assert(!Scanned && "Tried to scan the function twice!");
196   assert(AssumeHandles.empty() && "Already have assumes when scanning!");
197 
198   // Go through all instructions in all blocks, add all calls to @llvm.assume
199   // to this cache.
200   for (BasicBlock &B : F)
201     for (Instruction &II : B)
202       if (match(&II, m_Intrinsic<Intrinsic::assume>()))
203         AssumeHandles.push_back({&II, ExprResultIdx});
204 
205   // Mark the scan as complete.
206   Scanned = true;
207 
208   // Update affected values.
209   for (auto &A : AssumeHandles)
210     updateAffectedValues(cast<CallInst>(A));
211 }
212 
213 void AssumptionCache::registerAssumption(CallInst *CI) {
214   assert(match(CI, m_Intrinsic<Intrinsic::assume>()) &&
215          "Registered call does not call @llvm.assume");
216 
217   // If we haven't scanned the function yet, just drop this assumption. It will
218   // be found when we scan later.
219   if (!Scanned)
220     return;
221 
222   AssumeHandles.push_back({CI, ExprResultIdx});
223 
224 #ifndef NDEBUG
225   assert(CI->getParent() &&
226          "Cannot register @llvm.assume call not in a basic block");
227   assert(&F == CI->getParent()->getParent() &&
228          "Cannot register @llvm.assume call not in this function");
229 
230   // We expect the number of assumptions to be small, so in an asserts build
231   // check that we don't accumulate duplicates and that all assumptions point
232   // to the same function.
233   SmallPtrSet<Value *, 16> AssumptionSet;
234   for (auto &VH : AssumeHandles) {
235     if (!VH)
236       continue;
237 
238     assert(&F == cast<Instruction>(VH)->getParent()->getParent() &&
239            "Cached assumption not inside this function!");
240     assert(match(cast<CallInst>(VH), m_Intrinsic<Intrinsic::assume>()) &&
241            "Cached something other than a call to @llvm.assume!");
242     assert(AssumptionSet.insert(VH).second &&
243            "Cache contains multiple copies of a call!");
244   }
245 #endif
246 
247   updateAffectedValues(CI);
248 }
249 
250 AnalysisKey AssumptionAnalysis::Key;
251 
252 PreservedAnalyses AssumptionPrinterPass::run(Function &F,
253                                              FunctionAnalysisManager &AM) {
254   AssumptionCache &AC = AM.getResult<AssumptionAnalysis>(F);
255 
256   OS << "Cached assumptions for function: " << F.getName() << "\n";
257   for (auto &VH : AC.assumptions())
258     if (VH)
259       OS << "  " << *cast<CallInst>(VH)->getArgOperand(0) << "\n";
260 
261   return PreservedAnalyses::all();
262 }
263 
264 void AssumptionCacheTracker::FunctionCallbackVH::deleted() {
265   auto I = ACT->AssumptionCaches.find_as(cast<Function>(getValPtr()));
266   if (I != ACT->AssumptionCaches.end())
267     ACT->AssumptionCaches.erase(I);
268   // 'this' now dangles!
269 }
270 
271 AssumptionCache &AssumptionCacheTracker::getAssumptionCache(Function &F) {
272   // We probe the function map twice to try and avoid creating a value handle
273   // around the function in common cases. This makes insertion a bit slower,
274   // but if we have to insert we're going to scan the whole function so that
275   // shouldn't matter.
276   auto I = AssumptionCaches.find_as(&F);
277   if (I != AssumptionCaches.end())
278     return *I->second;
279 
280   // Ok, build a new cache by scanning the function, insert it and the value
281   // handle into our map, and return the newly populated cache.
282   auto IP = AssumptionCaches.insert(std::make_pair(
283       FunctionCallbackVH(&F, this), std::make_unique<AssumptionCache>(F)));
284   assert(IP.second && "Scanning function already in the map?");
285   return *IP.first->second;
286 }
287 
288 AssumptionCache *AssumptionCacheTracker::lookupAssumptionCache(Function &F) {
289   auto I = AssumptionCaches.find_as(&F);
290   if (I != AssumptionCaches.end())
291     return I->second.get();
292   return nullptr;
293 }
294 
295 void AssumptionCacheTracker::verifyAnalysis() const {
296   // FIXME: In the long term the verifier should not be controllable with a
297   // flag. We should either fix all passes to correctly update the assumption
298   // cache and enable the verifier unconditionally or somehow arrange for the
299   // assumption list to be updated automatically by passes.
300   if (!VerifyAssumptionCache)
301     return;
302 
303   SmallPtrSet<const CallInst *, 4> AssumptionSet;
304   for (const auto &I : AssumptionCaches) {
305     for (auto &VH : I.second->assumptions())
306       if (VH)
307         AssumptionSet.insert(cast<CallInst>(VH));
308 
309     for (const BasicBlock &B : cast<Function>(*I.first))
310       for (const Instruction &II : B)
311         if (match(&II, m_Intrinsic<Intrinsic::assume>()) &&
312             !AssumptionSet.count(cast<CallInst>(&II)))
313           report_fatal_error("Assumption in scanned function not in cache");
314   }
315 }
316 
317 AssumptionCacheTracker::AssumptionCacheTracker() : ImmutablePass(ID) {
318   initializeAssumptionCacheTrackerPass(*PassRegistry::getPassRegistry());
319 }
320 
321 AssumptionCacheTracker::~AssumptionCacheTracker() = default;
322 
323 char AssumptionCacheTracker::ID = 0;
324 
325 INITIALIZE_PASS(AssumptionCacheTracker, "assumption-cache-tracker",
326                 "Assumption Cache Tracker", false, true)
327