1 //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection  ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Populates the VFABI attribute with the scalar-to-vector mappings
10 // from the TargetLibraryInfo.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
15 #include "llvm/ADT/Statistic.h"
16 #include "llvm/Analysis/DemandedBits.h"
17 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
18 #include "llvm/Analysis/VectorUtils.h"
19 #include "llvm/IR/InstIterator.h"
20 #include "llvm/IR/IntrinsicInst.h"
21 #include "llvm/Transforms/Utils.h"
22 #include "llvm/Transforms/Utils/ModuleUtils.h"
23 
24 using namespace llvm;
25 
26 #define DEBUG_TYPE "inject-tli-mappings"
27 
28 STATISTIC(NumCallInjected,
29           "Number of calls in which the mappings have been injected.");
30 
31 STATISTIC(NumVFDeclAdded,
32           "Number of function declarations that have been added.");
33 STATISTIC(NumCompUsedAdded,
34           "Number of `@llvm.compiler.used` operands that have been added.");
35 
36 /// Helper function to map the TLI name to a strings that holds
37 /// scalar-to-vector mapping.
38 ///
39 ///    _ZGV<isa><mask><vlen><vparams>_<scalarname>(<vectorname>)
40 ///
41 /// where:
42 ///
43 /// <isa> = "_LLVM_"
44 /// <mask> = "N". Note: TLI does not support masked interfaces.
45 /// <vlen> = Number of concurrent lanes, stored in the `VectorizationFactor`
46 ///          field of the `VecDesc` struct.
47 /// <vparams> = "v", as many as are the number of parameters of CI.
48 /// <scalarname> = the name of the scalar function called by CI.
49 /// <vectorname> = the name of the vector function mapped by the TLI.
50 static std::string mangleTLIName(StringRef VectorName, const CallInst &CI,
51                                  unsigned VF) {
52   SmallString<256> Buffer;
53   llvm::raw_svector_ostream Out(Buffer);
54   Out << "_ZGV" << VFABI::_LLVM_ << "N" << VF;
55   for (unsigned I = 0; I < CI.getNumArgOperands(); ++I)
56     Out << "v";
57   Out << "_" << CI.getCalledFunction()->getName() << "(" << VectorName << ")";
58   return std::string(Out.str());
59 }
60 
61 /// A helper function for converting Scalar types to vector types.
62 /// If the incoming type is void, we return void. If the VF is 1, we return
63 /// the scalar type.
64 static Type *ToVectorTy(Type *Scalar, unsigned VF, bool isScalable = false) {
65   if (Scalar->isVoidTy() || VF == 1)
66     return Scalar;
67   return VectorType::get(Scalar, {VF, isScalable});
68 }
69 
70 /// A helper function that adds the vector function declaration that
71 /// vectorizes the CallInst CI with a vectorization factor of VF
72 /// lanes. The TLI assumes that all parameters and the return type of
73 /// CI (other than void) need to be widened to a VectorType of VF
74 /// lanes.
75 static void addVariantDeclaration(CallInst &CI, const unsigned VF,
76                                   const StringRef VFName) {
77   Module *M = CI.getModule();
78 
79   // Add function declaration.
80   Type *RetTy = ToVectorTy(CI.getType(), VF);
81   SmallVector<Type *, 4> Tys;
82   for (Value *ArgOperand : CI.arg_operands())
83     Tys.push_back(ToVectorTy(ArgOperand->getType(), VF));
84   assert(!CI.getFunctionType()->isVarArg() &&
85          "VarArg functions are not supported.");
86   FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false);
87   Function *VectorF =
88       Function::Create(FTy, Function::ExternalLinkage, VFName, M);
89   VectorF->copyAttributesFrom(CI.getCalledFunction());
90   ++NumVFDeclAdded;
91   LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
92                     << "` of type " << *(VectorF->getType()) << "\n");
93 
94   // Make function declaration (without a body) "sticky" in the IR by
95   // listing it in the @llvm.compiler.used intrinsic.
96   assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` "
97                              "only on declarations.");
98   appendToCompilerUsed(*M, {VectorF});
99   LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
100                     << "` to `@llvm.compiler.used`.\n");
101   ++NumCompUsedAdded;
102 }
103 
104 static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
105   // This is needed to make sure we don't query the TLI for calls to
106   // bitcast of function pointers, like `%call = call i32 (i32*, ...)
107   // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
108   // as such calls make the `isFunctionVectorizable` raise an
109   // exception.
110   if (CI.isNoBuiltin() || !CI.getCalledFunction())
111     return;
112 
113   const std::string ScalarName = std::string(CI.getCalledFunction()->getName());
114   // Nothing to be done if the TLI thinks the function is not
115   // vectorizable.
116   if (!TLI.isFunctionVectorizable(ScalarName))
117     return;
118   SmallVector<std::string, 8> Mappings;
119   VFABI::getVectorVariantNames(CI, Mappings);
120   Module *M = CI.getModule();
121   const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(),
122                                                    Mappings.end());
123   //  All VFs in the TLI are powers of 2.
124   for (unsigned VF = 2, WidestVF = TLI.getWidestVF(ScalarName); VF <= WidestVF;
125        VF *= 2) {
126     const std::string TLIName =
127         std::string(TLI.getVectorizedFunction(ScalarName, VF));
128     if (!TLIName.empty()) {
129       std::string MangledName = mangleTLIName(TLIName, CI, VF);
130       if (!OriginalSetOfMappings.count(MangledName)) {
131         Mappings.push_back(MangledName);
132         ++NumCallInjected;
133       }
134       Function *VariantF = M->getFunction(TLIName);
135       if (!VariantF)
136         addVariantDeclaration(CI, VF, TLIName);
137     }
138   }
139 
140   VFABI::setVectorVariantNames(&CI, Mappings);
141 }
142 
143 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
144   for (auto &I : instructions(F))
145     if (auto CI = dyn_cast<CallInst>(&I))
146       addMappingsFromTLI(TLI, *CI);
147   // Even if the pass adds IR attributes, the analyses are preserved.
148   return false;
149 }
150 
151 ////////////////////////////////////////////////////////////////////////////////
152 // New pass manager implementation.
153 ////////////////////////////////////////////////////////////////////////////////
154 PreservedAnalyses InjectTLIMappings::run(Function &F,
155                                          FunctionAnalysisManager &AM) {
156   const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
157   runImpl(TLI, F);
158   // Even if the pass adds IR attributes, the analyses are preserved.
159   return PreservedAnalyses::all();
160 }
161 
162 ////////////////////////////////////////////////////////////////////////////////
163 // Legacy PM Implementation.
164 ////////////////////////////////////////////////////////////////////////////////
165 bool InjectTLIMappingsLegacy::runOnFunction(Function &F) {
166   const TargetLibraryInfo &TLI =
167       getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
168   return runImpl(TLI, F);
169 }
170 
171 void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
172   AU.setPreservesCFG();
173   AU.addRequired<TargetLibraryInfoWrapperPass>();
174   AU.addPreserved<TargetLibraryInfoWrapperPass>();
175   AU.addPreserved<ScalarEvolutionWrapperPass>();
176   AU.addPreserved<AAResultsWrapperPass>();
177   AU.addPreserved<LoopAccessLegacyAnalysis>();
178   AU.addPreserved<DemandedBitsWrapperPass>();
179   AU.addPreserved<OptimizationRemarkEmitterWrapperPass>();
180 }
181 
182 ////////////////////////////////////////////////////////////////////////////////
183 // Legacy Pass manager initialization
184 ////////////////////////////////////////////////////////////////////////////////
185 char InjectTLIMappingsLegacy::ID = 0;
186 
187 INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE,
188                       "Inject TLI Mappings", false, false)
189 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
190 INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings",
191                     false, false)
192 
193 FunctionPass *llvm::createInjectTLIMappingsLegacyPass() {
194   return new InjectTLIMappingsLegacy();
195 }
196