1 //===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // The pass emits SPIRV intrinsics keeping essential high-level information for
10 // the translation of LLVM IR to SPIR-V.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SPIRV.h"
15 #include "SPIRVTargetMachine.h"
16 #include "SPIRVUtils.h"
17 #include "llvm/IR/IRBuilder.h"
18 #include "llvm/IR/InstIterator.h"
19 #include "llvm/IR/InstVisitor.h"
20 #include "llvm/IR/IntrinsicsSPIRV.h"
21 
22 #include <queue>
23 
24 // This pass performs the following transformation on LLVM IR level required
25 // for the following translation to SPIR-V:
26 // - replaces direct usages of aggregate constants with target-specific
27 //   intrinsics;
28 // - replaces aggregates-related instructions (extract/insert, ld/st, etc)
29 //   with a target-specific intrinsics;
30 // - emits intrinsics for the global variable initializers since IRTranslator
31 //   doesn't handle them and it's not very convenient to translate them
32 //   ourselves;
33 // - emits intrinsics to keep track of the string names assigned to the values;
34 // - emits intrinsics to keep track of constants (this is necessary to have an
35 //   LLVM IR constant after the IRTranslation is completed) for their further
36 //   deduplication;
37 // - emits intrinsics to keep track of original LLVM types of the values
38 //   to be able to emit proper SPIR-V types eventually.
39 //
40 // TODO: consider removing spv.track.constant in favor of spv.assign.type.
41 
42 using namespace llvm;
43 
44 namespace llvm {
45 void initializeSPIRVEmitIntrinsicsPass(PassRegistry &);
46 } // namespace llvm
47 
48 namespace {
49 class SPIRVEmitIntrinsics
50     : public FunctionPass,
51       public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
52   SPIRVTargetMachine *TM = nullptr;
53   IRBuilder<> *IRB = nullptr;
54   Function *F = nullptr;
55   bool TrackConstants = true;
56   DenseMap<Instruction *, Constant *> AggrConsts;
57   DenseSet<Instruction *> AggrStores;
58   void preprocessCompositeConstants();
59   CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
60                             Value *Arg, Value *Arg2) {
61     ConstantAsMetadata *CM = ValueAsMetadata::getConstant(Arg);
62     MDTuple *TyMD = MDNode::get(F->getContext(), CM);
63     MetadataAsValue *VMD = MetadataAsValue::get(F->getContext(), TyMD);
64     return IRB->CreateIntrinsic(IntrID, {Types}, {Arg2, VMD});
65   }
66   void replaceMemInstrUses(Instruction *Old, Instruction *New);
67   void processInstrAfterVisit(Instruction *I);
68   void insertAssignTypeIntrs(Instruction *I);
69   void processGlobalValue(GlobalVariable &GV);
70 
71 public:
72   static char ID;
73   SPIRVEmitIntrinsics() : FunctionPass(ID) {
74     initializeSPIRVEmitIntrinsicsPass(*PassRegistry::getPassRegistry());
75   }
76   SPIRVEmitIntrinsics(SPIRVTargetMachine *_TM) : FunctionPass(ID), TM(_TM) {
77     initializeSPIRVEmitIntrinsicsPass(*PassRegistry::getPassRegistry());
78   }
79   Instruction *visitInstruction(Instruction &I) { return &I; }
80   Instruction *visitSwitchInst(SwitchInst &I);
81   Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
82   Instruction *visitBitCastInst(BitCastInst &I);
83   Instruction *visitInsertElementInst(InsertElementInst &I);
84   Instruction *visitExtractElementInst(ExtractElementInst &I);
85   Instruction *visitInsertValueInst(InsertValueInst &I);
86   Instruction *visitExtractValueInst(ExtractValueInst &I);
87   Instruction *visitLoadInst(LoadInst &I);
88   Instruction *visitStoreInst(StoreInst &I);
89   Instruction *visitAllocaInst(AllocaInst &I);
90   bool runOnFunction(Function &F) override;
91 };
92 } // namespace
93 
94 char SPIRVEmitIntrinsics::ID = 0;
95 
96 INITIALIZE_PASS(SPIRVEmitIntrinsics, "emit-intrinsics", "SPIRV emit intrinsics",
97                 false, false)
98 
99 static inline bool isAssignTypeInstr(const Instruction *I) {
100   return isa<IntrinsicInst>(I) &&
101          cast<IntrinsicInst>(I)->getIntrinsicID() == Intrinsic::spv_assign_type;
102 }
103 
104 static bool isMemInstrToReplace(Instruction *I) {
105   return isa<StoreInst>(I) || isa<LoadInst>(I) || isa<InsertValueInst>(I) ||
106          isa<ExtractValueInst>(I);
107 }
108 
109 static bool isAggrToReplace(const Value *V) {
110   return isa<ConstantAggregate>(V) || isa<ConstantDataArray>(V) ||
111          (isa<ConstantAggregateZero>(V) && !V->getType()->isVectorTy());
112 }
113 
114 static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
115   if (isa<PHINode>(I))
116     B.SetInsertPoint(I->getParent(), I->getParent()->getFirstInsertionPt());
117   else
118     B.SetInsertPoint(I);
119 }
120 
121 static bool requireAssignType(Instruction *I) {
122   IntrinsicInst *Intr = dyn_cast<IntrinsicInst>(I);
123   if (Intr) {
124     switch (Intr->getIntrinsicID()) {
125     case Intrinsic::invariant_start:
126     case Intrinsic::invariant_end:
127       return false;
128     }
129   }
130   return true;
131 }
132 
133 void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
134                                               Instruction *New) {
135   while (!Old->user_empty()) {
136     auto *U = Old->user_back();
137     if (isMemInstrToReplace(U) || isa<ReturnInst>(U)) {
138       U->replaceUsesOfWith(Old, New);
139     } else if (isAssignTypeInstr(U)) {
140       IRB->SetInsertPoint(U);
141       SmallVector<Value *, 2> Args = {New, U->getOperand(1)};
142       IRB->CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args);
143       U->eraseFromParent();
144     } else {
145       llvm_unreachable("illegal aggregate intrinsic user");
146     }
147   }
148   Old->eraseFromParent();
149 }
150 
151 void SPIRVEmitIntrinsics::preprocessCompositeConstants() {
152   std::queue<Instruction *> Worklist;
153   for (auto &I : instructions(F))
154     Worklist.push(&I);
155 
156   while (!Worklist.empty()) {
157     auto *I = Worklist.front();
158     assert(I);
159     bool KeepInst = false;
160     for (const auto &Op : I->operands()) {
161       auto BuildCompositeIntrinsic = [&KeepInst, &Worklist, &I, &Op,
162                                       this](Constant *AggrC,
163                                             ArrayRef<Value *> Args) {
164         IRB->SetInsertPoint(I);
165         auto *CCI =
166             IRB->CreateIntrinsic(Intrinsic::spv_const_composite, {}, {Args});
167         Worklist.push(CCI);
168         I->replaceUsesOfWith(Op, CCI);
169         KeepInst = true;
170         AggrConsts[CCI] = AggrC;
171       };
172 
173       if (auto *AggrC = dyn_cast<ConstantAggregate>(Op)) {
174         SmallVector<Value *> Args(AggrC->op_begin(), AggrC->op_end());
175         BuildCompositeIntrinsic(AggrC, Args);
176       } else if (auto *AggrC = dyn_cast<ConstantDataArray>(Op)) {
177         SmallVector<Value *> Args;
178         for (unsigned i = 0; i < AggrC->getNumElements(); ++i)
179           Args.push_back(AggrC->getElementAsConstant(i));
180         BuildCompositeIntrinsic(AggrC, Args);
181       } else if (isa<ConstantAggregateZero>(Op) &&
182                  !Op->getType()->isVectorTy()) {
183         auto *AggrC = cast<ConstantAggregateZero>(Op);
184         SmallVector<Value *> Args(AggrC->op_begin(), AggrC->op_end());
185         BuildCompositeIntrinsic(AggrC, Args);
186       }
187     }
188     if (!KeepInst)
189       Worklist.pop();
190   }
191 }
192 
193 Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
194   SmallVector<Value *, 4> Args;
195   for (auto &Op : I.operands())
196     if (Op.get()->getType()->isSized())
197       Args.push_back(Op);
198   IRB->CreateIntrinsic(Intrinsic::spv_switch, {I.getOperand(0)->getType()},
199                        {Args});
200   return &I;
201 }
202 
203 Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
204   SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(0)->getType()};
205   SmallVector<Value *, 4> Args;
206   Args.push_back(IRB->getInt1(I.isInBounds()));
207   for (auto &Op : I.operands())
208     Args.push_back(Op);
209   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
210   I.replaceAllUsesWith(NewI);
211   I.eraseFromParent();
212   return NewI;
213 }
214 
215 Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
216   SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(0)->getType()};
217   SmallVector<Value *> Args(I.op_begin(), I.op_end());
218   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args});
219   std::string InstName = I.hasName() ? I.getName().str() : "";
220   I.replaceAllUsesWith(NewI);
221   I.eraseFromParent();
222   NewI->setName(InstName);
223   return NewI;
224 }
225 
226 Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
227   SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(0)->getType(),
228                                   I.getOperand(1)->getType(),
229                                   I.getOperand(2)->getType()};
230   SmallVector<Value *> Args(I.op_begin(), I.op_end());
231   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
232   std::string InstName = I.hasName() ? I.getName().str() : "";
233   I.replaceAllUsesWith(NewI);
234   I.eraseFromParent();
235   NewI->setName(InstName);
236   return NewI;
237 }
238 
239 Instruction *
240 SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
241   SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
242                                   I.getIndexOperand()->getType()};
243   SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
244   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
245   std::string InstName = I.hasName() ? I.getName().str() : "";
246   I.replaceAllUsesWith(NewI);
247   I.eraseFromParent();
248   NewI->setName(InstName);
249   return NewI;
250 }
251 
252 Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
253   SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
254   SmallVector<Value *> Args;
255   for (auto &Op : I.operands())
256     if (isa<UndefValue>(Op))
257       Args.push_back(UndefValue::get(IRB->getInt32Ty()));
258     else
259       Args.push_back(Op);
260   for (auto &Op : I.indices())
261     Args.push_back(IRB->getInt32(Op));
262   Instruction *NewI =
263       IRB->CreateIntrinsic(Intrinsic::spv_insertv, {Types}, {Args});
264   replaceMemInstrUses(&I, NewI);
265   return NewI;
266 }
267 
268 Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
269   SmallVector<Value *> Args;
270   for (auto &Op : I.operands())
271     Args.push_back(Op);
272   for (auto &Op : I.indices())
273     Args.push_back(IRB->getInt32(Op));
274   auto *NewI =
275       IRB->CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args});
276   I.replaceAllUsesWith(NewI);
277   I.eraseFromParent();
278   return NewI;
279 }
280 
281 Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
282   if (!I.getType()->isAggregateType())
283     return &I;
284   TrackConstants = false;
285   const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
286   MachineMemOperand::Flags Flags =
287       TLI->getLoadMemOperandFlags(I, F->getParent()->getDataLayout());
288   auto *NewI =
289       IRB->CreateIntrinsic(Intrinsic::spv_load, {I.getOperand(0)->getType()},
290                            {I.getPointerOperand(), IRB->getInt16(Flags),
291                             IRB->getInt8(I.getAlign().value())});
292   replaceMemInstrUses(&I, NewI);
293   return NewI;
294 }
295 
296 Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
297   if (!AggrStores.contains(&I))
298     return &I;
299   TrackConstants = false;
300   const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
301   MachineMemOperand::Flags Flags =
302       TLI->getStoreMemOperandFlags(I, F->getParent()->getDataLayout());
303   auto *PtrOp = I.getPointerOperand();
304   auto *NewI =
305       IRB->CreateIntrinsic(Intrinsic::spv_store, {PtrOp->getType()},
306                            {I.getValueOperand(), PtrOp, IRB->getInt16(Flags),
307                             IRB->getInt8(I.getAlign().value())});
308   I.eraseFromParent();
309   return NewI;
310 }
311 
312 Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
313   TrackConstants = false;
314   return &I;
315 }
316 
317 void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV) {
318   // Skip special artifical variable llvm.global.annotations.
319   if (GV.getName() == "llvm.global.annotations")
320     return;
321   if (GV.hasInitializer() && !isa<UndefValue>(GV.getInitializer())) {
322     Constant *Init = GV.getInitializer();
323     Type *Ty = isAggrToReplace(Init) ? IRB->getInt32Ty() : Init->getType();
324     Constant *Const = isAggrToReplace(Init) ? IRB->getInt32(1) : Init;
325     auto *InitInst = IRB->CreateIntrinsic(Intrinsic::spv_init_global,
326                                           {GV.getType(), Ty}, {&GV, Const});
327     InitInst->setArgOperand(1, Init);
328   }
329   if ((!GV.hasInitializer() || isa<UndefValue>(GV.getInitializer())) &&
330       GV.getNumUses() == 0)
331     IRB->CreateIntrinsic(Intrinsic::spv_unref_global, GV.getType(), &GV);
332 }
333 
334 void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I) {
335   Type *Ty = I->getType();
336   if (!Ty->isVoidTy() && requireAssignType(I)) {
337     setInsertPointSkippingPhis(*IRB, I->getNextNode());
338     Type *TypeToAssign = Ty;
339     if (auto *II = dyn_cast<IntrinsicInst>(I)) {
340       if (II->getIntrinsicID() == Intrinsic::spv_const_composite) {
341         auto t = AggrConsts.find(II);
342         assert(t != AggrConsts.end());
343         TypeToAssign = t->second->getType();
344       }
345     }
346     Constant *Const = Constant::getNullValue(TypeToAssign);
347     buildIntrWithMD(Intrinsic::spv_assign_type, {Ty}, Const, I);
348   }
349   for (const auto &Op : I->operands()) {
350     if (isa<ConstantPointerNull>(Op) || isa<UndefValue>(Op) ||
351         // Check GetElementPtrConstantExpr case.
352         (isa<ConstantExpr>(Op) && isa<GEPOperator>(Op))) {
353       IRB->SetInsertPoint(I);
354       buildIntrWithMD(Intrinsic::spv_assign_type, {Op->getType()}, Op, Op);
355     }
356   }
357   // StoreInst's operand type can be changed in the next stage so we need to
358   // store it in the set.
359   if (isa<StoreInst>(I) &&
360       cast<StoreInst>(I)->getValueOperand()->getType()->isAggregateType())
361     AggrStores.insert(I);
362 }
363 
364 void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I) {
365   auto *II = dyn_cast<IntrinsicInst>(I);
366   if (II && II->getIntrinsicID() == Intrinsic::spv_const_composite &&
367       TrackConstants) {
368     IRB->SetInsertPoint(I->getNextNode());
369     Type *Ty = IRB->getInt32Ty();
370     auto t = AggrConsts.find(I);
371     assert(t != AggrConsts.end());
372     auto *NewOp =
373         buildIntrWithMD(Intrinsic::spv_track_constant, {Ty, Ty}, t->second, I);
374     I->replaceAllUsesWith(NewOp);
375     NewOp->setArgOperand(0, I);
376   }
377   for (const auto &Op : I->operands()) {
378     if ((isa<ConstantAggregateZero>(Op) && Op->getType()->isVectorTy()) ||
379         isa<PHINode>(I) || isa<SwitchInst>(I))
380       TrackConstants = false;
381     if (isa<ConstantData>(Op) && TrackConstants) {
382       unsigned OpNo = Op.getOperandNo();
383       if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
384                  (II->paramHasAttr(OpNo, Attribute::ImmArg))))
385         continue;
386       IRB->SetInsertPoint(I);
387       auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant,
388                                     {Op->getType(), Op->getType()}, Op, Op);
389       I->setOperand(OpNo, NewOp);
390     }
391   }
392   if (I->hasName()) {
393     setInsertPointSkippingPhis(*IRB, I->getNextNode());
394     std::vector<Value *> Args = {I};
395     addStringImm(I->getName(), *IRB, Args);
396     IRB->CreateIntrinsic(Intrinsic::spv_assign_name, {I->getType()}, Args);
397   }
398 }
399 
400 bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
401   if (Func.isDeclaration())
402     return false;
403   F = &Func;
404   IRB = new IRBuilder<>(Func.getContext());
405   AggrConsts.clear();
406   AggrStores.clear();
407 
408   IRB->SetInsertPoint(&Func.getEntryBlock().front());
409 
410   for (auto &GV : Func.getParent()->globals())
411     processGlobalValue(GV);
412 
413   preprocessCompositeConstants();
414   SmallVector<Instruction *> Worklist;
415   for (auto &I : instructions(Func))
416     Worklist.push_back(&I);
417 
418   for (auto &I : Worklist)
419     insertAssignTypeIntrs(I);
420 
421   for (auto *I : Worklist) {
422     TrackConstants = true;
423     if (!I->getType()->isVoidTy() || isa<StoreInst>(I))
424       IRB->SetInsertPoint(I->getNextNode());
425     I = visit(*I);
426     processInstrAfterVisit(I);
427   }
428   return true;
429 }
430 
431 FunctionPass *llvm::createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM) {
432   return new SPIRVEmitIntrinsics(TM);
433 }
434