1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the IRBuilder class, which is used as a convenient way
10 // to create LLVM instructions with a consistent and simplified interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/None.h"
17 #include "llvm/IR/Constant.h"
18 #include "llvm/IR/Constants.h"
19 #include "llvm/IR/DerivedTypes.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/GlobalVariable.h"
23 #include "llvm/IR/IntrinsicInst.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Operator.h"
27 #include "llvm/IR/NoFolder.h"
28 #include "llvm/IR/Statepoint.h"
29 #include "llvm/IR/Type.h"
30 #include "llvm/IR/Value.h"
31 #include "llvm/Support/Casting.h"
32 #include "llvm/Support/MathExtras.h"
33 #include <cassert>
34 #include <cstdint>
35 #include <vector>
36 
37 using namespace llvm;
38 
39 /// CreateGlobalString - Make a new global variable with an initializer that
40 /// has array of i8 type filled in with the nul terminated string value
41 /// specified.  If Name is specified, it is the name of the global variable
42 /// created.
43 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
44                                                   const Twine &Name,
45                                                   unsigned AddressSpace) {
46   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
47   Module &M = *BB->getParent()->getParent();
48   auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
49                                 GlobalValue::PrivateLinkage, StrConstant, Name,
50                                 nullptr, GlobalVariable::NotThreadLocal,
51                                 AddressSpace);
52   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
53   GV->setAlignment(Align(1));
54   return GV;
55 }
56 
57 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
58   assert(BB && BB->getParent() && "No current function!");
59   return BB->getParent()->getReturnType();
60 }
61 
62 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
63   auto *PT = cast<PointerType>(Ptr->getType());
64   if (PT->getElementType()->isIntegerTy(8))
65     return Ptr;
66 
67   // Otherwise, we need to insert a bitcast.
68   return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace()));
69 }
70 
71 static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
72                                   IRBuilderBase *Builder,
73                                   const Twine &Name = "",
74                                   Instruction *FMFSource = nullptr) {
75   CallInst *CI = Builder->CreateCall(Callee, Ops, Name);
76   if (FMFSource)
77     CI->copyFastMathFlags(FMFSource);
78   return CI;
79 }
80 
81 CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
82                                       MaybeAlign Align, bool isVolatile,
83                                       MDNode *TBAATag, MDNode *ScopeTag,
84                                       MDNode *NoAliasTag) {
85   Ptr = getCastedInt8PtrValue(Ptr);
86   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
87   Type *Tys[] = { Ptr->getType(), Size->getType() };
88   Module *M = BB->getParent()->getParent();
89   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
90 
91   CallInst *CI = createCallHelper(TheFn, Ops, this);
92 
93   if (Align)
94     cast<MemSetInst>(CI)->setDestAlignment(Align->value());
95 
96   // Set the TBAA info if present.
97   if (TBAATag)
98     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
99 
100   if (ScopeTag)
101     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
102 
103   if (NoAliasTag)
104     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
105 
106   return CI;
107 }
108 
109 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
110     Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
111     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
112 
113   Ptr = getCastedInt8PtrValue(Ptr);
114   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
115   Type *Tys[] = {Ptr->getType(), Size->getType()};
116   Module *M = BB->getParent()->getParent();
117   Function *TheFn = Intrinsic::getDeclaration(
118       M, Intrinsic::memset_element_unordered_atomic, Tys);
119 
120   CallInst *CI = createCallHelper(TheFn, Ops, this);
121 
122   cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
123 
124   // Set the TBAA info if present.
125   if (TBAATag)
126     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
127 
128   if (ScopeTag)
129     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
130 
131   if (NoAliasTag)
132     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
133 
134   return CI;
135 }
136 
137 CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src,
138                                       unsigned SrcAlign, Value *Size,
139                                       bool isVolatile, MDNode *TBAATag,
140                                       MDNode *TBAAStructTag, MDNode *ScopeTag,
141                                       MDNode *NoAliasTag) {
142   return CreateMemCpy(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign),
143                       Size, isVolatile, TBAATag, TBAAStructTag, ScopeTag,
144                       NoAliasTag);
145 }
146 
147 CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, MaybeAlign DstAlign,
148                                       Value *Src, MaybeAlign SrcAlign,
149                                       Value *Size, bool isVolatile,
150                                       MDNode *TBAATag, MDNode *TBAAStructTag,
151                                       MDNode *ScopeTag, MDNode *NoAliasTag) {
152   Dst = getCastedInt8PtrValue(Dst);
153   Src = getCastedInt8PtrValue(Src);
154 
155   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
156   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
157   Module *M = BB->getParent()->getParent();
158   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
159 
160   CallInst *CI = createCallHelper(TheFn, Ops, this);
161 
162   auto* MCI = cast<MemCpyInst>(CI);
163   if (DstAlign)
164     MCI->setDestAlignment(*DstAlign);
165   if (SrcAlign)
166     MCI->setSourceAlignment(*SrcAlign);
167 
168   // Set the TBAA info if present.
169   if (TBAATag)
170     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
171 
172   // Set the TBAA Struct info if present.
173   if (TBAAStructTag)
174     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
175 
176   if (ScopeTag)
177     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
178 
179   if (NoAliasTag)
180     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
181 
182   return CI;
183 }
184 
185 CallInst *IRBuilderBase::CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign,
186                                             Value *Src, MaybeAlign SrcAlign,
187                                             Value *Size) {
188   Dst = getCastedInt8PtrValue(Dst);
189   Src = getCastedInt8PtrValue(Src);
190   Value *IsVolatile = getInt1(false);
191 
192   Value *Ops[] = {Dst, Src, Size, IsVolatile};
193   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
194   Function *F = BB->getParent();
195   Module *M = F->getParent();
196   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys);
197 
198   CallInst *CI = createCallHelper(TheFn, Ops, this);
199 
200   auto *MCI = cast<MemCpyInlineInst>(CI);
201   if (DstAlign)
202     MCI->setDestAlignment(*DstAlign);
203   if (SrcAlign)
204     MCI->setSourceAlignment(*SrcAlign);
205 
206   return CI;
207 }
208 
209 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
210     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
211     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
212     MDNode *ScopeTag, MDNode *NoAliasTag) {
213   assert(DstAlign >= ElementSize &&
214          "Pointer alignment must be at least element size");
215   assert(SrcAlign >= ElementSize &&
216          "Pointer alignment must be at least element size");
217   Dst = getCastedInt8PtrValue(Dst);
218   Src = getCastedInt8PtrValue(Src);
219 
220   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
221   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
222   Module *M = BB->getParent()->getParent();
223   Function *TheFn = Intrinsic::getDeclaration(
224       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
225 
226   CallInst *CI = createCallHelper(TheFn, Ops, this);
227 
228   // Set the alignment of the pointer args.
229   auto *AMCI = cast<AtomicMemCpyInst>(CI);
230   AMCI->setDestAlignment(DstAlign);
231   AMCI->setSourceAlignment(SrcAlign);
232 
233   // Set the TBAA info if present.
234   if (TBAATag)
235     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
236 
237   // Set the TBAA Struct info if present.
238   if (TBAAStructTag)
239     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
240 
241   if (ScopeTag)
242     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
243 
244   if (NoAliasTag)
245     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
246 
247   return CI;
248 }
249 
250 CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign,
251                                        Value *Src, MaybeAlign SrcAlign,
252                                        Value *Size, bool isVolatile,
253                                        MDNode *TBAATag, MDNode *ScopeTag,
254                                        MDNode *NoAliasTag) {
255   Dst = getCastedInt8PtrValue(Dst);
256   Src = getCastedInt8PtrValue(Src);
257 
258   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
259   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
260   Module *M = BB->getParent()->getParent();
261   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
262 
263   CallInst *CI = createCallHelper(TheFn, Ops, this);
264 
265   auto *MMI = cast<MemMoveInst>(CI);
266   if (DstAlign)
267     MMI->setDestAlignment(*DstAlign);
268   if (SrcAlign)
269     MMI->setSourceAlignment(*SrcAlign);
270 
271   // Set the TBAA info if present.
272   if (TBAATag)
273     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
274 
275   if (ScopeTag)
276     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
277 
278   if (NoAliasTag)
279     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
280 
281   return CI;
282 }
283 
284 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
285     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
286     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
287     MDNode *ScopeTag, MDNode *NoAliasTag) {
288   assert(DstAlign >= ElementSize &&
289          "Pointer alignment must be at least element size");
290   assert(SrcAlign >= ElementSize &&
291          "Pointer alignment must be at least element size");
292   Dst = getCastedInt8PtrValue(Dst);
293   Src = getCastedInt8PtrValue(Src);
294 
295   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
296   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
297   Module *M = BB->getParent()->getParent();
298   Function *TheFn = Intrinsic::getDeclaration(
299       M, Intrinsic::memmove_element_unordered_atomic, Tys);
300 
301   CallInst *CI = createCallHelper(TheFn, Ops, this);
302 
303   // Set the alignment of the pointer args.
304   CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
305   CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
306 
307   // Set the TBAA info if present.
308   if (TBAATag)
309     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
310 
311   // Set the TBAA Struct info if present.
312   if (TBAAStructTag)
313     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
314 
315   if (ScopeTag)
316     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
317 
318   if (NoAliasTag)
319     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
320 
321   return CI;
322 }
323 
324 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
325                                     Value *Src) {
326   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
327   Value *Ops[] = {Src};
328   Type *Tys[] = { Src->getType() };
329   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
330   return createCallHelper(Decl, Ops, Builder);
331 }
332 
333 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
334   Module *M = GetInsertBlock()->getParent()->getParent();
335   Value *Ops[] = {Acc, Src};
336   Type *Tys[] = {Acc->getType(), Src->getType()};
337   auto Decl = Intrinsic::getDeclaration(
338       M, Intrinsic::experimental_vector_reduce_v2_fadd, Tys);
339   return createCallHelper(Decl, Ops, this);
340 }
341 
342 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
343   Module *M = GetInsertBlock()->getParent()->getParent();
344   Value *Ops[] = {Acc, Src};
345   Type *Tys[] = {Acc->getType(), Src->getType()};
346   auto Decl = Intrinsic::getDeclaration(
347       M, Intrinsic::experimental_vector_reduce_v2_fmul, Tys);
348   return createCallHelper(Decl, Ops, this);
349 }
350 
351 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
352   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
353                                Src);
354 }
355 
356 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
357   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
358                                Src);
359 }
360 
361 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
362   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
363                                Src);
364 }
365 
366 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
367   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
368                                Src);
369 }
370 
371 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
372   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
373                                Src);
374 }
375 
376 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
377   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
378                      : Intrinsic::experimental_vector_reduce_umax;
379   return getReductionIntrinsic(this, ID, Src);
380 }
381 
382 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
383   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
384                      : Intrinsic::experimental_vector_reduce_umin;
385   return getReductionIntrinsic(this, ID, Src);
386 }
387 
388 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
389   auto Rdx = getReductionIntrinsic(
390       this, Intrinsic::experimental_vector_reduce_fmax, Src);
391   if (NoNaN) {
392     FastMathFlags FMF;
393     FMF.setNoNaNs();
394     Rdx->setFastMathFlags(FMF);
395   }
396   return Rdx;
397 }
398 
399 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
400   auto Rdx = getReductionIntrinsic(
401       this, Intrinsic::experimental_vector_reduce_fmin, Src);
402   if (NoNaN) {
403     FastMathFlags FMF;
404     FMF.setNoNaNs();
405     Rdx->setFastMathFlags(FMF);
406   }
407   return Rdx;
408 }
409 
410 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
411   assert(isa<PointerType>(Ptr->getType()) &&
412          "lifetime.start only applies to pointers.");
413   Ptr = getCastedInt8PtrValue(Ptr);
414   if (!Size)
415     Size = getInt64(-1);
416   else
417     assert(Size->getType() == getInt64Ty() &&
418            "lifetime.start requires the size to be an i64");
419   Value *Ops[] = { Size, Ptr };
420   Module *M = BB->getParent()->getParent();
421   Function *TheFn =
422       Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
423   return createCallHelper(TheFn, Ops, this);
424 }
425 
426 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
427   assert(isa<PointerType>(Ptr->getType()) &&
428          "lifetime.end only applies to pointers.");
429   Ptr = getCastedInt8PtrValue(Ptr);
430   if (!Size)
431     Size = getInt64(-1);
432   else
433     assert(Size->getType() == getInt64Ty() &&
434            "lifetime.end requires the size to be an i64");
435   Value *Ops[] = { Size, Ptr };
436   Module *M = BB->getParent()->getParent();
437   Function *TheFn =
438       Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
439   return createCallHelper(TheFn, Ops, this);
440 }
441 
442 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
443 
444   assert(isa<PointerType>(Ptr->getType()) &&
445          "invariant.start only applies to pointers.");
446   Ptr = getCastedInt8PtrValue(Ptr);
447   if (!Size)
448     Size = getInt64(-1);
449   else
450     assert(Size->getType() == getInt64Ty() &&
451            "invariant.start requires the size to be an i64");
452 
453   Value *Ops[] = {Size, Ptr};
454   // Fill in the single overloaded type: memory object type.
455   Type *ObjectPtr[1] = {Ptr->getType()};
456   Module *M = BB->getParent()->getParent();
457   Function *TheFn =
458       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
459   return createCallHelper(TheFn, Ops, this);
460 }
461 
462 CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
463   assert(Cond->getType() == getInt1Ty() &&
464          "an assumption condition must be of type i1");
465 
466   Value *Ops[] = { Cond };
467   Module *M = BB->getParent()->getParent();
468   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
469   return createCallHelper(FnAssume, Ops, this);
470 }
471 
472 /// Create a call to a Masked Load intrinsic.
473 /// \p Ptr       - base pointer for the load
474 /// \p Alignment - alignment of the source location
475 /// \p Mask      - vector of booleans which indicates what vector lanes should
476 ///                be accessed in memory
477 /// \p PassThru  - pass-through value that is used to fill the masked-off lanes
478 ///                of the result
479 /// \p Name      - name of the result variable
480 CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, Align Alignment,
481                                           Value *Mask, Value *PassThru,
482                                           const Twine &Name) {
483   auto *PtrTy = cast<PointerType>(Ptr->getType());
484   Type *DataTy = PtrTy->getElementType();
485   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
486   assert(Mask && "Mask should not be all-ones (null)");
487   if (!PassThru)
488     PassThru = UndefValue::get(DataTy);
489   Type *OverloadedTypes[] = { DataTy, PtrTy };
490   Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
491   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
492                                OverloadedTypes, Name);
493 }
494 
495 /// Create a call to a Masked Store intrinsic.
496 /// \p Val       - data to be stored,
497 /// \p Ptr       - base pointer for the store
498 /// \p Alignment - alignment of the destination location
499 /// \p Mask      - vector of booleans which indicates what vector lanes should
500 ///                be accessed in memory
501 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
502                                            Align Alignment, Value *Mask) {
503   auto *PtrTy = cast<PointerType>(Ptr->getType());
504   Type *DataTy = PtrTy->getElementType();
505   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
506   assert(Mask && "Mask should not be all-ones (null)");
507   Type *OverloadedTypes[] = { DataTy, PtrTy };
508   Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
509   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
510 }
511 
512 /// Create a call to a Masked intrinsic, with given intrinsic Id,
513 /// an array of operands - Ops, and an array of overloaded types -
514 /// OverloadedTypes.
515 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
516                                                ArrayRef<Value *> Ops,
517                                                ArrayRef<Type *> OverloadedTypes,
518                                                const Twine &Name) {
519   Module *M = BB->getParent()->getParent();
520   Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
521   return createCallHelper(TheFn, Ops, this, Name);
522 }
523 
524 /// Create a call to a Masked Gather intrinsic.
525 /// \p Ptrs     - vector of pointers for loading
526 /// \p Align    - alignment for one element
527 /// \p Mask     - vector of booleans which indicates what vector lanes should
528 ///               be accessed in memory
529 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
530 ///               of the result
531 /// \p Name     - name of the result variable
532 CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, Align Alignment,
533                                             Value *Mask, Value *PassThru,
534                                             const Twine &Name) {
535   auto PtrsTy = cast<VectorType>(Ptrs->getType());
536   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
537   unsigned NumElts = PtrsTy->getVectorNumElements();
538   Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
539 
540   if (!Mask)
541     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
542                                      NumElts));
543 
544   if (!PassThru)
545     PassThru = UndefValue::get(DataTy);
546 
547   Type *OverloadedTypes[] = {DataTy, PtrsTy};
548   Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
549 
550   // We specify only one type when we create this intrinsic. Types of other
551   // arguments are derived from this type.
552   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
553                                Name);
554 }
555 
556 /// Create a call to a Masked Scatter intrinsic.
557 /// \p Data  - data to be stored,
558 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
559 ///            stored
560 /// \p Align - alignment for one element
561 /// \p Mask  - vector of booleans which indicates what vector lanes should
562 ///            be accessed in memory
563 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
564                                              Align Alignment, Value *Mask) {
565   auto PtrsTy = cast<VectorType>(Ptrs->getType());
566   auto DataTy = cast<VectorType>(Data->getType());
567   unsigned NumElts = PtrsTy->getVectorNumElements();
568 
569 #ifndef NDEBUG
570   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
571   assert(NumElts == DataTy->getVectorNumElements() &&
572          PtrTy->getElementType() == DataTy->getElementType() &&
573          "Incompatible pointer and data types");
574 #endif
575 
576   if (!Mask)
577     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
578                                      NumElts));
579 
580   Type *OverloadedTypes[] = {DataTy, PtrsTy};
581   Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
582 
583   // We specify only one type when we create this intrinsic. Types of other
584   // arguments are derived from this type.
585   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
586 }
587 
588 template <typename T0, typename T1, typename T2, typename T3>
589 static std::vector<Value *>
590 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
591                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
592                   ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
593                   ArrayRef<T3> GCArgs) {
594   std::vector<Value *> Args;
595   Args.push_back(B.getInt64(ID));
596   Args.push_back(B.getInt32(NumPatchBytes));
597   Args.push_back(ActualCallee);
598   Args.push_back(B.getInt32(CallArgs.size()));
599   Args.push_back(B.getInt32(Flags));
600   Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
601   Args.push_back(B.getInt32(TransitionArgs.size()));
602   Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
603   Args.push_back(B.getInt32(DeoptArgs.size()));
604   Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
605   Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
606 
607   return Args;
608 }
609 
610 template <typename T0, typename T1, typename T2, typename T3>
611 static CallInst *CreateGCStatepointCallCommon(
612     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
613     Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
614     ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
615     const Twine &Name) {
616   // Extract out the type of the callee.
617   auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
618   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
619          "actual callee must be a callable value");
620 
621   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
622   // Fill in the one generic type'd argument (the function is also vararg)
623   Type *ArgTypes[] = { FuncPtrType };
624   Function *FnStatepoint =
625     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
626                               ArgTypes);
627 
628   std::vector<Value *> Args =
629       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
630                         CallArgs, TransitionArgs, DeoptArgs, GCArgs);
631   return createCallHelper(FnStatepoint, Args, Builder, Name);
632 }
633 
634 CallInst *IRBuilderBase::CreateGCStatepointCall(
635     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
636     ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
637     ArrayRef<Value *> GCArgs, const Twine &Name) {
638   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
639       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
640       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
641 }
642 
643 CallInst *IRBuilderBase::CreateGCStatepointCall(
644     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
645     ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
646     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
647   return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
648       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
649       DeoptArgs, GCArgs, Name);
650 }
651 
652 CallInst *IRBuilderBase::CreateGCStatepointCall(
653     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
654     ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
655     ArrayRef<Value *> GCArgs, const Twine &Name) {
656   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
657       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
658       CallArgs, None, DeoptArgs, GCArgs, Name);
659 }
660 
661 template <typename T0, typename T1, typename T2, typename T3>
662 static InvokeInst *CreateGCStatepointInvokeCommon(
663     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
664     Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
665     uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
666     ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
667   // Extract out the type of the callee.
668   auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
669   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
670          "actual callee must be a callable value");
671 
672   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
673   // Fill in the one generic type'd argument (the function is also vararg)
674   Function *FnStatepoint = Intrinsic::getDeclaration(
675       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
676 
677   std::vector<Value *> Args =
678       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
679                         InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
680   return Builder->CreateInvoke(FnStatepoint, NormalDest, UnwindDest, Args,
681                                Name);
682 }
683 
684 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
685     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
686     BasicBlock *NormalDest, BasicBlock *UnwindDest,
687     ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
688     ArrayRef<Value *> GCArgs, const Twine &Name) {
689   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
690       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
691       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
692       DeoptArgs, GCArgs, Name);
693 }
694 
695 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
696     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
697     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
698     ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
699     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
700   return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
701       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
702       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
703 }
704 
705 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
706     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
707     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
708     ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
709   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
710       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
711       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
712       Name);
713 }
714 
715 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
716                                        Type *ResultType,
717                                        const Twine &Name) {
718  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
719  Module *M = BB->getParent()->getParent();
720  Type *Types[] = {ResultType};
721  Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
722 
723  Value *Args[] = {Statepoint};
724  return createCallHelper(FnGCResult, Args, this, Name);
725 }
726 
727 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
728                                          int BaseOffset,
729                                          int DerivedOffset,
730                                          Type *ResultType,
731                                          const Twine &Name) {
732  Module *M = BB->getParent()->getParent();
733  Type *Types[] = {ResultType};
734  Function *FnGCRelocate =
735      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
736 
737  Value *Args[] = {Statepoint,
738                   getInt32(BaseOffset),
739                   getInt32(DerivedOffset)};
740  return createCallHelper(FnGCRelocate, Args, this, Name);
741 }
742 
743 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
744                                               Instruction *FMFSource,
745                                               const Twine &Name) {
746   Module *M = BB->getModule();
747   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
748   return createCallHelper(Fn, {V}, this, Name, FMFSource);
749 }
750 
751 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
752                                                Value *RHS,
753                                                Instruction *FMFSource,
754                                                const Twine &Name) {
755   Module *M = BB->getModule();
756   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
757   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
758 }
759 
760 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
761                                          ArrayRef<Type *> Types,
762                                          ArrayRef<Value *> Args,
763                                          Instruction *FMFSource,
764                                          const Twine &Name) {
765   Module *M = BB->getModule();
766   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
767   return createCallHelper(Fn, Args, this, Name, FMFSource);
768 }
769 
770 IRBuilderDefaultInserter::~IRBuilderDefaultInserter() {}
771 IRBuilderCallbackInserter::~IRBuilderCallbackInserter() {}
772 IRBuilderFolder::~IRBuilderFolder() {}
773 void ConstantFolder::anchor() {}
774 void NoFolder::anchor() {}
775