1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the IRBuilder class, which is used as a convenient way
10 // to create LLVM instructions with a consistent and simplified interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/None.h"
17 #include "llvm/IR/Constant.h"
18 #include "llvm/IR/Constants.h"
19 #include "llvm/IR/DerivedTypes.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/GlobalVariable.h"
23 #include "llvm/IR/IntrinsicInst.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/NoFolder.h"
27 #include "llvm/IR/Operator.h"
28 #include "llvm/IR/Statepoint.h"
29 #include "llvm/IR/Type.h"
30 #include "llvm/IR/Value.h"
31 #include "llvm/Support/Casting.h"
32 #include "llvm/Support/MathExtras.h"
33 #include <cassert>
34 #include <cstdint>
35 #include <vector>
36 
37 using namespace llvm;
38 
39 /// CreateGlobalString - Make a new global variable with an initializer that
40 /// has array of i8 type filled in with the nul terminated string value
41 /// specified.  If Name is specified, it is the name of the global variable
42 /// created.
43 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
44                                                   const Twine &Name,
45                                                   unsigned AddressSpace,
46                                                   Module *M) {
47   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
48   if (!M)
49     M = BB->getParent()->getParent();
50   auto *GV = new GlobalVariable(
51       *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
52       StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
53   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
54   GV->setAlignment(Align(1));
55   return GV;
56 }
57 
58 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
59   assert(BB && BB->getParent() && "No current function!");
60   return BB->getParent()->getReturnType();
61 }
62 
63 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
64   auto *PT = cast<PointerType>(Ptr->getType());
65   if (PT->getElementType()->isIntegerTy(8))
66     return Ptr;
67 
68   // Otherwise, we need to insert a bitcast.
69   return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace()));
70 }
71 
72 static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
73                                   IRBuilderBase *Builder,
74                                   const Twine &Name = "",
75                                   Instruction *FMFSource = nullptr,
76                                   ArrayRef<OperandBundleDef> OpBundles = {}) {
77   CallInst *CI = Builder->CreateCall(Callee, Ops, OpBundles, Name);
78   if (FMFSource)
79     CI->copyFastMathFlags(FMFSource);
80   return CI;
81 }
82 
83 Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) {
84   Module *M = GetInsertBlock()->getParent()->getParent();
85   assert(isa<ConstantInt>(Scaling) && "Expected constant integer");
86   Function *TheFn =
87       Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()});
88   CallInst *CI = createCallHelper(TheFn, {}, this, Name);
89   return cast<ConstantInt>(Scaling)->getSExtValue() == 1
90              ? CI
91              : CreateMul(CI, Scaling);
92 }
93 
94 Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
95   if (isa<ScalableVectorType>(DstType))
96     return CreateIntrinsic(Intrinsic::experimental_stepvector, {DstType}, {},
97                            nullptr, Name);
98 
99   Type *STy = DstType->getScalarType();
100   unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements();
101 
102   // Create a vector of consecutive numbers from zero to VF.
103   SmallVector<Constant *, 8> Indices;
104   for (unsigned i = 0; i < NumEls; ++i)
105     Indices.push_back(ConstantInt::get(STy, i));
106 
107   // Add the consecutive indices to the vector value.
108   return ConstantVector::get(Indices);
109 }
110 
111 CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
112                                       MaybeAlign Align, bool isVolatile,
113                                       MDNode *TBAATag, MDNode *ScopeTag,
114                                       MDNode *NoAliasTag) {
115   Ptr = getCastedInt8PtrValue(Ptr);
116   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
117   Type *Tys[] = { Ptr->getType(), Size->getType() };
118   Module *M = BB->getParent()->getParent();
119   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
120 
121   CallInst *CI = createCallHelper(TheFn, Ops, this);
122 
123   if (Align)
124     cast<MemSetInst>(CI)->setDestAlignment(Align->value());
125 
126   // Set the TBAA info if present.
127   if (TBAATag)
128     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
129 
130   if (ScopeTag)
131     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
132 
133   if (NoAliasTag)
134     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
135 
136   return CI;
137 }
138 
139 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
140     Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
141     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
142 
143   Ptr = getCastedInt8PtrValue(Ptr);
144   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
145   Type *Tys[] = {Ptr->getType(), Size->getType()};
146   Module *M = BB->getParent()->getParent();
147   Function *TheFn = Intrinsic::getDeclaration(
148       M, Intrinsic::memset_element_unordered_atomic, Tys);
149 
150   CallInst *CI = createCallHelper(TheFn, Ops, this);
151 
152   cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
153 
154   // Set the TBAA info if present.
155   if (TBAATag)
156     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
157 
158   if (ScopeTag)
159     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
160 
161   if (NoAliasTag)
162     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
163 
164   return CI;
165 }
166 
167 CallInst *IRBuilderBase::CreateMemTransferInst(
168     Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
169     MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag,
170     MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
171   Dst = getCastedInt8PtrValue(Dst);
172   Src = getCastedInt8PtrValue(Src);
173 
174   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
175   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
176   Module *M = BB->getParent()->getParent();
177   Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys);
178 
179   CallInst *CI = createCallHelper(TheFn, Ops, this);
180 
181   auto* MCI = cast<MemTransferInst>(CI);
182   if (DstAlign)
183     MCI->setDestAlignment(*DstAlign);
184   if (SrcAlign)
185     MCI->setSourceAlignment(*SrcAlign);
186 
187   // Set the TBAA info if present.
188   if (TBAATag)
189     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
190 
191   // Set the TBAA Struct info if present.
192   if (TBAAStructTag)
193     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
194 
195   if (ScopeTag)
196     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
197 
198   if (NoAliasTag)
199     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
200 
201   return CI;
202 }
203 
204 CallInst *IRBuilderBase::CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign,
205                                             Value *Src, MaybeAlign SrcAlign,
206                                             Value *Size) {
207   Dst = getCastedInt8PtrValue(Dst);
208   Src = getCastedInt8PtrValue(Src);
209   Value *IsVolatile = getInt1(false);
210 
211   Value *Ops[] = {Dst, Src, Size, IsVolatile};
212   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
213   Function *F = BB->getParent();
214   Module *M = F->getParent();
215   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys);
216 
217   CallInst *CI = createCallHelper(TheFn, Ops, this);
218 
219   auto *MCI = cast<MemCpyInlineInst>(CI);
220   if (DstAlign)
221     MCI->setDestAlignment(*DstAlign);
222   if (SrcAlign)
223     MCI->setSourceAlignment(*SrcAlign);
224 
225   return CI;
226 }
227 
228 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
229     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
230     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
231     MDNode *ScopeTag, MDNode *NoAliasTag) {
232   assert(DstAlign >= ElementSize &&
233          "Pointer alignment must be at least element size");
234   assert(SrcAlign >= ElementSize &&
235          "Pointer alignment must be at least element size");
236   Dst = getCastedInt8PtrValue(Dst);
237   Src = getCastedInt8PtrValue(Src);
238 
239   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
240   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
241   Module *M = BB->getParent()->getParent();
242   Function *TheFn = Intrinsic::getDeclaration(
243       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
244 
245   CallInst *CI = createCallHelper(TheFn, Ops, this);
246 
247   // Set the alignment of the pointer args.
248   auto *AMCI = cast<AtomicMemCpyInst>(CI);
249   AMCI->setDestAlignment(DstAlign);
250   AMCI->setSourceAlignment(SrcAlign);
251 
252   // Set the TBAA info if present.
253   if (TBAATag)
254     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
255 
256   // Set the TBAA Struct info if present.
257   if (TBAAStructTag)
258     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
259 
260   if (ScopeTag)
261     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
262 
263   if (NoAliasTag)
264     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
265 
266   return CI;
267 }
268 
269 CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign,
270                                        Value *Src, MaybeAlign SrcAlign,
271                                        Value *Size, bool isVolatile,
272                                        MDNode *TBAATag, MDNode *ScopeTag,
273                                        MDNode *NoAliasTag) {
274   Dst = getCastedInt8PtrValue(Dst);
275   Src = getCastedInt8PtrValue(Src);
276 
277   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
278   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
279   Module *M = BB->getParent()->getParent();
280   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
281 
282   CallInst *CI = createCallHelper(TheFn, Ops, this);
283 
284   auto *MMI = cast<MemMoveInst>(CI);
285   if (DstAlign)
286     MMI->setDestAlignment(*DstAlign);
287   if (SrcAlign)
288     MMI->setSourceAlignment(*SrcAlign);
289 
290   // Set the TBAA info if present.
291   if (TBAATag)
292     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
293 
294   if (ScopeTag)
295     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
296 
297   if (NoAliasTag)
298     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
299 
300   return CI;
301 }
302 
303 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
304     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
305     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
306     MDNode *ScopeTag, MDNode *NoAliasTag) {
307   assert(DstAlign >= ElementSize &&
308          "Pointer alignment must be at least element size");
309   assert(SrcAlign >= ElementSize &&
310          "Pointer alignment must be at least element size");
311   Dst = getCastedInt8PtrValue(Dst);
312   Src = getCastedInt8PtrValue(Src);
313 
314   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
315   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
316   Module *M = BB->getParent()->getParent();
317   Function *TheFn = Intrinsic::getDeclaration(
318       M, Intrinsic::memmove_element_unordered_atomic, Tys);
319 
320   CallInst *CI = createCallHelper(TheFn, Ops, this);
321 
322   // Set the alignment of the pointer args.
323   CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
324   CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
325 
326   // Set the TBAA info if present.
327   if (TBAATag)
328     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
329 
330   // Set the TBAA Struct info if present.
331   if (TBAAStructTag)
332     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
333 
334   if (ScopeTag)
335     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
336 
337   if (NoAliasTag)
338     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
339 
340   return CI;
341 }
342 
343 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
344                                     Value *Src) {
345   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
346   Value *Ops[] = {Src};
347   Type *Tys[] = { Src->getType() };
348   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
349   return createCallHelper(Decl, Ops, Builder);
350 }
351 
352 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
353   Module *M = GetInsertBlock()->getParent()->getParent();
354   Value *Ops[] = {Acc, Src};
355   auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd,
356                                         {Src->getType()});
357   return createCallHelper(Decl, Ops, this);
358 }
359 
360 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
361   Module *M = GetInsertBlock()->getParent()->getParent();
362   Value *Ops[] = {Acc, Src};
363   auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul,
364                                         {Src->getType()});
365   return createCallHelper(Decl, Ops, this);
366 }
367 
368 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
369   return getReductionIntrinsic(this, Intrinsic::vector_reduce_add, Src);
370 }
371 
372 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
373   return getReductionIntrinsic(this, Intrinsic::vector_reduce_mul, Src);
374 }
375 
376 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
377   return getReductionIntrinsic(this, Intrinsic::vector_reduce_and, Src);
378 }
379 
380 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
381   return getReductionIntrinsic(this, Intrinsic::vector_reduce_or, Src);
382 }
383 
384 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
385   return getReductionIntrinsic(this, Intrinsic::vector_reduce_xor, Src);
386 }
387 
388 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
389   auto ID =
390       IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
391   return getReductionIntrinsic(this, ID, Src);
392 }
393 
394 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
395   auto ID =
396       IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
397   return getReductionIntrinsic(this, ID, Src);
398 }
399 
400 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
401   return getReductionIntrinsic(this, Intrinsic::vector_reduce_fmax, Src);
402 }
403 
404 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src) {
405   return getReductionIntrinsic(this, Intrinsic::vector_reduce_fmin, Src);
406 }
407 
408 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
409   assert(isa<PointerType>(Ptr->getType()) &&
410          "lifetime.start only applies to pointers.");
411   Ptr = getCastedInt8PtrValue(Ptr);
412   if (!Size)
413     Size = getInt64(-1);
414   else
415     assert(Size->getType() == getInt64Ty() &&
416            "lifetime.start requires the size to be an i64");
417   Value *Ops[] = { Size, Ptr };
418   Module *M = BB->getParent()->getParent();
419   Function *TheFn =
420       Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
421   return createCallHelper(TheFn, Ops, this);
422 }
423 
424 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
425   assert(isa<PointerType>(Ptr->getType()) &&
426          "lifetime.end only applies to pointers.");
427   Ptr = getCastedInt8PtrValue(Ptr);
428   if (!Size)
429     Size = getInt64(-1);
430   else
431     assert(Size->getType() == getInt64Ty() &&
432            "lifetime.end requires the size to be an i64");
433   Value *Ops[] = { Size, Ptr };
434   Module *M = BB->getParent()->getParent();
435   Function *TheFn =
436       Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
437   return createCallHelper(TheFn, Ops, this);
438 }
439 
440 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
441 
442   assert(isa<PointerType>(Ptr->getType()) &&
443          "invariant.start only applies to pointers.");
444   Ptr = getCastedInt8PtrValue(Ptr);
445   if (!Size)
446     Size = getInt64(-1);
447   else
448     assert(Size->getType() == getInt64Ty() &&
449            "invariant.start requires the size to be an i64");
450 
451   Value *Ops[] = {Size, Ptr};
452   // Fill in the single overloaded type: memory object type.
453   Type *ObjectPtr[1] = {Ptr->getType()};
454   Module *M = BB->getParent()->getParent();
455   Function *TheFn =
456       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
457   return createCallHelper(TheFn, Ops, this);
458 }
459 
460 CallInst *
461 IRBuilderBase::CreateAssumption(Value *Cond,
462                                 ArrayRef<OperandBundleDef> OpBundles) {
463   assert(Cond->getType() == getInt1Ty() &&
464          "an assumption condition must be of type i1");
465 
466   Value *Ops[] = { Cond };
467   Module *M = BB->getParent()->getParent();
468   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
469   return createCallHelper(FnAssume, Ops, this, "", nullptr, OpBundles);
470 }
471 
472 Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
473   Module *M = BB->getModule();
474   auto *FnIntrinsic = Intrinsic::getDeclaration(
475       M, Intrinsic::experimental_noalias_scope_decl, {});
476   return createCallHelper(FnIntrinsic, {Scope}, this);
477 }
478 
479 /// Create a call to a Masked Load intrinsic.
480 /// \p Ptr       - base pointer for the load
481 /// \p Alignment - alignment of the source location
482 /// \p Mask      - vector of booleans which indicates what vector lanes should
483 ///                be accessed in memory
484 /// \p PassThru  - pass-through value that is used to fill the masked-off lanes
485 ///                of the result
486 /// \p Name      - name of the result variable
487 CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, Align Alignment,
488                                           Value *Mask, Value *PassThru,
489                                           const Twine &Name) {
490   auto *PtrTy = cast<PointerType>(Ptr->getType());
491   Type *DataTy = PtrTy->getElementType();
492   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
493   assert(Mask && "Mask should not be all-ones (null)");
494   if (!PassThru)
495     PassThru = UndefValue::get(DataTy);
496   Type *OverloadedTypes[] = { DataTy, PtrTy };
497   Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
498   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
499                                OverloadedTypes, Name);
500 }
501 
502 /// Create a call to a Masked Store intrinsic.
503 /// \p Val       - data to be stored,
504 /// \p Ptr       - base pointer for the store
505 /// \p Alignment - alignment of the destination location
506 /// \p Mask      - vector of booleans which indicates what vector lanes should
507 ///                be accessed in memory
508 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
509                                            Align Alignment, Value *Mask) {
510   auto *PtrTy = cast<PointerType>(Ptr->getType());
511   Type *DataTy = PtrTy->getElementType();
512   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
513   assert(Mask && "Mask should not be all-ones (null)");
514   Type *OverloadedTypes[] = { DataTy, PtrTy };
515   Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
516   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
517 }
518 
519 /// Create a call to a Masked intrinsic, with given intrinsic Id,
520 /// an array of operands - Ops, and an array of overloaded types -
521 /// OverloadedTypes.
522 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
523                                                ArrayRef<Value *> Ops,
524                                                ArrayRef<Type *> OverloadedTypes,
525                                                const Twine &Name) {
526   Module *M = BB->getParent()->getParent();
527   Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
528   return createCallHelper(TheFn, Ops, this, Name);
529 }
530 
531 /// Create a call to a Masked Gather intrinsic.
532 /// \p Ptrs     - vector of pointers for loading
533 /// \p Align    - alignment for one element
534 /// \p Mask     - vector of booleans which indicates what vector lanes should
535 ///               be accessed in memory
536 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
537 ///               of the result
538 /// \p Name     - name of the result variable
539 CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, Align Alignment,
540                                             Value *Mask, Value *PassThru,
541                                             const Twine &Name) {
542   auto *PtrsTy = cast<VectorType>(Ptrs->getType());
543   auto *PtrTy = cast<PointerType>(PtrsTy->getElementType());
544   ElementCount NumElts = PtrsTy->getElementCount();
545   auto *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
546 
547   if (!Mask)
548     Mask = Constant::getAllOnesValue(
549         VectorType::get(Type::getInt1Ty(Context), NumElts));
550 
551   if (!PassThru)
552     PassThru = UndefValue::get(DataTy);
553 
554   Type *OverloadedTypes[] = {DataTy, PtrsTy};
555   Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
556 
557   // We specify only one type when we create this intrinsic. Types of other
558   // arguments are derived from this type.
559   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
560                                Name);
561 }
562 
563 /// Create a call to a Masked Scatter intrinsic.
564 /// \p Data  - data to be stored,
565 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
566 ///            stored
567 /// \p Align - alignment for one element
568 /// \p Mask  - vector of booleans which indicates what vector lanes should
569 ///            be accessed in memory
570 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
571                                              Align Alignment, Value *Mask) {
572   auto *PtrsTy = cast<VectorType>(Ptrs->getType());
573   auto *DataTy = cast<VectorType>(Data->getType());
574   ElementCount NumElts = PtrsTy->getElementCount();
575 
576 #ifndef NDEBUG
577   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
578   assert(NumElts == DataTy->getElementCount() &&
579          PtrTy->getElementType() == DataTy->getElementType() &&
580          "Incompatible pointer and data types");
581 #endif
582 
583   if (!Mask)
584     Mask = Constant::getAllOnesValue(
585         VectorType::get(Type::getInt1Ty(Context), NumElts));
586 
587   Type *OverloadedTypes[] = {DataTy, PtrsTy};
588   Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
589 
590   // We specify only one type when we create this intrinsic. Types of other
591   // arguments are derived from this type.
592   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
593 }
594 
595 template <typename T0>
596 static std::vector<Value *>
597 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
598                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
599   std::vector<Value *> Args;
600   Args.push_back(B.getInt64(ID));
601   Args.push_back(B.getInt32(NumPatchBytes));
602   Args.push_back(ActualCallee);
603   Args.push_back(B.getInt32(CallArgs.size()));
604   Args.push_back(B.getInt32(Flags));
605   llvm::append_range(Args, CallArgs);
606   // GC Transition and Deopt args are now always handled via operand bundle.
607   // They will be removed from the signature of gc.statepoint shortly.
608   Args.push_back(B.getInt32(0));
609   Args.push_back(B.getInt32(0));
610   // GC args are now encoded in the gc-live operand bundle
611   return Args;
612 }
613 
614 template<typename T1, typename T2, typename T3>
615 static std::vector<OperandBundleDef>
616 getStatepointBundles(Optional<ArrayRef<T1>> TransitionArgs,
617                      Optional<ArrayRef<T2>> DeoptArgs,
618                      ArrayRef<T3> GCArgs) {
619   std::vector<OperandBundleDef> Rval;
620   if (DeoptArgs) {
621     SmallVector<Value*, 16> DeoptValues;
622     llvm::append_range(DeoptValues, *DeoptArgs);
623     Rval.emplace_back("deopt", DeoptValues);
624   }
625   if (TransitionArgs) {
626     SmallVector<Value*, 16> TransitionValues;
627     llvm::append_range(TransitionValues, *TransitionArgs);
628     Rval.emplace_back("gc-transition", TransitionValues);
629   }
630   if (GCArgs.size()) {
631     SmallVector<Value*, 16> LiveValues;
632     llvm::append_range(LiveValues, GCArgs);
633     Rval.emplace_back("gc-live", LiveValues);
634   }
635   return Rval;
636 }
637 
638 template <typename T0, typename T1, typename T2, typename T3>
639 static CallInst *CreateGCStatepointCallCommon(
640     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
641     Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
642     Optional<ArrayRef<T1>> TransitionArgs,
643     Optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
644     const Twine &Name) {
645   // Extract out the type of the callee.
646   auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
647   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
648          "actual callee must be a callable value");
649 
650   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
651   // Fill in the one generic type'd argument (the function is also vararg)
652   Type *ArgTypes[] = { FuncPtrType };
653   Function *FnStatepoint =
654     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
655                               ArgTypes);
656 
657   std::vector<Value *> Args =
658       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
659                         CallArgs);
660 
661   return Builder->CreateCall(FnStatepoint, Args,
662                              getStatepointBundles(TransitionArgs, DeoptArgs,
663                                                   GCArgs),
664                              Name);
665 }
666 
667 CallInst *IRBuilderBase::CreateGCStatepointCall(
668     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
669     ArrayRef<Value *> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs,
670     ArrayRef<Value *> GCArgs, const Twine &Name) {
671   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
672       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
673       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
674 }
675 
676 CallInst *IRBuilderBase::CreateGCStatepointCall(
677     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
678     ArrayRef<Value *> CallArgs, Optional<ArrayRef<Use>> TransitionArgs,
679     Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
680     const Twine &Name) {
681   return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
682       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
683       DeoptArgs, GCArgs, Name);
684 }
685 
686 CallInst *IRBuilderBase::CreateGCStatepointCall(
687     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
688     ArrayRef<Use> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs,
689     ArrayRef<Value *> GCArgs, const Twine &Name) {
690   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
691       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
692       CallArgs, None, DeoptArgs, GCArgs, Name);
693 }
694 
695 template <typename T0, typename T1, typename T2, typename T3>
696 static InvokeInst *CreateGCStatepointInvokeCommon(
697     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
698     Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
699     uint32_t Flags, ArrayRef<T0> InvokeArgs,
700     Optional<ArrayRef<T1>> TransitionArgs, Optional<ArrayRef<T2>> DeoptArgs,
701     ArrayRef<T3> GCArgs, const Twine &Name) {
702   // Extract out the type of the callee.
703   auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
704   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
705          "actual callee must be a callable value");
706 
707   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
708   // Fill in the one generic type'd argument (the function is also vararg)
709   Function *FnStatepoint = Intrinsic::getDeclaration(
710       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
711 
712   std::vector<Value *> Args =
713       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
714                         InvokeArgs);
715 
716   return Builder->CreateInvoke(FnStatepoint, NormalDest, UnwindDest, Args,
717                                getStatepointBundles(TransitionArgs, DeoptArgs,
718                                                     GCArgs),
719                                Name);
720 }
721 
722 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
723     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
724     BasicBlock *NormalDest, BasicBlock *UnwindDest,
725     ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Value *>> DeoptArgs,
726     ArrayRef<Value *> GCArgs, const Twine &Name) {
727   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
728       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
729       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
730       DeoptArgs, GCArgs, Name);
731 }
732 
733 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
734     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
735     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
736     ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
737     Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
738   return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
739       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
740       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
741 }
742 
743 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
744     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
745     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
746     Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
747   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
748       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
749       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
750       Name);
751 }
752 
753 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
754                                        Type *ResultType,
755                                        const Twine &Name) {
756  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
757  Module *M = BB->getParent()->getParent();
758  Type *Types[] = {ResultType};
759  Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
760 
761  Value *Args[] = {Statepoint};
762  return createCallHelper(FnGCResult, Args, this, Name);
763 }
764 
765 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
766                                          int BaseOffset,
767                                          int DerivedOffset,
768                                          Type *ResultType,
769                                          const Twine &Name) {
770  Module *M = BB->getParent()->getParent();
771  Type *Types[] = {ResultType};
772  Function *FnGCRelocate =
773      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
774 
775  Value *Args[] = {Statepoint,
776                   getInt32(BaseOffset),
777                   getInt32(DerivedOffset)};
778  return createCallHelper(FnGCRelocate, Args, this, Name);
779 }
780 
781 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
782                                               Instruction *FMFSource,
783                                               const Twine &Name) {
784   Module *M = BB->getModule();
785   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
786   return createCallHelper(Fn, {V}, this, Name, FMFSource);
787 }
788 
789 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
790                                                Value *RHS,
791                                                Instruction *FMFSource,
792                                                const Twine &Name) {
793   Module *M = BB->getModule();
794   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
795   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
796 }
797 
798 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
799                                          ArrayRef<Type *> Types,
800                                          ArrayRef<Value *> Args,
801                                          Instruction *FMFSource,
802                                          const Twine &Name) {
803   Module *M = BB->getModule();
804   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
805   return createCallHelper(Fn, Args, this, Name, FMFSource);
806 }
807 
808 CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
809     Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
810     const Twine &Name, MDNode *FPMathTag,
811     Optional<RoundingMode> Rounding,
812     Optional<fp::ExceptionBehavior> Except) {
813   Value *RoundingV = getConstrainedFPRounding(Rounding);
814   Value *ExceptV = getConstrainedFPExcept(Except);
815 
816   FastMathFlags UseFMF = FMF;
817   if (FMFSource)
818     UseFMF = FMFSource->getFastMathFlags();
819 
820   CallInst *C = CreateIntrinsic(ID, {L->getType()},
821                                 {L, R, RoundingV, ExceptV}, nullptr, Name);
822   setConstrainedFPCallAttr(C);
823   setFPAttrs(C, FPMathTag, UseFMF);
824   return C;
825 }
826 
827 Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
828                                    const Twine &Name, MDNode *FPMathTag) {
829   if (Instruction::isBinaryOp(Opc)) {
830     assert(Ops.size() == 2 && "Invalid number of operands!");
831     return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),
832                        Ops[0], Ops[1], Name, FPMathTag);
833   }
834   if (Instruction::isUnaryOp(Opc)) {
835     assert(Ops.size() == 1 && "Invalid number of operands!");
836     return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),
837                       Ops[0], Name, FPMathTag);
838   }
839   llvm_unreachable("Unexpected opcode!");
840 }
841 
842 CallInst *IRBuilderBase::CreateConstrainedFPCast(
843     Intrinsic::ID ID, Value *V, Type *DestTy,
844     Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag,
845     Optional<RoundingMode> Rounding,
846     Optional<fp::ExceptionBehavior> Except) {
847   Value *ExceptV = getConstrainedFPExcept(Except);
848 
849   FastMathFlags UseFMF = FMF;
850   if (FMFSource)
851     UseFMF = FMFSource->getFastMathFlags();
852 
853   CallInst *C;
854   bool HasRoundingMD = false;
855   switch (ID) {
856   default:
857     break;
858 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
859   case Intrinsic::INTRINSIC:                                \
860     HasRoundingMD = ROUND_MODE;                             \
861     break;
862 #include "llvm/IR/ConstrainedOps.def"
863   }
864   if (HasRoundingMD) {
865     Value *RoundingV = getConstrainedFPRounding(Rounding);
866     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
867                         nullptr, Name);
868   } else
869     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
870                         Name);
871 
872   setConstrainedFPCallAttr(C);
873 
874   if (isa<FPMathOperator>(C))
875     setFPAttrs(C, FPMathTag, UseFMF);
876   return C;
877 }
878 
879 Value *IRBuilderBase::CreateFCmpHelper(
880     CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name,
881     MDNode *FPMathTag, bool IsSignaling) {
882   if (IsFPConstrained) {
883     auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
884                           : Intrinsic::experimental_constrained_fcmp;
885     return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);
886   }
887 
888   if (auto *LC = dyn_cast<Constant>(LHS))
889     if (auto *RC = dyn_cast<Constant>(RHS))
890       return Insert(Folder.CreateFCmp(P, LC, RC), Name);
891   return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name);
892 }
893 
894 CallInst *IRBuilderBase::CreateConstrainedFPCmp(
895     Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
896     const Twine &Name, Optional<fp::ExceptionBehavior> Except) {
897   Value *PredicateV = getConstrainedFPPredicate(P);
898   Value *ExceptV = getConstrainedFPExcept(Except);
899 
900   CallInst *C = CreateIntrinsic(ID, {L->getType()},
901                                 {L, R, PredicateV, ExceptV}, nullptr, Name);
902   setConstrainedFPCallAttr(C);
903   return C;
904 }
905 
906 CallInst *IRBuilderBase::CreateConstrainedFPCall(
907     Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
908     Optional<RoundingMode> Rounding,
909     Optional<fp::ExceptionBehavior> Except) {
910   llvm::SmallVector<Value *, 6> UseArgs;
911 
912   append_range(UseArgs, Args);
913   bool HasRoundingMD = false;
914   switch (Callee->getIntrinsicID()) {
915   default:
916     break;
917 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
918   case Intrinsic::INTRINSIC:                                \
919     HasRoundingMD = ROUND_MODE;                             \
920     break;
921 #include "llvm/IR/ConstrainedOps.def"
922   }
923   if (HasRoundingMD)
924     UseArgs.push_back(getConstrainedFPRounding(Rounding));
925   UseArgs.push_back(getConstrainedFPExcept(Except));
926 
927   CallInst *C = CreateCall(Callee, UseArgs, Name);
928   setConstrainedFPCallAttr(C);
929   return C;
930 }
931 
932 Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
933                                    const Twine &Name, Instruction *MDFrom) {
934   if (auto *CC = dyn_cast<Constant>(C))
935     if (auto *TC = dyn_cast<Constant>(True))
936       if (auto *FC = dyn_cast<Constant>(False))
937         return Insert(Folder.CreateSelect(CC, TC, FC), Name);
938 
939   SelectInst *Sel = SelectInst::Create(C, True, False);
940   if (MDFrom) {
941     MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);
942     MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);
943     Sel = addBranchMetadata(Sel, Prof, Unpred);
944   }
945   if (isa<FPMathOperator>(Sel))
946     setFPAttrs(Sel, nullptr /* MDNode* */, FMF);
947   return Insert(Sel, Name);
948 }
949 
950 Value *IRBuilderBase::CreatePtrDiff(Value *LHS, Value *RHS,
951                                     const Twine &Name) {
952   assert(LHS->getType() == RHS->getType() &&
953          "Pointer subtraction operand types must match!");
954   auto *ArgType = cast<PointerType>(LHS->getType());
955   Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context));
956   Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context));
957   Value *Difference = CreateSub(LHS_int, RHS_int);
958   return CreateExactSDiv(Difference,
959                          ConstantExpr::getSizeOf(ArgType->getElementType()),
960                          Name);
961 }
962 
963 Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
964   assert(isa<PointerType>(Ptr->getType()) &&
965          "launder.invariant.group only applies to pointers.");
966   // FIXME: we could potentially avoid casts to/from i8*.
967   auto *PtrType = Ptr->getType();
968   auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
969   if (PtrType != Int8PtrTy)
970     Ptr = CreateBitCast(Ptr, Int8PtrTy);
971   Module *M = BB->getParent()->getParent();
972   Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration(
973       M, Intrinsic::launder_invariant_group, {Int8PtrTy});
974 
975   assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy &&
976          FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
977              Int8PtrTy &&
978          "LaunderInvariantGroup should take and return the same type");
979 
980   CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr});
981 
982   if (PtrType != Int8PtrTy)
983     return CreateBitCast(Fn, PtrType);
984   return Fn;
985 }
986 
987 Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
988   assert(isa<PointerType>(Ptr->getType()) &&
989          "strip.invariant.group only applies to pointers.");
990 
991   // FIXME: we could potentially avoid casts to/from i8*.
992   auto *PtrType = Ptr->getType();
993   auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
994   if (PtrType != Int8PtrTy)
995     Ptr = CreateBitCast(Ptr, Int8PtrTy);
996   Module *M = BB->getParent()->getParent();
997   Function *FnStripInvariantGroup = Intrinsic::getDeclaration(
998       M, Intrinsic::strip_invariant_group, {Int8PtrTy});
999 
1000   assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy &&
1001          FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
1002              Int8PtrTy &&
1003          "StripInvariantGroup should take and return the same type");
1004 
1005   CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr});
1006 
1007   if (PtrType != Int8PtrTy)
1008     return CreateBitCast(Fn, PtrType);
1009   return Fn;
1010 }
1011 
1012 Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {
1013   auto *Ty = cast<VectorType>(V->getType());
1014   if (isa<ScalableVectorType>(Ty)) {
1015     Module *M = BB->getParent()->getParent();
1016     Function *F = Intrinsic::getDeclaration(
1017         M, Intrinsic::experimental_vector_reverse, Ty);
1018     return Insert(CallInst::Create(F, V), Name);
1019   }
1020   // Keep the original behaviour for fixed vector
1021   SmallVector<int, 8> ShuffleMask;
1022   int NumElts = Ty->getElementCount().getKnownMinValue();
1023   for (int i = 0; i < NumElts; ++i)
1024     ShuffleMask.push_back(NumElts - i - 1);
1025   return CreateShuffleVector(V, ShuffleMask, Name);
1026 }
1027 
1028 Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
1029                                         const Twine &Name) {
1030   auto EC = ElementCount::getFixed(NumElts);
1031   return CreateVectorSplat(EC, V, Name);
1032 }
1033 
1034 Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
1035                                         const Twine &Name) {
1036   assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1037 
1038   // First insert it into a poison vector so we can shuffle it.
1039   Type *I32Ty = getInt32Ty();
1040   Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
1041   V = CreateInsertElement(Poison, V, ConstantInt::get(I32Ty, 0),
1042                           Name + ".splatinsert");
1043 
1044   // Shuffle the value across the desired number of elements.
1045   SmallVector<int, 16> Zeros;
1046   Zeros.resize(EC.getKnownMinValue());
1047   return CreateShuffleVector(V, Zeros, Name + ".splat");
1048 }
1049 
1050 Value *IRBuilderBase::CreateExtractInteger(
1051     const DataLayout &DL, Value *From, IntegerType *ExtractedTy,
1052     uint64_t Offset, const Twine &Name) {
1053   auto *IntTy = cast<IntegerType>(From->getType());
1054   assert(DL.getTypeStoreSize(ExtractedTy) + Offset <=
1055              DL.getTypeStoreSize(IntTy) &&
1056          "Element extends past full value");
1057   uint64_t ShAmt = 8 * Offset;
1058   Value *V = From;
1059   if (DL.isBigEndian())
1060     ShAmt = 8 * (DL.getTypeStoreSize(IntTy) -
1061                  DL.getTypeStoreSize(ExtractedTy) - Offset);
1062   if (ShAmt) {
1063     V = CreateLShr(V, ShAmt, Name + ".shift");
1064   }
1065   assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() &&
1066          "Cannot extract to a larger integer!");
1067   if (ExtractedTy != IntTy) {
1068     V = CreateTrunc(V, ExtractedTy, Name + ".trunc");
1069   }
1070   return V;
1071 }
1072 
1073 Value *IRBuilderBase::CreatePreserveArrayAccessIndex(
1074     Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex,
1075     MDNode *DbgInfo) {
1076   assert(isa<PointerType>(Base->getType()) &&
1077          "Invalid Base ptr type for preserve.array.access.index.");
1078   auto *BaseType = Base->getType();
1079 
1080   Value *LastIndexV = getInt32(LastIndex);
1081   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1082   SmallVector<Value *, 4> IdxList(Dimension, Zero);
1083   IdxList.push_back(LastIndexV);
1084 
1085   Type *ResultType =
1086       GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList);
1087 
1088   Module *M = BB->getParent()->getParent();
1089   Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration(
1090       M, Intrinsic::preserve_array_access_index, {ResultType, BaseType});
1091 
1092   Value *DimV = getInt32(Dimension);
1093   CallInst *Fn =
1094       CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV});
1095   if (DbgInfo)
1096     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1097 
1098   return Fn;
1099 }
1100 
1101 Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
1102     Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1103   assert(isa<PointerType>(Base->getType()) &&
1104          "Invalid Base ptr type for preserve.union.access.index.");
1105   auto *BaseType = Base->getType();
1106 
1107   Module *M = BB->getParent()->getParent();
1108   Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration(
1109       M, Intrinsic::preserve_union_access_index, {BaseType, BaseType});
1110 
1111   Value *DIIndex = getInt32(FieldIndex);
1112   CallInst *Fn =
1113       CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex});
1114   if (DbgInfo)
1115     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1116 
1117   return Fn;
1118 }
1119 
1120 Value *IRBuilderBase::CreatePreserveStructAccessIndex(
1121     Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1122     MDNode *DbgInfo) {
1123   assert(isa<PointerType>(Base->getType()) &&
1124          "Invalid Base ptr type for preserve.struct.access.index.");
1125   auto *BaseType = Base->getType();
1126 
1127   Value *GEPIndex = getInt32(Index);
1128   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1129   Type *ResultType =
1130       GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex});
1131 
1132   Module *M = BB->getParent()->getParent();
1133   Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration(
1134       M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType});
1135 
1136   Value *DIIndex = getInt32(FieldIndex);
1137   CallInst *Fn = CreateCall(FnPreserveStructAccessIndex,
1138                             {Base, GEPIndex, DIIndex});
1139   if (DbgInfo)
1140     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1141 
1142   return Fn;
1143 }
1144 
1145 CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1146                                                          Value *PtrValue,
1147                                                          Value *AlignValue,
1148                                                          Value *OffsetValue) {
1149   SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1150   if (OffsetValue)
1151     Vals.push_back(OffsetValue);
1152   OperandBundleDefT<Value *> AlignOpB("align", Vals);
1153   return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB});
1154 }
1155 
1156 CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1157                                                    Value *PtrValue,
1158                                                    unsigned Alignment,
1159                                                    Value *OffsetValue) {
1160   assert(isa<PointerType>(PtrValue->getType()) &&
1161          "trying to create an alignment assumption on a non-pointer?");
1162   assert(Alignment != 0 && "Invalid Alignment");
1163   auto *PtrTy = cast<PointerType>(PtrValue->getType());
1164   Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
1165   Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment);
1166   return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1167 }
1168 
1169 CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1170                                                    Value *PtrValue,
1171                                                    Value *Alignment,
1172                                                    Value *OffsetValue) {
1173   assert(isa<PointerType>(PtrValue->getType()) &&
1174          "trying to create an alignment assumption on a non-pointer?");
1175   return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
1176 }
1177 
1178 IRBuilderDefaultInserter::~IRBuilderDefaultInserter() {}
1179 IRBuilderCallbackInserter::~IRBuilderCallbackInserter() {}
1180 IRBuilderFolder::~IRBuilderFolder() {}
1181 void ConstantFolder::anchor() {}
1182 void NoFolder::anchor() {}
1183