1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the IRBuilder class, which is used as a convenient way
10 // to create LLVM instructions with a consistent and simplified interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/None.h"
17 #include "llvm/IR/Constant.h"
18 #include "llvm/IR/Constants.h"
19 #include "llvm/IR/DerivedTypes.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/GlobalVariable.h"
23 #include "llvm/IR/IntrinsicInst.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/NoFolder.h"
27 #include "llvm/IR/Operator.h"
28 #include "llvm/IR/Statepoint.h"
29 #include "llvm/IR/Type.h"
30 #include "llvm/IR/Value.h"
31 #include "llvm/Support/Casting.h"
32 #include "llvm/Support/MathExtras.h"
33 #include <cassert>
34 #include <cstdint>
35 #include <vector>
36 
37 using namespace llvm;
38 
39 /// CreateGlobalString - Make a new global variable with an initializer that
40 /// has array of i8 type filled in with the nul terminated string value
41 /// specified.  If Name is specified, it is the name of the global variable
42 /// created.
43 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
44                                                   const Twine &Name,
45                                                   unsigned AddressSpace) {
46   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
47   Module &M = *BB->getParent()->getParent();
48   auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
49                                 GlobalValue::PrivateLinkage, StrConstant, Name,
50                                 nullptr, GlobalVariable::NotThreadLocal,
51                                 AddressSpace);
52   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
53   GV->setAlignment(Align(1));
54   return GV;
55 }
56 
57 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
58   assert(BB && BB->getParent() && "No current function!");
59   return BB->getParent()->getReturnType();
60 }
61 
62 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
63   auto *PT = cast<PointerType>(Ptr->getType());
64   if (PT->getElementType()->isIntegerTy(8))
65     return Ptr;
66 
67   // Otherwise, we need to insert a bitcast.
68   return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace()));
69 }
70 
71 static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
72                                   IRBuilderBase *Builder,
73                                   const Twine &Name = "",
74                                   Instruction *FMFSource = nullptr,
75                                   ArrayRef<OperandBundleDef> OpBundles = {}) {
76   CallInst *CI = Builder->CreateCall(Callee, Ops, OpBundles, Name);
77   if (FMFSource)
78     CI->copyFastMathFlags(FMFSource);
79   return CI;
80 }
81 
82 CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
83                                       MaybeAlign Align, bool isVolatile,
84                                       MDNode *TBAATag, MDNode *ScopeTag,
85                                       MDNode *NoAliasTag) {
86   Ptr = getCastedInt8PtrValue(Ptr);
87   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
88   Type *Tys[] = { Ptr->getType(), Size->getType() };
89   Module *M = BB->getParent()->getParent();
90   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
91 
92   CallInst *CI = createCallHelper(TheFn, Ops, this);
93 
94   if (Align)
95     cast<MemSetInst>(CI)->setDestAlignment(Align->value());
96 
97   // Set the TBAA info if present.
98   if (TBAATag)
99     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
100 
101   if (ScopeTag)
102     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
103 
104   if (NoAliasTag)
105     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
106 
107   return CI;
108 }
109 
110 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
111     Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
112     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
113 
114   Ptr = getCastedInt8PtrValue(Ptr);
115   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
116   Type *Tys[] = {Ptr->getType(), Size->getType()};
117   Module *M = BB->getParent()->getParent();
118   Function *TheFn = Intrinsic::getDeclaration(
119       M, Intrinsic::memset_element_unordered_atomic, Tys);
120 
121   CallInst *CI = createCallHelper(TheFn, Ops, this);
122 
123   cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
124 
125   // Set the TBAA info if present.
126   if (TBAATag)
127     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
128 
129   if (ScopeTag)
130     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
131 
132   if (NoAliasTag)
133     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
134 
135   return CI;
136 }
137 
138 CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, MaybeAlign DstAlign,
139                                       Value *Src, MaybeAlign SrcAlign,
140                                       Value *Size, bool isVolatile,
141                                       MDNode *TBAATag, MDNode *TBAAStructTag,
142                                       MDNode *ScopeTag, MDNode *NoAliasTag) {
143   Dst = getCastedInt8PtrValue(Dst);
144   Src = getCastedInt8PtrValue(Src);
145 
146   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
147   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
148   Module *M = BB->getParent()->getParent();
149   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
150 
151   CallInst *CI = createCallHelper(TheFn, Ops, this);
152 
153   auto* MCI = cast<MemCpyInst>(CI);
154   if (DstAlign)
155     MCI->setDestAlignment(*DstAlign);
156   if (SrcAlign)
157     MCI->setSourceAlignment(*SrcAlign);
158 
159   // Set the TBAA info if present.
160   if (TBAATag)
161     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
162 
163   // Set the TBAA Struct info if present.
164   if (TBAAStructTag)
165     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
166 
167   if (ScopeTag)
168     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
169 
170   if (NoAliasTag)
171     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
172 
173   return CI;
174 }
175 
176 CallInst *IRBuilderBase::CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign,
177                                             Value *Src, MaybeAlign SrcAlign,
178                                             Value *Size) {
179   Dst = getCastedInt8PtrValue(Dst);
180   Src = getCastedInt8PtrValue(Src);
181   Value *IsVolatile = getInt1(false);
182 
183   Value *Ops[] = {Dst, Src, Size, IsVolatile};
184   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
185   Function *F = BB->getParent();
186   Module *M = F->getParent();
187   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys);
188 
189   CallInst *CI = createCallHelper(TheFn, Ops, this);
190 
191   auto *MCI = cast<MemCpyInlineInst>(CI);
192   if (DstAlign)
193     MCI->setDestAlignment(*DstAlign);
194   if (SrcAlign)
195     MCI->setSourceAlignment(*SrcAlign);
196 
197   return CI;
198 }
199 
200 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
201     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
202     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
203     MDNode *ScopeTag, MDNode *NoAliasTag) {
204   assert(DstAlign >= ElementSize &&
205          "Pointer alignment must be at least element size");
206   assert(SrcAlign >= ElementSize &&
207          "Pointer alignment must be at least element size");
208   Dst = getCastedInt8PtrValue(Dst);
209   Src = getCastedInt8PtrValue(Src);
210 
211   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
212   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
213   Module *M = BB->getParent()->getParent();
214   Function *TheFn = Intrinsic::getDeclaration(
215       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
216 
217   CallInst *CI = createCallHelper(TheFn, Ops, this);
218 
219   // Set the alignment of the pointer args.
220   auto *AMCI = cast<AtomicMemCpyInst>(CI);
221   AMCI->setDestAlignment(DstAlign);
222   AMCI->setSourceAlignment(SrcAlign);
223 
224   // Set the TBAA info if present.
225   if (TBAATag)
226     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
227 
228   // Set the TBAA Struct info if present.
229   if (TBAAStructTag)
230     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
231 
232   if (ScopeTag)
233     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
234 
235   if (NoAliasTag)
236     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
237 
238   return CI;
239 }
240 
241 CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign,
242                                        Value *Src, MaybeAlign SrcAlign,
243                                        Value *Size, bool isVolatile,
244                                        MDNode *TBAATag, MDNode *ScopeTag,
245                                        MDNode *NoAliasTag) {
246   Dst = getCastedInt8PtrValue(Dst);
247   Src = getCastedInt8PtrValue(Src);
248 
249   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
250   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
251   Module *M = BB->getParent()->getParent();
252   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
253 
254   CallInst *CI = createCallHelper(TheFn, Ops, this);
255 
256   auto *MMI = cast<MemMoveInst>(CI);
257   if (DstAlign)
258     MMI->setDestAlignment(*DstAlign);
259   if (SrcAlign)
260     MMI->setSourceAlignment(*SrcAlign);
261 
262   // Set the TBAA info if present.
263   if (TBAATag)
264     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
265 
266   if (ScopeTag)
267     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
268 
269   if (NoAliasTag)
270     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
271 
272   return CI;
273 }
274 
275 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
276     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
277     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
278     MDNode *ScopeTag, MDNode *NoAliasTag) {
279   assert(DstAlign >= ElementSize &&
280          "Pointer alignment must be at least element size");
281   assert(SrcAlign >= ElementSize &&
282          "Pointer alignment must be at least element size");
283   Dst = getCastedInt8PtrValue(Dst);
284   Src = getCastedInt8PtrValue(Src);
285 
286   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
287   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
288   Module *M = BB->getParent()->getParent();
289   Function *TheFn = Intrinsic::getDeclaration(
290       M, Intrinsic::memmove_element_unordered_atomic, Tys);
291 
292   CallInst *CI = createCallHelper(TheFn, Ops, this);
293 
294   // Set the alignment of the pointer args.
295   CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
296   CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
297 
298   // Set the TBAA info if present.
299   if (TBAATag)
300     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
301 
302   // Set the TBAA Struct info if present.
303   if (TBAAStructTag)
304     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
305 
306   if (ScopeTag)
307     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
308 
309   if (NoAliasTag)
310     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
311 
312   return CI;
313 }
314 
315 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
316                                     Value *Src) {
317   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
318   Value *Ops[] = {Src};
319   Type *Tys[] = { Src->getType() };
320   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
321   return createCallHelper(Decl, Ops, Builder);
322 }
323 
324 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
325   Module *M = GetInsertBlock()->getParent()->getParent();
326   Value *Ops[] = {Acc, Src};
327   Type *Tys[] = {Acc->getType(), Src->getType()};
328   auto Decl = Intrinsic::getDeclaration(
329       M, Intrinsic::experimental_vector_reduce_v2_fadd, Tys);
330   return createCallHelper(Decl, Ops, this);
331 }
332 
333 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
334   Module *M = GetInsertBlock()->getParent()->getParent();
335   Value *Ops[] = {Acc, Src};
336   Type *Tys[] = {Acc->getType(), Src->getType()};
337   auto Decl = Intrinsic::getDeclaration(
338       M, Intrinsic::experimental_vector_reduce_v2_fmul, Tys);
339   return createCallHelper(Decl, Ops, this);
340 }
341 
342 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
343   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
344                                Src);
345 }
346 
347 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
348   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
349                                Src);
350 }
351 
352 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
353   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
354                                Src);
355 }
356 
357 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
358   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
359                                Src);
360 }
361 
362 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
363   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
364                                Src);
365 }
366 
367 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
368   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
369                      : Intrinsic::experimental_vector_reduce_umax;
370   return getReductionIntrinsic(this, ID, Src);
371 }
372 
373 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
374   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
375                      : Intrinsic::experimental_vector_reduce_umin;
376   return getReductionIntrinsic(this, ID, Src);
377 }
378 
379 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
380   auto Rdx = getReductionIntrinsic(
381       this, Intrinsic::experimental_vector_reduce_fmax, Src);
382   if (NoNaN) {
383     FastMathFlags FMF;
384     FMF.setNoNaNs();
385     Rdx->setFastMathFlags(FMF);
386   }
387   return Rdx;
388 }
389 
390 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
391   auto Rdx = getReductionIntrinsic(
392       this, Intrinsic::experimental_vector_reduce_fmin, Src);
393   if (NoNaN) {
394     FastMathFlags FMF;
395     FMF.setNoNaNs();
396     Rdx->setFastMathFlags(FMF);
397   }
398   return Rdx;
399 }
400 
401 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
402   assert(isa<PointerType>(Ptr->getType()) &&
403          "lifetime.start only applies to pointers.");
404   Ptr = getCastedInt8PtrValue(Ptr);
405   if (!Size)
406     Size = getInt64(-1);
407   else
408     assert(Size->getType() == getInt64Ty() &&
409            "lifetime.start requires the size to be an i64");
410   Value *Ops[] = { Size, Ptr };
411   Module *M = BB->getParent()->getParent();
412   Function *TheFn =
413       Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
414   return createCallHelper(TheFn, Ops, this);
415 }
416 
417 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
418   assert(isa<PointerType>(Ptr->getType()) &&
419          "lifetime.end only applies to pointers.");
420   Ptr = getCastedInt8PtrValue(Ptr);
421   if (!Size)
422     Size = getInt64(-1);
423   else
424     assert(Size->getType() == getInt64Ty() &&
425            "lifetime.end requires the size to be an i64");
426   Value *Ops[] = { Size, Ptr };
427   Module *M = BB->getParent()->getParent();
428   Function *TheFn =
429       Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
430   return createCallHelper(TheFn, Ops, this);
431 }
432 
433 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
434 
435   assert(isa<PointerType>(Ptr->getType()) &&
436          "invariant.start only applies to pointers.");
437   Ptr = getCastedInt8PtrValue(Ptr);
438   if (!Size)
439     Size = getInt64(-1);
440   else
441     assert(Size->getType() == getInt64Ty() &&
442            "invariant.start requires the size to be an i64");
443 
444   Value *Ops[] = {Size, Ptr};
445   // Fill in the single overloaded type: memory object type.
446   Type *ObjectPtr[1] = {Ptr->getType()};
447   Module *M = BB->getParent()->getParent();
448   Function *TheFn =
449       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
450   return createCallHelper(TheFn, Ops, this);
451 }
452 
453 CallInst *
454 IRBuilderBase::CreateAssumption(Value *Cond,
455                                 ArrayRef<OperandBundleDef> OpBundles) {
456   assert(Cond->getType() == getInt1Ty() &&
457          "an assumption condition must be of type i1");
458 
459   Value *Ops[] = { Cond };
460   Module *M = BB->getParent()->getParent();
461   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
462   return createCallHelper(FnAssume, Ops, this, "", nullptr, OpBundles);
463 }
464 
465 /// Create a call to a Masked Load intrinsic.
466 /// \p Ptr       - base pointer for the load
467 /// \p Alignment - alignment of the source location
468 /// \p Mask      - vector of booleans which indicates what vector lanes should
469 ///                be accessed in memory
470 /// \p PassThru  - pass-through value that is used to fill the masked-off lanes
471 ///                of the result
472 /// \p Name      - name of the result variable
473 CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, Align Alignment,
474                                           Value *Mask, Value *PassThru,
475                                           const Twine &Name) {
476   auto *PtrTy = cast<PointerType>(Ptr->getType());
477   Type *DataTy = PtrTy->getElementType();
478   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
479   assert(Mask && "Mask should not be all-ones (null)");
480   if (!PassThru)
481     PassThru = UndefValue::get(DataTy);
482   Type *OverloadedTypes[] = { DataTy, PtrTy };
483   Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
484   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
485                                OverloadedTypes, Name);
486 }
487 
488 /// Create a call to a Masked Store intrinsic.
489 /// \p Val       - data to be stored,
490 /// \p Ptr       - base pointer for the store
491 /// \p Alignment - alignment of the destination location
492 /// \p Mask      - vector of booleans which indicates what vector lanes should
493 ///                be accessed in memory
494 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
495                                            Align Alignment, Value *Mask) {
496   auto *PtrTy = cast<PointerType>(Ptr->getType());
497   Type *DataTy = PtrTy->getElementType();
498   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
499   assert(Mask && "Mask should not be all-ones (null)");
500   Type *OverloadedTypes[] = { DataTy, PtrTy };
501   Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
502   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
503 }
504 
505 /// Create a call to a Masked intrinsic, with given intrinsic Id,
506 /// an array of operands - Ops, and an array of overloaded types -
507 /// OverloadedTypes.
508 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
509                                                ArrayRef<Value *> Ops,
510                                                ArrayRef<Type *> OverloadedTypes,
511                                                const Twine &Name) {
512   Module *M = BB->getParent()->getParent();
513   Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
514   return createCallHelper(TheFn, Ops, this, Name);
515 }
516 
517 /// Create a call to a Masked Gather intrinsic.
518 /// \p Ptrs     - vector of pointers for loading
519 /// \p Align    - alignment for one element
520 /// \p Mask     - vector of booleans which indicates what vector lanes should
521 ///               be accessed in memory
522 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
523 ///               of the result
524 /// \p Name     - name of the result variable
525 CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, Align Alignment,
526                                             Value *Mask, Value *PassThru,
527                                             const Twine &Name) {
528   auto PtrsTy = cast<VectorType>(Ptrs->getType());
529   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
530   unsigned NumElts = PtrsTy->getNumElements();
531   auto *DataTy = FixedVectorType::get(PtrTy->getElementType(), NumElts);
532 
533   if (!Mask)
534     Mask = Constant::getAllOnesValue(
535         FixedVectorType::get(Type::getInt1Ty(Context), NumElts));
536 
537   if (!PassThru)
538     PassThru = UndefValue::get(DataTy);
539 
540   Type *OverloadedTypes[] = {DataTy, PtrsTy};
541   Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
542 
543   // We specify only one type when we create this intrinsic. Types of other
544   // arguments are derived from this type.
545   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
546                                Name);
547 }
548 
549 /// Create a call to a Masked Scatter intrinsic.
550 /// \p Data  - data to be stored,
551 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
552 ///            stored
553 /// \p Align - alignment for one element
554 /// \p Mask  - vector of booleans which indicates what vector lanes should
555 ///            be accessed in memory
556 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
557                                              Align Alignment, Value *Mask) {
558   auto PtrsTy = cast<VectorType>(Ptrs->getType());
559   auto DataTy = cast<VectorType>(Data->getType());
560   unsigned NumElts = PtrsTy->getNumElements();
561 
562 #ifndef NDEBUG
563   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
564   assert(NumElts == DataTy->getNumElements() &&
565          PtrTy->getElementType() == DataTy->getElementType() &&
566          "Incompatible pointer and data types");
567 #endif
568 
569   if (!Mask)
570     Mask = Constant::getAllOnesValue(
571         FixedVectorType::get(Type::getInt1Ty(Context), NumElts));
572 
573   Type *OverloadedTypes[] = {DataTy, PtrsTy};
574   Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
575 
576   // We specify only one type when we create this intrinsic. Types of other
577   // arguments are derived from this type.
578   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
579 }
580 
581 template <typename T0>
582 static std::vector<Value *>
583 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
584                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
585   std::vector<Value *> Args;
586   Args.push_back(B.getInt64(ID));
587   Args.push_back(B.getInt32(NumPatchBytes));
588   Args.push_back(ActualCallee);
589   Args.push_back(B.getInt32(CallArgs.size()));
590   Args.push_back(B.getInt32(Flags));
591   Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
592   // GC Transition and Deopt args are now always handled via operand bundle.
593   // They will be removed from the signature of gc.statepoint shortly.
594   Args.push_back(B.getInt32(0));
595   Args.push_back(B.getInt32(0));
596   // GC args are now encoded in the gc-live operand bundle
597   return Args;
598 }
599 
600 template<typename T1, typename T2, typename T3>
601 static std::vector<OperandBundleDef>
602 getStatepointBundles(Optional<ArrayRef<T1>> TransitionArgs,
603                      Optional<ArrayRef<T2>> DeoptArgs,
604                      ArrayRef<T3> GCArgs) {
605   std::vector<OperandBundleDef> Rval;
606   if (DeoptArgs) {
607     SmallVector<Value*, 16> DeoptValues;
608     DeoptValues.insert(DeoptValues.end(), DeoptArgs->begin(), DeoptArgs->end());
609     Rval.emplace_back("deopt", DeoptValues);
610   }
611   if (TransitionArgs) {
612     SmallVector<Value*, 16> TransitionValues;
613     TransitionValues.insert(TransitionValues.end(),
614                             TransitionArgs->begin(), TransitionArgs->end());
615     Rval.emplace_back("gc-transition", TransitionValues);
616   }
617   if (GCArgs.size()) {
618     SmallVector<Value*, 16> LiveValues;
619     LiveValues.insert(LiveValues.end(), GCArgs.begin(), GCArgs.end());
620     Rval.emplace_back("gc-live", LiveValues);
621   }
622   return Rval;
623 }
624 
625 template <typename T0, typename T1, typename T2, typename T3>
626 static CallInst *CreateGCStatepointCallCommon(
627     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
628     Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
629     Optional<ArrayRef<T1>> TransitionArgs,
630     Optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
631     const Twine &Name) {
632   // Extract out the type of the callee.
633   auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
634   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
635          "actual callee must be a callable value");
636 
637   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
638   // Fill in the one generic type'd argument (the function is also vararg)
639   Type *ArgTypes[] = { FuncPtrType };
640   Function *FnStatepoint =
641     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
642                               ArgTypes);
643 
644   std::vector<Value *> Args =
645       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
646                         CallArgs);
647 
648   return Builder->CreateCall(FnStatepoint, Args,
649                              getStatepointBundles(TransitionArgs, DeoptArgs,
650                                                   GCArgs),
651                              Name);
652 }
653 
654 CallInst *IRBuilderBase::CreateGCStatepointCall(
655     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
656     ArrayRef<Value *> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs,
657     ArrayRef<Value *> GCArgs, const Twine &Name) {
658   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
659       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
660       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
661 }
662 
663 CallInst *IRBuilderBase::CreateGCStatepointCall(
664     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
665     ArrayRef<Use> CallArgs, Optional<ArrayRef<Use>> TransitionArgs,
666     Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
667     const Twine &Name) {
668   return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
669       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
670       DeoptArgs, GCArgs, Name);
671 }
672 
673 CallInst *IRBuilderBase::CreateGCStatepointCall(
674     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
675     ArrayRef<Use> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs,
676     ArrayRef<Value *> GCArgs, const Twine &Name) {
677   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
678       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
679       CallArgs, None, DeoptArgs, GCArgs, Name);
680 }
681 
682 template <typename T0, typename T1, typename T2, typename T3>
683 static InvokeInst *CreateGCStatepointInvokeCommon(
684     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
685     Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
686     uint32_t Flags, ArrayRef<T0> InvokeArgs,
687     Optional<ArrayRef<T1>> TransitionArgs, Optional<ArrayRef<T2>> DeoptArgs,
688     ArrayRef<T3> GCArgs, const Twine &Name) {
689   // Extract out the type of the callee.
690   auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
691   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
692          "actual callee must be a callable value");
693 
694   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
695   // Fill in the one generic type'd argument (the function is also vararg)
696   Function *FnStatepoint = Intrinsic::getDeclaration(
697       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
698 
699   std::vector<Value *> Args =
700       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
701                         InvokeArgs);
702 
703   return Builder->CreateInvoke(FnStatepoint, NormalDest, UnwindDest, Args,
704                                getStatepointBundles(TransitionArgs, DeoptArgs,
705                                                     GCArgs),
706                                Name);
707 }
708 
709 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
710     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
711     BasicBlock *NormalDest, BasicBlock *UnwindDest,
712     ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Value *>> DeoptArgs,
713     ArrayRef<Value *> GCArgs, const Twine &Name) {
714   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
715       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
716       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
717       DeoptArgs, GCArgs, Name);
718 }
719 
720 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
721     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
722     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
723     ArrayRef<Use> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
724     Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
725   return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
726       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
727       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
728 }
729 
730 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
731     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
732     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
733     Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
734   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
735       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
736       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
737       Name);
738 }
739 
740 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
741                                        Type *ResultType,
742                                        const Twine &Name) {
743  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
744  Module *M = BB->getParent()->getParent();
745  Type *Types[] = {ResultType};
746  Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
747 
748  Value *Args[] = {Statepoint};
749  return createCallHelper(FnGCResult, Args, this, Name);
750 }
751 
752 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
753                                          int BaseOffset,
754                                          int DerivedOffset,
755                                          Type *ResultType,
756                                          const Twine &Name) {
757  Module *M = BB->getParent()->getParent();
758  Type *Types[] = {ResultType};
759  Function *FnGCRelocate =
760      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
761 
762  Value *Args[] = {Statepoint,
763                   getInt32(BaseOffset),
764                   getInt32(DerivedOffset)};
765  return createCallHelper(FnGCRelocate, Args, this, Name);
766 }
767 
768 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
769                                               Instruction *FMFSource,
770                                               const Twine &Name) {
771   Module *M = BB->getModule();
772   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
773   return createCallHelper(Fn, {V}, this, Name, FMFSource);
774 }
775 
776 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
777                                                Value *RHS,
778                                                Instruction *FMFSource,
779                                                const Twine &Name) {
780   Module *M = BB->getModule();
781   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
782   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
783 }
784 
785 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
786                                          ArrayRef<Type *> Types,
787                                          ArrayRef<Value *> Args,
788                                          Instruction *FMFSource,
789                                          const Twine &Name) {
790   Module *M = BB->getModule();
791   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
792   return createCallHelper(Fn, Args, this, Name, FMFSource);
793 }
794 
795 CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
796     Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
797     const Twine &Name, MDNode *FPMathTag,
798     Optional<RoundingMode> Rounding,
799     Optional<fp::ExceptionBehavior> Except) {
800   Value *RoundingV = getConstrainedFPRounding(Rounding);
801   Value *ExceptV = getConstrainedFPExcept(Except);
802 
803   FastMathFlags UseFMF = FMF;
804   if (FMFSource)
805     UseFMF = FMFSource->getFastMathFlags();
806 
807   CallInst *C = CreateIntrinsic(ID, {L->getType()},
808                                 {L, R, RoundingV, ExceptV}, nullptr, Name);
809   setConstrainedFPCallAttr(C);
810   setFPAttrs(C, FPMathTag, UseFMF);
811   return C;
812 }
813 
814 Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
815                                    const Twine &Name, MDNode *FPMathTag) {
816   if (Instruction::isBinaryOp(Opc)) {
817     assert(Ops.size() == 2 && "Invalid number of operands!");
818     return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),
819                        Ops[0], Ops[1], Name, FPMathTag);
820   }
821   if (Instruction::isUnaryOp(Opc)) {
822     assert(Ops.size() == 1 && "Invalid number of operands!");
823     return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),
824                       Ops[0], Name, FPMathTag);
825   }
826   llvm_unreachable("Unexpected opcode!");
827 }
828 
829 CallInst *IRBuilderBase::CreateConstrainedFPCast(
830     Intrinsic::ID ID, Value *V, Type *DestTy,
831     Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag,
832     Optional<RoundingMode> Rounding,
833     Optional<fp::ExceptionBehavior> Except) {
834   Value *ExceptV = getConstrainedFPExcept(Except);
835 
836   FastMathFlags UseFMF = FMF;
837   if (FMFSource)
838     UseFMF = FMFSource->getFastMathFlags();
839 
840   CallInst *C;
841   bool HasRoundingMD = false;
842   switch (ID) {
843   default:
844     break;
845 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
846   case Intrinsic::INTRINSIC:                                \
847     HasRoundingMD = ROUND_MODE;                             \
848     break;
849 #include "llvm/IR/ConstrainedOps.def"
850   }
851   if (HasRoundingMD) {
852     Value *RoundingV = getConstrainedFPRounding(Rounding);
853     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
854                         nullptr, Name);
855   } else
856     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
857                         Name);
858 
859   setConstrainedFPCallAttr(C);
860 
861   if (isa<FPMathOperator>(C))
862     setFPAttrs(C, FPMathTag, UseFMF);
863   return C;
864 }
865 
866 Value *IRBuilderBase::CreateFCmpHelper(
867     CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name,
868     MDNode *FPMathTag, bool IsSignaling) {
869   if (IsFPConstrained) {
870     auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
871                           : Intrinsic::experimental_constrained_fcmp;
872     return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);
873   }
874 
875   if (auto *LC = dyn_cast<Constant>(LHS))
876     if (auto *RC = dyn_cast<Constant>(RHS))
877       return Insert(Folder.CreateFCmp(P, LC, RC), Name);
878   return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name);
879 }
880 
881 CallInst *IRBuilderBase::CreateConstrainedFPCmp(
882     Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
883     const Twine &Name, Optional<fp::ExceptionBehavior> Except) {
884   Value *PredicateV = getConstrainedFPPredicate(P);
885   Value *ExceptV = getConstrainedFPExcept(Except);
886 
887   CallInst *C = CreateIntrinsic(ID, {L->getType()},
888                                 {L, R, PredicateV, ExceptV}, nullptr, Name);
889   setConstrainedFPCallAttr(C);
890   return C;
891 }
892 
893 CallInst *IRBuilderBase::CreateConstrainedFPCall(
894     Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
895     Optional<RoundingMode> Rounding,
896     Optional<fp::ExceptionBehavior> Except) {
897   llvm::SmallVector<Value *, 6> UseArgs;
898 
899   for (auto *OneArg : Args)
900     UseArgs.push_back(OneArg);
901   bool HasRoundingMD = false;
902   switch (Callee->getIntrinsicID()) {
903   default:
904     break;
905 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
906   case Intrinsic::INTRINSIC:                                \
907     HasRoundingMD = ROUND_MODE;                             \
908     break;
909 #include "llvm/IR/ConstrainedOps.def"
910   }
911   if (HasRoundingMD)
912     UseArgs.push_back(getConstrainedFPRounding(Rounding));
913   UseArgs.push_back(getConstrainedFPExcept(Except));
914 
915   CallInst *C = CreateCall(Callee, UseArgs, Name);
916   setConstrainedFPCallAttr(C);
917   return C;
918 }
919 
920 Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
921                                    const Twine &Name, Instruction *MDFrom) {
922   if (auto *CC = dyn_cast<Constant>(C))
923     if (auto *TC = dyn_cast<Constant>(True))
924       if (auto *FC = dyn_cast<Constant>(False))
925         return Insert(Folder.CreateSelect(CC, TC, FC), Name);
926 
927   SelectInst *Sel = SelectInst::Create(C, True, False);
928   if (MDFrom) {
929     MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);
930     MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);
931     Sel = addBranchMetadata(Sel, Prof, Unpred);
932   }
933   if (isa<FPMathOperator>(Sel))
934     setFPAttrs(Sel, nullptr /* MDNode* */, FMF);
935   return Insert(Sel, Name);
936 }
937 
938 Value *IRBuilderBase::CreatePtrDiff(Value *LHS, Value *RHS,
939                                     const Twine &Name) {
940   assert(LHS->getType() == RHS->getType() &&
941          "Pointer subtraction operand types must match!");
942   auto *ArgType = cast<PointerType>(LHS->getType());
943   Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context));
944   Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context));
945   Value *Difference = CreateSub(LHS_int, RHS_int);
946   return CreateExactSDiv(Difference,
947                          ConstantExpr::getSizeOf(ArgType->getElementType()),
948                          Name);
949 }
950 
951 Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
952   assert(isa<PointerType>(Ptr->getType()) &&
953          "launder.invariant.group only applies to pointers.");
954   // FIXME: we could potentially avoid casts to/from i8*.
955   auto *PtrType = Ptr->getType();
956   auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
957   if (PtrType != Int8PtrTy)
958     Ptr = CreateBitCast(Ptr, Int8PtrTy);
959   Module *M = BB->getParent()->getParent();
960   Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration(
961       M, Intrinsic::launder_invariant_group, {Int8PtrTy});
962 
963   assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy &&
964          FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
965              Int8PtrTy &&
966          "LaunderInvariantGroup should take and return the same type");
967 
968   CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr});
969 
970   if (PtrType != Int8PtrTy)
971     return CreateBitCast(Fn, PtrType);
972   return Fn;
973 }
974 
975 Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
976   assert(isa<PointerType>(Ptr->getType()) &&
977          "strip.invariant.group only applies to pointers.");
978 
979   // FIXME: we could potentially avoid casts to/from i8*.
980   auto *PtrType = Ptr->getType();
981   auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
982   if (PtrType != Int8PtrTy)
983     Ptr = CreateBitCast(Ptr, Int8PtrTy);
984   Module *M = BB->getParent()->getParent();
985   Function *FnStripInvariantGroup = Intrinsic::getDeclaration(
986       M, Intrinsic::strip_invariant_group, {Int8PtrTy});
987 
988   assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy &&
989          FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
990              Int8PtrTy &&
991          "StripInvariantGroup should take and return the same type");
992 
993   CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr});
994 
995   if (PtrType != Int8PtrTy)
996     return CreateBitCast(Fn, PtrType);
997   return Fn;
998 }
999 
1000 Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
1001                                         const Twine &Name) {
1002   assert(NumElts > 0 && "Cannot splat to an empty vector!");
1003 
1004   // First insert it into an undef vector so we can shuffle it.
1005   Type *I32Ty = getInt32Ty();
1006   Value *Undef = UndefValue::get(FixedVectorType::get(V->getType(), NumElts));
1007   V = CreateInsertElement(Undef, V, ConstantInt::get(I32Ty, 0),
1008                           Name + ".splatinsert");
1009 
1010   // Shuffle the value across the desired number of elements.
1011   Value *Zeros =
1012       ConstantAggregateZero::get(FixedVectorType::get(I32Ty, NumElts));
1013   return CreateShuffleVector(V, Undef, Zeros, Name + ".splat");
1014 }
1015 
1016 Value *IRBuilderBase::CreateExtractInteger(
1017     const DataLayout &DL, Value *From, IntegerType *ExtractedTy,
1018     uint64_t Offset, const Twine &Name) {
1019   auto *IntTy = cast<IntegerType>(From->getType());
1020   assert(DL.getTypeStoreSize(ExtractedTy) + Offset <=
1021              DL.getTypeStoreSize(IntTy) &&
1022          "Element extends past full value");
1023   uint64_t ShAmt = 8 * Offset;
1024   Value *V = From;
1025   if (DL.isBigEndian())
1026     ShAmt = 8 * (DL.getTypeStoreSize(IntTy) -
1027                  DL.getTypeStoreSize(ExtractedTy) - Offset);
1028   if (ShAmt) {
1029     V = CreateLShr(V, ShAmt, Name + ".shift");
1030   }
1031   assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() &&
1032          "Cannot extract to a larger integer!");
1033   if (ExtractedTy != IntTy) {
1034     V = CreateTrunc(V, ExtractedTy, Name + ".trunc");
1035   }
1036   return V;
1037 }
1038 
1039 Value *IRBuilderBase::CreatePreserveArrayAccessIndex(
1040     Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex,
1041     MDNode *DbgInfo) {
1042   assert(isa<PointerType>(Base->getType()) &&
1043          "Invalid Base ptr type for preserve.array.access.index.");
1044   auto *BaseType = Base->getType();
1045 
1046   Value *LastIndexV = getInt32(LastIndex);
1047   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1048   SmallVector<Value *, 4> IdxList;
1049   for (unsigned I = 0; I < Dimension; ++I)
1050     IdxList.push_back(Zero);
1051   IdxList.push_back(LastIndexV);
1052 
1053   Type *ResultType =
1054       GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList);
1055 
1056   Module *M = BB->getParent()->getParent();
1057   Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration(
1058       M, Intrinsic::preserve_array_access_index, {ResultType, BaseType});
1059 
1060   Value *DimV = getInt32(Dimension);
1061   CallInst *Fn =
1062       CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV});
1063   if (DbgInfo)
1064     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1065 
1066   return Fn;
1067 }
1068 
1069 Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
1070     Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1071   assert(isa<PointerType>(Base->getType()) &&
1072          "Invalid Base ptr type for preserve.union.access.index.");
1073   auto *BaseType = Base->getType();
1074 
1075   Module *M = BB->getParent()->getParent();
1076   Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration(
1077       M, Intrinsic::preserve_union_access_index, {BaseType, BaseType});
1078 
1079   Value *DIIndex = getInt32(FieldIndex);
1080   CallInst *Fn =
1081       CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex});
1082   if (DbgInfo)
1083     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1084 
1085   return Fn;
1086 }
1087 
1088 Value *IRBuilderBase::CreatePreserveStructAccessIndex(
1089     Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1090     MDNode *DbgInfo) {
1091   assert(isa<PointerType>(Base->getType()) &&
1092          "Invalid Base ptr type for preserve.struct.access.index.");
1093   auto *BaseType = Base->getType();
1094 
1095   Value *GEPIndex = getInt32(Index);
1096   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1097   Type *ResultType =
1098       GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex});
1099 
1100   Module *M = BB->getParent()->getParent();
1101   Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration(
1102       M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType});
1103 
1104   Value *DIIndex = getInt32(FieldIndex);
1105   CallInst *Fn = CreateCall(FnPreserveStructAccessIndex,
1106                             {Base, GEPIndex, DIIndex});
1107   if (DbgInfo)
1108     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1109 
1110   return Fn;
1111 }
1112 
1113 CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1114                                                          Value *PtrValue,
1115                                                          Value *AlignValue,
1116                                                          Value *OffsetValue) {
1117   SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1118   if (OffsetValue)
1119     Vals.push_back(OffsetValue);
1120   OperandBundleDefT<Value *> AlignOpB("align", Vals);
1121   return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB});
1122 }
1123 
1124 CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1125                                                    Value *PtrValue,
1126                                                    unsigned Alignment,
1127                                                    Value *OffsetValue) {
1128   assert(isa<PointerType>(PtrValue->getType()) &&
1129          "trying to create an alignment assumption on a non-pointer?");
1130   assert(Alignment != 0 && "Invalid Alignment");
1131   auto *PtrTy = cast<PointerType>(PtrValue->getType());
1132   Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
1133   Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment);
1134   return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1135 }
1136 
1137 CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1138                                                    Value *PtrValue,
1139                                                    Value *Alignment,
1140                                                    Value *OffsetValue) {
1141   assert(isa<PointerType>(PtrValue->getType()) &&
1142          "trying to create an alignment assumption on a non-pointer?");
1143   return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
1144 }
1145 
1146 IRBuilderDefaultInserter::~IRBuilderDefaultInserter() {}
1147 IRBuilderCallbackInserter::~IRBuilderCallbackInserter() {}
1148 IRBuilderFolder::~IRBuilderFolder() {}
1149 void ConstantFolder::anchor() {}
1150 void NoFolder::anchor() {}
1151