1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the IRBuilder class, which is used as a convenient way
10 // to create LLVM instructions with a consistent and simplified interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/None.h"
17 #include "llvm/IR/Constant.h"
18 #include "llvm/IR/Constants.h"
19 #include "llvm/IR/DerivedTypes.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/GlobalVariable.h"
23 #include "llvm/IR/IntrinsicInst.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/NoFolder.h"
27 #include "llvm/IR/Operator.h"
28 #include "llvm/IR/Statepoint.h"
29 #include "llvm/IR/Type.h"
30 #include "llvm/IR/Value.h"
31 #include "llvm/Support/Casting.h"
32 #include "llvm/Support/MathExtras.h"
33 #include <cassert>
34 #include <cstdint>
35 #include <vector>
36 
37 using namespace llvm;
38 
39 /// CreateGlobalString - Make a new global variable with an initializer that
40 /// has array of i8 type filled in with the nul terminated string value
41 /// specified.  If Name is specified, it is the name of the global variable
42 /// created.
43 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
44                                                   const Twine &Name,
45                                                   unsigned AddressSpace,
46                                                   Module *M) {
47   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
48   if (!M)
49     M = BB->getParent()->getParent();
50   auto *GV = new GlobalVariable(
51       *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
52       StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
53   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
54   GV->setAlignment(Align(1));
55   return GV;
56 }
57 
58 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
59   assert(BB && BB->getParent() && "No current function!");
60   return BB->getParent()->getReturnType();
61 }
62 
63 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
64   auto *PT = cast<PointerType>(Ptr->getType());
65   if (PT->getElementType()->isIntegerTy(8))
66     return Ptr;
67 
68   // Otherwise, we need to insert a bitcast.
69   return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace()));
70 }
71 
72 static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
73                                   IRBuilderBase *Builder,
74                                   const Twine &Name = "",
75                                   Instruction *FMFSource = nullptr,
76                                   ArrayRef<OperandBundleDef> OpBundles = {}) {
77   CallInst *CI = Builder->CreateCall(Callee, Ops, OpBundles, Name);
78   if (FMFSource)
79     CI->copyFastMathFlags(FMFSource);
80   return CI;
81 }
82 
83 CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
84                                       MaybeAlign Align, bool isVolatile,
85                                       MDNode *TBAATag, MDNode *ScopeTag,
86                                       MDNode *NoAliasTag) {
87   Ptr = getCastedInt8PtrValue(Ptr);
88   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
89   Type *Tys[] = { Ptr->getType(), Size->getType() };
90   Module *M = BB->getParent()->getParent();
91   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
92 
93   CallInst *CI = createCallHelper(TheFn, Ops, this);
94 
95   if (Align)
96     cast<MemSetInst>(CI)->setDestAlignment(Align->value());
97 
98   // Set the TBAA info if present.
99   if (TBAATag)
100     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
101 
102   if (ScopeTag)
103     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
104 
105   if (NoAliasTag)
106     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
107 
108   return CI;
109 }
110 
111 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
112     Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
113     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
114 
115   Ptr = getCastedInt8PtrValue(Ptr);
116   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
117   Type *Tys[] = {Ptr->getType(), Size->getType()};
118   Module *M = BB->getParent()->getParent();
119   Function *TheFn = Intrinsic::getDeclaration(
120       M, Intrinsic::memset_element_unordered_atomic, Tys);
121 
122   CallInst *CI = createCallHelper(TheFn, Ops, this);
123 
124   cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
125 
126   // Set the TBAA info if present.
127   if (TBAATag)
128     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
129 
130   if (ScopeTag)
131     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
132 
133   if (NoAliasTag)
134     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
135 
136   return CI;
137 }
138 
139 CallInst *IRBuilderBase::CreateMemTransferInst(
140     Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
141     MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag,
142     MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
143   Dst = getCastedInt8PtrValue(Dst);
144   Src = getCastedInt8PtrValue(Src);
145 
146   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
147   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
148   Module *M = BB->getParent()->getParent();
149   Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys);
150 
151   CallInst *CI = createCallHelper(TheFn, Ops, this);
152 
153   auto* MCI = cast<MemTransferInst>(CI);
154   if (DstAlign)
155     MCI->setDestAlignment(*DstAlign);
156   if (SrcAlign)
157     MCI->setSourceAlignment(*SrcAlign);
158 
159   // Set the TBAA info if present.
160   if (TBAATag)
161     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
162 
163   // Set the TBAA Struct info if present.
164   if (TBAAStructTag)
165     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
166 
167   if (ScopeTag)
168     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
169 
170   if (NoAliasTag)
171     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
172 
173   return CI;
174 }
175 
176 CallInst *IRBuilderBase::CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign,
177                                             Value *Src, MaybeAlign SrcAlign,
178                                             Value *Size) {
179   Dst = getCastedInt8PtrValue(Dst);
180   Src = getCastedInt8PtrValue(Src);
181   Value *IsVolatile = getInt1(false);
182 
183   Value *Ops[] = {Dst, Src, Size, IsVolatile};
184   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
185   Function *F = BB->getParent();
186   Module *M = F->getParent();
187   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys);
188 
189   CallInst *CI = createCallHelper(TheFn, Ops, this);
190 
191   auto *MCI = cast<MemCpyInlineInst>(CI);
192   if (DstAlign)
193     MCI->setDestAlignment(*DstAlign);
194   if (SrcAlign)
195     MCI->setSourceAlignment(*SrcAlign);
196 
197   return CI;
198 }
199 
200 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
201     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
202     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
203     MDNode *ScopeTag, MDNode *NoAliasTag) {
204   assert(DstAlign >= ElementSize &&
205          "Pointer alignment must be at least element size");
206   assert(SrcAlign >= ElementSize &&
207          "Pointer alignment must be at least element size");
208   Dst = getCastedInt8PtrValue(Dst);
209   Src = getCastedInt8PtrValue(Src);
210 
211   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
212   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
213   Module *M = BB->getParent()->getParent();
214   Function *TheFn = Intrinsic::getDeclaration(
215       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
216 
217   CallInst *CI = createCallHelper(TheFn, Ops, this);
218 
219   // Set the alignment of the pointer args.
220   auto *AMCI = cast<AtomicMemCpyInst>(CI);
221   AMCI->setDestAlignment(DstAlign);
222   AMCI->setSourceAlignment(SrcAlign);
223 
224   // Set the TBAA info if present.
225   if (TBAATag)
226     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
227 
228   // Set the TBAA Struct info if present.
229   if (TBAAStructTag)
230     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
231 
232   if (ScopeTag)
233     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
234 
235   if (NoAliasTag)
236     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
237 
238   return CI;
239 }
240 
241 CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign,
242                                        Value *Src, MaybeAlign SrcAlign,
243                                        Value *Size, bool isVolatile,
244                                        MDNode *TBAATag, MDNode *ScopeTag,
245                                        MDNode *NoAliasTag) {
246   Dst = getCastedInt8PtrValue(Dst);
247   Src = getCastedInt8PtrValue(Src);
248 
249   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
250   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
251   Module *M = BB->getParent()->getParent();
252   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
253 
254   CallInst *CI = createCallHelper(TheFn, Ops, this);
255 
256   auto *MMI = cast<MemMoveInst>(CI);
257   if (DstAlign)
258     MMI->setDestAlignment(*DstAlign);
259   if (SrcAlign)
260     MMI->setSourceAlignment(*SrcAlign);
261 
262   // Set the TBAA info if present.
263   if (TBAATag)
264     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
265 
266   if (ScopeTag)
267     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
268 
269   if (NoAliasTag)
270     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
271 
272   return CI;
273 }
274 
275 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
276     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
277     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
278     MDNode *ScopeTag, MDNode *NoAliasTag) {
279   assert(DstAlign >= ElementSize &&
280          "Pointer alignment must be at least element size");
281   assert(SrcAlign >= ElementSize &&
282          "Pointer alignment must be at least element size");
283   Dst = getCastedInt8PtrValue(Dst);
284   Src = getCastedInt8PtrValue(Src);
285 
286   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
287   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
288   Module *M = BB->getParent()->getParent();
289   Function *TheFn = Intrinsic::getDeclaration(
290       M, Intrinsic::memmove_element_unordered_atomic, Tys);
291 
292   CallInst *CI = createCallHelper(TheFn, Ops, this);
293 
294   // Set the alignment of the pointer args.
295   CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
296   CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
297 
298   // Set the TBAA info if present.
299   if (TBAATag)
300     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
301 
302   // Set the TBAA Struct info if present.
303   if (TBAAStructTag)
304     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
305 
306   if (ScopeTag)
307     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
308 
309   if (NoAliasTag)
310     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
311 
312   return CI;
313 }
314 
315 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
316                                     Value *Src) {
317   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
318   Value *Ops[] = {Src};
319   Type *Tys[] = { Src->getType() };
320   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
321   return createCallHelper(Decl, Ops, Builder);
322 }
323 
324 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
325   Module *M = GetInsertBlock()->getParent()->getParent();
326   Value *Ops[] = {Acc, Src};
327   auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd,
328                                         {Src->getType()});
329   return createCallHelper(Decl, Ops, this);
330 }
331 
332 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
333   Module *M = GetInsertBlock()->getParent()->getParent();
334   Value *Ops[] = {Acc, Src};
335   auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul,
336                                         {Src->getType()});
337   return createCallHelper(Decl, Ops, this);
338 }
339 
340 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
341   return getReductionIntrinsic(this, Intrinsic::vector_reduce_add, Src);
342 }
343 
344 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
345   return getReductionIntrinsic(this, Intrinsic::vector_reduce_mul, Src);
346 }
347 
348 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
349   return getReductionIntrinsic(this, Intrinsic::vector_reduce_and, Src);
350 }
351 
352 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
353   return getReductionIntrinsic(this, Intrinsic::vector_reduce_or, Src);
354 }
355 
356 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
357   return getReductionIntrinsic(this, Intrinsic::vector_reduce_xor, Src);
358 }
359 
360 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
361   auto ID =
362       IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
363   return getReductionIntrinsic(this, ID, Src);
364 }
365 
366 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
367   auto ID =
368       IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
369   return getReductionIntrinsic(this, ID, Src);
370 }
371 
372 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
373   auto Rdx = getReductionIntrinsic(this, Intrinsic::vector_reduce_fmax, Src);
374   if (NoNaN) {
375     FastMathFlags FMF;
376     FMF.setNoNaNs();
377     Rdx->setFastMathFlags(FMF);
378   }
379   return Rdx;
380 }
381 
382 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
383   auto Rdx = getReductionIntrinsic(this, Intrinsic::vector_reduce_fmin, Src);
384   if (NoNaN) {
385     FastMathFlags FMF;
386     FMF.setNoNaNs();
387     Rdx->setFastMathFlags(FMF);
388   }
389   return Rdx;
390 }
391 
392 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
393   assert(isa<PointerType>(Ptr->getType()) &&
394          "lifetime.start only applies to pointers.");
395   Ptr = getCastedInt8PtrValue(Ptr);
396   if (!Size)
397     Size = getInt64(-1);
398   else
399     assert(Size->getType() == getInt64Ty() &&
400            "lifetime.start requires the size to be an i64");
401   Value *Ops[] = { Size, Ptr };
402   Module *M = BB->getParent()->getParent();
403   Function *TheFn =
404       Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
405   return createCallHelper(TheFn, Ops, this);
406 }
407 
408 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
409   assert(isa<PointerType>(Ptr->getType()) &&
410          "lifetime.end only applies to pointers.");
411   Ptr = getCastedInt8PtrValue(Ptr);
412   if (!Size)
413     Size = getInt64(-1);
414   else
415     assert(Size->getType() == getInt64Ty() &&
416            "lifetime.end requires the size to be an i64");
417   Value *Ops[] = { Size, Ptr };
418   Module *M = BB->getParent()->getParent();
419   Function *TheFn =
420       Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
421   return createCallHelper(TheFn, Ops, this);
422 }
423 
424 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
425 
426   assert(isa<PointerType>(Ptr->getType()) &&
427          "invariant.start only applies to pointers.");
428   Ptr = getCastedInt8PtrValue(Ptr);
429   if (!Size)
430     Size = getInt64(-1);
431   else
432     assert(Size->getType() == getInt64Ty() &&
433            "invariant.start requires the size to be an i64");
434 
435   Value *Ops[] = {Size, Ptr};
436   // Fill in the single overloaded type: memory object type.
437   Type *ObjectPtr[1] = {Ptr->getType()};
438   Module *M = BB->getParent()->getParent();
439   Function *TheFn =
440       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
441   return createCallHelper(TheFn, Ops, this);
442 }
443 
444 CallInst *
445 IRBuilderBase::CreateAssumption(Value *Cond,
446                                 ArrayRef<OperandBundleDef> OpBundles) {
447   assert(Cond->getType() == getInt1Ty() &&
448          "an assumption condition must be of type i1");
449 
450   Value *Ops[] = { Cond };
451   Module *M = BB->getParent()->getParent();
452   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
453   return createCallHelper(FnAssume, Ops, this, "", nullptr, OpBundles);
454 }
455 
456 /// Create a call to a Masked Load intrinsic.
457 /// \p Ptr       - base pointer for the load
458 /// \p Alignment - alignment of the source location
459 /// \p Mask      - vector of booleans which indicates what vector lanes should
460 ///                be accessed in memory
461 /// \p PassThru  - pass-through value that is used to fill the masked-off lanes
462 ///                of the result
463 /// \p Name      - name of the result variable
464 CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, Align Alignment,
465                                           Value *Mask, Value *PassThru,
466                                           const Twine &Name) {
467   auto *PtrTy = cast<PointerType>(Ptr->getType());
468   Type *DataTy = PtrTy->getElementType();
469   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
470   assert(Mask && "Mask should not be all-ones (null)");
471   if (!PassThru)
472     PassThru = UndefValue::get(DataTy);
473   Type *OverloadedTypes[] = { DataTy, PtrTy };
474   Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
475   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
476                                OverloadedTypes, Name);
477 }
478 
479 /// Create a call to a Masked Store intrinsic.
480 /// \p Val       - data to be stored,
481 /// \p Ptr       - base pointer for the store
482 /// \p Alignment - alignment of the destination location
483 /// \p Mask      - vector of booleans which indicates what vector lanes should
484 ///                be accessed in memory
485 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
486                                            Align Alignment, Value *Mask) {
487   auto *PtrTy = cast<PointerType>(Ptr->getType());
488   Type *DataTy = PtrTy->getElementType();
489   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
490   assert(Mask && "Mask should not be all-ones (null)");
491   Type *OverloadedTypes[] = { DataTy, PtrTy };
492   Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
493   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
494 }
495 
496 /// Create a call to a Masked intrinsic, with given intrinsic Id,
497 /// an array of operands - Ops, and an array of overloaded types -
498 /// OverloadedTypes.
499 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
500                                                ArrayRef<Value *> Ops,
501                                                ArrayRef<Type *> OverloadedTypes,
502                                                const Twine &Name) {
503   Module *M = BB->getParent()->getParent();
504   Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
505   return createCallHelper(TheFn, Ops, this, Name);
506 }
507 
508 /// Create a call to a Masked Gather intrinsic.
509 /// \p Ptrs     - vector of pointers for loading
510 /// \p Align    - alignment for one element
511 /// \p Mask     - vector of booleans which indicates what vector lanes should
512 ///               be accessed in memory
513 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
514 ///               of the result
515 /// \p Name     - name of the result variable
516 CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, Align Alignment,
517                                             Value *Mask, Value *PassThru,
518                                             const Twine &Name) {
519   auto *PtrsTy = cast<FixedVectorType>(Ptrs->getType());
520   auto *PtrTy = cast<PointerType>(PtrsTy->getElementType());
521   unsigned NumElts = PtrsTy->getNumElements();
522   auto *DataTy = FixedVectorType::get(PtrTy->getElementType(), NumElts);
523 
524   if (!Mask)
525     Mask = Constant::getAllOnesValue(
526         FixedVectorType::get(Type::getInt1Ty(Context), NumElts));
527 
528   if (!PassThru)
529     PassThru = UndefValue::get(DataTy);
530 
531   Type *OverloadedTypes[] = {DataTy, PtrsTy};
532   Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
533 
534   // We specify only one type when we create this intrinsic. Types of other
535   // arguments are derived from this type.
536   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
537                                Name);
538 }
539 
540 /// Create a call to a Masked Scatter intrinsic.
541 /// \p Data  - data to be stored,
542 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
543 ///            stored
544 /// \p Align - alignment for one element
545 /// \p Mask  - vector of booleans which indicates what vector lanes should
546 ///            be accessed in memory
547 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
548                                              Align Alignment, Value *Mask) {
549   auto *PtrsTy = cast<FixedVectorType>(Ptrs->getType());
550   auto *DataTy = cast<FixedVectorType>(Data->getType());
551   unsigned NumElts = PtrsTy->getNumElements();
552 
553 #ifndef NDEBUG
554   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
555   assert(NumElts == DataTy->getNumElements() &&
556          PtrTy->getElementType() == DataTy->getElementType() &&
557          "Incompatible pointer and data types");
558 #endif
559 
560   if (!Mask)
561     Mask = Constant::getAllOnesValue(
562         FixedVectorType::get(Type::getInt1Ty(Context), NumElts));
563 
564   Type *OverloadedTypes[] = {DataTy, PtrsTy};
565   Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
566 
567   // We specify only one type when we create this intrinsic. Types of other
568   // arguments are derived from this type.
569   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
570 }
571 
572 template <typename T0>
573 static std::vector<Value *>
574 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
575                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
576   std::vector<Value *> Args;
577   Args.push_back(B.getInt64(ID));
578   Args.push_back(B.getInt32(NumPatchBytes));
579   Args.push_back(ActualCallee);
580   Args.push_back(B.getInt32(CallArgs.size()));
581   Args.push_back(B.getInt32(Flags));
582   Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
583   // GC Transition and Deopt args are now always handled via operand bundle.
584   // They will be removed from the signature of gc.statepoint shortly.
585   Args.push_back(B.getInt32(0));
586   Args.push_back(B.getInt32(0));
587   // GC args are now encoded in the gc-live operand bundle
588   return Args;
589 }
590 
591 template<typename T1, typename T2, typename T3>
592 static std::vector<OperandBundleDef>
593 getStatepointBundles(Optional<ArrayRef<T1>> TransitionArgs,
594                      Optional<ArrayRef<T2>> DeoptArgs,
595                      ArrayRef<T3> GCArgs) {
596   std::vector<OperandBundleDef> Rval;
597   if (DeoptArgs) {
598     SmallVector<Value*, 16> DeoptValues;
599     DeoptValues.insert(DeoptValues.end(), DeoptArgs->begin(), DeoptArgs->end());
600     Rval.emplace_back("deopt", DeoptValues);
601   }
602   if (TransitionArgs) {
603     SmallVector<Value*, 16> TransitionValues;
604     TransitionValues.insert(TransitionValues.end(),
605                             TransitionArgs->begin(), TransitionArgs->end());
606     Rval.emplace_back("gc-transition", TransitionValues);
607   }
608   if (GCArgs.size()) {
609     SmallVector<Value*, 16> LiveValues;
610     LiveValues.insert(LiveValues.end(), GCArgs.begin(), GCArgs.end());
611     Rval.emplace_back("gc-live", LiveValues);
612   }
613   return Rval;
614 }
615 
616 template <typename T0, typename T1, typename T2, typename T3>
617 static CallInst *CreateGCStatepointCallCommon(
618     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
619     Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
620     Optional<ArrayRef<T1>> TransitionArgs,
621     Optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
622     const Twine &Name) {
623   // Extract out the type of the callee.
624   auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
625   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
626          "actual callee must be a callable value");
627 
628   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
629   // Fill in the one generic type'd argument (the function is also vararg)
630   Type *ArgTypes[] = { FuncPtrType };
631   Function *FnStatepoint =
632     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
633                               ArgTypes);
634 
635   std::vector<Value *> Args =
636       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
637                         CallArgs);
638 
639   return Builder->CreateCall(FnStatepoint, Args,
640                              getStatepointBundles(TransitionArgs, DeoptArgs,
641                                                   GCArgs),
642                              Name);
643 }
644 
645 CallInst *IRBuilderBase::CreateGCStatepointCall(
646     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
647     ArrayRef<Value *> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs,
648     ArrayRef<Value *> GCArgs, const Twine &Name) {
649   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
650       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
651       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
652 }
653 
654 CallInst *IRBuilderBase::CreateGCStatepointCall(
655     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
656     ArrayRef<Value *> CallArgs, Optional<ArrayRef<Use>> TransitionArgs,
657     Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
658     const Twine &Name) {
659   return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
660       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
661       DeoptArgs, GCArgs, Name);
662 }
663 
664 CallInst *IRBuilderBase::CreateGCStatepointCall(
665     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
666     ArrayRef<Use> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs,
667     ArrayRef<Value *> GCArgs, const Twine &Name) {
668   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
669       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
670       CallArgs, None, DeoptArgs, GCArgs, Name);
671 }
672 
673 template <typename T0, typename T1, typename T2, typename T3>
674 static InvokeInst *CreateGCStatepointInvokeCommon(
675     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
676     Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
677     uint32_t Flags, ArrayRef<T0> InvokeArgs,
678     Optional<ArrayRef<T1>> TransitionArgs, Optional<ArrayRef<T2>> DeoptArgs,
679     ArrayRef<T3> GCArgs, const Twine &Name) {
680   // Extract out the type of the callee.
681   auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
682   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
683          "actual callee must be a callable value");
684 
685   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
686   // Fill in the one generic type'd argument (the function is also vararg)
687   Function *FnStatepoint = Intrinsic::getDeclaration(
688       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
689 
690   std::vector<Value *> Args =
691       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
692                         InvokeArgs);
693 
694   return Builder->CreateInvoke(FnStatepoint, NormalDest, UnwindDest, Args,
695                                getStatepointBundles(TransitionArgs, DeoptArgs,
696                                                     GCArgs),
697                                Name);
698 }
699 
700 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
701     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
702     BasicBlock *NormalDest, BasicBlock *UnwindDest,
703     ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Value *>> DeoptArgs,
704     ArrayRef<Value *> GCArgs, const Twine &Name) {
705   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
706       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
707       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
708       DeoptArgs, GCArgs, Name);
709 }
710 
711 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
712     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
713     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
714     ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
715     Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
716   return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
717       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
718       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
719 }
720 
721 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
722     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
723     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
724     Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
725   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
726       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
727       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
728       Name);
729 }
730 
731 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
732                                        Type *ResultType,
733                                        const Twine &Name) {
734  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
735  Module *M = BB->getParent()->getParent();
736  Type *Types[] = {ResultType};
737  Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
738 
739  Value *Args[] = {Statepoint};
740  return createCallHelper(FnGCResult, Args, this, Name);
741 }
742 
743 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
744                                          int BaseOffset,
745                                          int DerivedOffset,
746                                          Type *ResultType,
747                                          const Twine &Name) {
748  Module *M = BB->getParent()->getParent();
749  Type *Types[] = {ResultType};
750  Function *FnGCRelocate =
751      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
752 
753  Value *Args[] = {Statepoint,
754                   getInt32(BaseOffset),
755                   getInt32(DerivedOffset)};
756  return createCallHelper(FnGCRelocate, Args, this, Name);
757 }
758 
759 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
760                                               Instruction *FMFSource,
761                                               const Twine &Name) {
762   Module *M = BB->getModule();
763   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
764   return createCallHelper(Fn, {V}, this, Name, FMFSource);
765 }
766 
767 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
768                                                Value *RHS,
769                                                Instruction *FMFSource,
770                                                const Twine &Name) {
771   Module *M = BB->getModule();
772   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
773   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
774 }
775 
776 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
777                                          ArrayRef<Type *> Types,
778                                          ArrayRef<Value *> Args,
779                                          Instruction *FMFSource,
780                                          const Twine &Name) {
781   Module *M = BB->getModule();
782   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
783   return createCallHelper(Fn, Args, this, Name, FMFSource);
784 }
785 
786 CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
787     Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
788     const Twine &Name, MDNode *FPMathTag,
789     Optional<RoundingMode> Rounding,
790     Optional<fp::ExceptionBehavior> Except) {
791   Value *RoundingV = getConstrainedFPRounding(Rounding);
792   Value *ExceptV = getConstrainedFPExcept(Except);
793 
794   FastMathFlags UseFMF = FMF;
795   if (FMFSource)
796     UseFMF = FMFSource->getFastMathFlags();
797 
798   CallInst *C = CreateIntrinsic(ID, {L->getType()},
799                                 {L, R, RoundingV, ExceptV}, nullptr, Name);
800   setConstrainedFPCallAttr(C);
801   setFPAttrs(C, FPMathTag, UseFMF);
802   return C;
803 }
804 
805 Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
806                                    const Twine &Name, MDNode *FPMathTag) {
807   if (Instruction::isBinaryOp(Opc)) {
808     assert(Ops.size() == 2 && "Invalid number of operands!");
809     return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),
810                        Ops[0], Ops[1], Name, FPMathTag);
811   }
812   if (Instruction::isUnaryOp(Opc)) {
813     assert(Ops.size() == 1 && "Invalid number of operands!");
814     return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),
815                       Ops[0], Name, FPMathTag);
816   }
817   llvm_unreachable("Unexpected opcode!");
818 }
819 
820 CallInst *IRBuilderBase::CreateConstrainedFPCast(
821     Intrinsic::ID ID, Value *V, Type *DestTy,
822     Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag,
823     Optional<RoundingMode> Rounding,
824     Optional<fp::ExceptionBehavior> Except) {
825   Value *ExceptV = getConstrainedFPExcept(Except);
826 
827   FastMathFlags UseFMF = FMF;
828   if (FMFSource)
829     UseFMF = FMFSource->getFastMathFlags();
830 
831   CallInst *C;
832   bool HasRoundingMD = false;
833   switch (ID) {
834   default:
835     break;
836 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
837   case Intrinsic::INTRINSIC:                                \
838     HasRoundingMD = ROUND_MODE;                             \
839     break;
840 #include "llvm/IR/ConstrainedOps.def"
841   }
842   if (HasRoundingMD) {
843     Value *RoundingV = getConstrainedFPRounding(Rounding);
844     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
845                         nullptr, Name);
846   } else
847     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
848                         Name);
849 
850   setConstrainedFPCallAttr(C);
851 
852   if (isa<FPMathOperator>(C))
853     setFPAttrs(C, FPMathTag, UseFMF);
854   return C;
855 }
856 
857 Value *IRBuilderBase::CreateFCmpHelper(
858     CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name,
859     MDNode *FPMathTag, bool IsSignaling) {
860   if (IsFPConstrained) {
861     auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
862                           : Intrinsic::experimental_constrained_fcmp;
863     return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);
864   }
865 
866   if (auto *LC = dyn_cast<Constant>(LHS))
867     if (auto *RC = dyn_cast<Constant>(RHS))
868       return Insert(Folder.CreateFCmp(P, LC, RC), Name);
869   return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name);
870 }
871 
872 CallInst *IRBuilderBase::CreateConstrainedFPCmp(
873     Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
874     const Twine &Name, Optional<fp::ExceptionBehavior> Except) {
875   Value *PredicateV = getConstrainedFPPredicate(P);
876   Value *ExceptV = getConstrainedFPExcept(Except);
877 
878   CallInst *C = CreateIntrinsic(ID, {L->getType()},
879                                 {L, R, PredicateV, ExceptV}, nullptr, Name);
880   setConstrainedFPCallAttr(C);
881   return C;
882 }
883 
884 CallInst *IRBuilderBase::CreateConstrainedFPCall(
885     Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
886     Optional<RoundingMode> Rounding,
887     Optional<fp::ExceptionBehavior> Except) {
888   llvm::SmallVector<Value *, 6> UseArgs;
889 
890   for (auto *OneArg : Args)
891     UseArgs.push_back(OneArg);
892   bool HasRoundingMD = false;
893   switch (Callee->getIntrinsicID()) {
894   default:
895     break;
896 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
897   case Intrinsic::INTRINSIC:                                \
898     HasRoundingMD = ROUND_MODE;                             \
899     break;
900 #include "llvm/IR/ConstrainedOps.def"
901   }
902   if (HasRoundingMD)
903     UseArgs.push_back(getConstrainedFPRounding(Rounding));
904   UseArgs.push_back(getConstrainedFPExcept(Except));
905 
906   CallInst *C = CreateCall(Callee, UseArgs, Name);
907   setConstrainedFPCallAttr(C);
908   return C;
909 }
910 
911 Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
912                                    const Twine &Name, Instruction *MDFrom) {
913   if (auto *CC = dyn_cast<Constant>(C))
914     if (auto *TC = dyn_cast<Constant>(True))
915       if (auto *FC = dyn_cast<Constant>(False))
916         return Insert(Folder.CreateSelect(CC, TC, FC), Name);
917 
918   SelectInst *Sel = SelectInst::Create(C, True, False);
919   if (MDFrom) {
920     MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);
921     MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);
922     Sel = addBranchMetadata(Sel, Prof, Unpred);
923   }
924   if (isa<FPMathOperator>(Sel))
925     setFPAttrs(Sel, nullptr /* MDNode* */, FMF);
926   return Insert(Sel, Name);
927 }
928 
929 Value *IRBuilderBase::CreatePtrDiff(Value *LHS, Value *RHS,
930                                     const Twine &Name) {
931   assert(LHS->getType() == RHS->getType() &&
932          "Pointer subtraction operand types must match!");
933   auto *ArgType = cast<PointerType>(LHS->getType());
934   Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context));
935   Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context));
936   Value *Difference = CreateSub(LHS_int, RHS_int);
937   return CreateExactSDiv(Difference,
938                          ConstantExpr::getSizeOf(ArgType->getElementType()),
939                          Name);
940 }
941 
942 Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
943   assert(isa<PointerType>(Ptr->getType()) &&
944          "launder.invariant.group only applies to pointers.");
945   // FIXME: we could potentially avoid casts to/from i8*.
946   auto *PtrType = Ptr->getType();
947   auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
948   if (PtrType != Int8PtrTy)
949     Ptr = CreateBitCast(Ptr, Int8PtrTy);
950   Module *M = BB->getParent()->getParent();
951   Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration(
952       M, Intrinsic::launder_invariant_group, {Int8PtrTy});
953 
954   assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy &&
955          FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
956              Int8PtrTy &&
957          "LaunderInvariantGroup should take and return the same type");
958 
959   CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr});
960 
961   if (PtrType != Int8PtrTy)
962     return CreateBitCast(Fn, PtrType);
963   return Fn;
964 }
965 
966 Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
967   assert(isa<PointerType>(Ptr->getType()) &&
968          "strip.invariant.group only applies to pointers.");
969 
970   // FIXME: we could potentially avoid casts to/from i8*.
971   auto *PtrType = Ptr->getType();
972   auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
973   if (PtrType != Int8PtrTy)
974     Ptr = CreateBitCast(Ptr, Int8PtrTy);
975   Module *M = BB->getParent()->getParent();
976   Function *FnStripInvariantGroup = Intrinsic::getDeclaration(
977       M, Intrinsic::strip_invariant_group, {Int8PtrTy});
978 
979   assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy &&
980          FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
981              Int8PtrTy &&
982          "StripInvariantGroup should take and return the same type");
983 
984   CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr});
985 
986   if (PtrType != Int8PtrTy)
987     return CreateBitCast(Fn, PtrType);
988   return Fn;
989 }
990 
991 Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
992                                         const Twine &Name) {
993   auto EC = ElementCount::getFixed(NumElts);
994   return CreateVectorSplat(EC, V, Name);
995 }
996 
997 Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
998                                         const Twine &Name) {
999   assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1000 
1001   // First insert it into an undef vector so we can shuffle it.
1002   Type *I32Ty = getInt32Ty();
1003   Value *Undef = UndefValue::get(VectorType::get(V->getType(), EC));
1004   V = CreateInsertElement(Undef, V, ConstantInt::get(I32Ty, 0),
1005                           Name + ".splatinsert");
1006 
1007   // Shuffle the value across the desired number of elements.
1008   Value *Zeros = ConstantAggregateZero::get(VectorType::get(I32Ty, EC));
1009   return CreateShuffleVector(V, Undef, Zeros, Name + ".splat");
1010 }
1011 
1012 Value *IRBuilderBase::CreateExtractInteger(
1013     const DataLayout &DL, Value *From, IntegerType *ExtractedTy,
1014     uint64_t Offset, const Twine &Name) {
1015   auto *IntTy = cast<IntegerType>(From->getType());
1016   assert(DL.getTypeStoreSize(ExtractedTy) + Offset <=
1017              DL.getTypeStoreSize(IntTy) &&
1018          "Element extends past full value");
1019   uint64_t ShAmt = 8 * Offset;
1020   Value *V = From;
1021   if (DL.isBigEndian())
1022     ShAmt = 8 * (DL.getTypeStoreSize(IntTy) -
1023                  DL.getTypeStoreSize(ExtractedTy) - Offset);
1024   if (ShAmt) {
1025     V = CreateLShr(V, ShAmt, Name + ".shift");
1026   }
1027   assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() &&
1028          "Cannot extract to a larger integer!");
1029   if (ExtractedTy != IntTy) {
1030     V = CreateTrunc(V, ExtractedTy, Name + ".trunc");
1031   }
1032   return V;
1033 }
1034 
1035 Value *IRBuilderBase::CreatePreserveArrayAccessIndex(
1036     Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex,
1037     MDNode *DbgInfo) {
1038   assert(isa<PointerType>(Base->getType()) &&
1039          "Invalid Base ptr type for preserve.array.access.index.");
1040   auto *BaseType = Base->getType();
1041 
1042   Value *LastIndexV = getInt32(LastIndex);
1043   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1044   SmallVector<Value *, 4> IdxList;
1045   for (unsigned I = 0; I < Dimension; ++I)
1046     IdxList.push_back(Zero);
1047   IdxList.push_back(LastIndexV);
1048 
1049   Type *ResultType =
1050       GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList);
1051 
1052   Module *M = BB->getParent()->getParent();
1053   Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration(
1054       M, Intrinsic::preserve_array_access_index, {ResultType, BaseType});
1055 
1056   Value *DimV = getInt32(Dimension);
1057   CallInst *Fn =
1058       CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV});
1059   if (DbgInfo)
1060     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1061 
1062   return Fn;
1063 }
1064 
1065 Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
1066     Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1067   assert(isa<PointerType>(Base->getType()) &&
1068          "Invalid Base ptr type for preserve.union.access.index.");
1069   auto *BaseType = Base->getType();
1070 
1071   Module *M = BB->getParent()->getParent();
1072   Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration(
1073       M, Intrinsic::preserve_union_access_index, {BaseType, BaseType});
1074 
1075   Value *DIIndex = getInt32(FieldIndex);
1076   CallInst *Fn =
1077       CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex});
1078   if (DbgInfo)
1079     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1080 
1081   return Fn;
1082 }
1083 
1084 Value *IRBuilderBase::CreatePreserveStructAccessIndex(
1085     Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1086     MDNode *DbgInfo) {
1087   assert(isa<PointerType>(Base->getType()) &&
1088          "Invalid Base ptr type for preserve.struct.access.index.");
1089   auto *BaseType = Base->getType();
1090 
1091   Value *GEPIndex = getInt32(Index);
1092   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1093   Type *ResultType =
1094       GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex});
1095 
1096   Module *M = BB->getParent()->getParent();
1097   Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration(
1098       M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType});
1099 
1100   Value *DIIndex = getInt32(FieldIndex);
1101   CallInst *Fn = CreateCall(FnPreserveStructAccessIndex,
1102                             {Base, GEPIndex, DIIndex});
1103   if (DbgInfo)
1104     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1105 
1106   return Fn;
1107 }
1108 
1109 CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1110                                                          Value *PtrValue,
1111                                                          Value *AlignValue,
1112                                                          Value *OffsetValue) {
1113   SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1114   if (OffsetValue)
1115     Vals.push_back(OffsetValue);
1116   OperandBundleDefT<Value *> AlignOpB("align", Vals);
1117   return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB});
1118 }
1119 
1120 CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1121                                                    Value *PtrValue,
1122                                                    unsigned Alignment,
1123                                                    Value *OffsetValue) {
1124   assert(isa<PointerType>(PtrValue->getType()) &&
1125          "trying to create an alignment assumption on a non-pointer?");
1126   assert(Alignment != 0 && "Invalid Alignment");
1127   auto *PtrTy = cast<PointerType>(PtrValue->getType());
1128   Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
1129   Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment);
1130   return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1131 }
1132 
1133 CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1134                                                    Value *PtrValue,
1135                                                    Value *Alignment,
1136                                                    Value *OffsetValue) {
1137   assert(isa<PointerType>(PtrValue->getType()) &&
1138          "trying to create an alignment assumption on a non-pointer?");
1139   return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
1140 }
1141 
1142 IRBuilderDefaultInserter::~IRBuilderDefaultInserter() {}
1143 IRBuilderCallbackInserter::~IRBuilderCallbackInserter() {}
1144 IRBuilderFolder::~IRBuilderFolder() {}
1145 void ConstantFolder::anchor() {}
1146 void NoFolder::anchor() {}
1147