1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the IRBuilder class, which is used as a convenient way
10 // to create LLVM instructions with a consistent and simplified interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/None.h"
17 #include "llvm/IR/Constant.h"
18 #include "llvm/IR/Constants.h"
19 #include "llvm/IR/DebugInfoMetadata.h"
20 #include "llvm/IR/DerivedTypes.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/GlobalValue.h"
23 #include "llvm/IR/GlobalVariable.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/Intrinsics.h"
26 #include "llvm/IR/LLVMContext.h"
27 #include "llvm/IR/NoFolder.h"
28 #include "llvm/IR/Operator.h"
29 #include "llvm/IR/Statepoint.h"
30 #include "llvm/IR/Type.h"
31 #include "llvm/IR/Value.h"
32 #include "llvm/Support/Casting.h"
33 #include <cassert>
34 #include <cstdint>
35 #include <vector>
36 
37 using namespace llvm;
38 
39 /// CreateGlobalString - Make a new global variable with an initializer that
40 /// has array of i8 type filled in with the nul terminated string value
41 /// specified.  If Name is specified, it is the name of the global variable
42 /// created.
43 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
44                                                   const Twine &Name,
45                                                   unsigned AddressSpace,
46                                                   Module *M) {
47   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
48   if (!M)
49     M = BB->getParent()->getParent();
50   auto *GV = new GlobalVariable(
51       *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
52       StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
53   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
54   GV->setAlignment(Align(1));
55   return GV;
56 }
57 
58 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
59   assert(BB && BB->getParent() && "No current function!");
60   return BB->getParent()->getReturnType();
61 }
62 
63 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
64   auto *PT = cast<PointerType>(Ptr->getType());
65   if (PT->isOpaqueOrPointeeTypeMatches(getInt8Ty()))
66     return Ptr;
67 
68   // Otherwise, we need to insert a bitcast.
69   return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace()));
70 }
71 
72 DebugLoc IRBuilderBase::getCurrentDebugLocation() const {
73   for (auto &KV : MetadataToCopy)
74     if (KV.first == LLVMContext::MD_dbg)
75       return {cast<DILocation>(KV.second)};
76 
77   return {};
78 }
79 void IRBuilderBase::SetInstDebugLocation(Instruction *I) const {
80   for (const auto &KV : MetadataToCopy)
81     if (KV.first == LLVMContext::MD_dbg) {
82       I->setDebugLoc(DebugLoc(KV.second));
83       return;
84     }
85 }
86 
87 static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
88                                   IRBuilderBase *Builder,
89                                   const Twine &Name = "",
90                                   Instruction *FMFSource = nullptr,
91                                   ArrayRef<OperandBundleDef> OpBundles = {}) {
92   CallInst *CI = Builder->CreateCall(Callee, Ops, OpBundles, Name);
93   if (FMFSource)
94     CI->copyFastMathFlags(FMFSource);
95   return CI;
96 }
97 
98 Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) {
99   assert(isa<ConstantInt>(Scaling) && "Expected constant integer");
100   if (cast<ConstantInt>(Scaling)->isZero())
101     return Scaling;
102   Module *M = GetInsertBlock()->getParent()->getParent();
103   Function *TheFn =
104       Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()});
105   CallInst *CI = createCallHelper(TheFn, {}, this, Name);
106   return cast<ConstantInt>(Scaling)->getSExtValue() == 1
107              ? CI
108              : CreateMul(CI, Scaling);
109 }
110 
111 Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
112   Type *STy = DstType->getScalarType();
113   if (isa<ScalableVectorType>(DstType)) {
114     Type *StepVecType = DstType;
115     // TODO: We expect this special case (element type < 8 bits) to be
116     // temporary - once the intrinsic properly supports < 8 bits this code
117     // can be removed.
118     if (STy->getScalarSizeInBits() < 8)
119       StepVecType =
120           VectorType::get(getInt8Ty(), cast<ScalableVectorType>(DstType));
121     Value *Res = CreateIntrinsic(Intrinsic::experimental_stepvector,
122                                  {StepVecType}, {}, nullptr, Name);
123     if (StepVecType != DstType)
124       Res = CreateTrunc(Res, DstType);
125     return Res;
126   }
127 
128   unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements();
129 
130   // Create a vector of consecutive numbers from zero to VF.
131   SmallVector<Constant *, 8> Indices;
132   for (unsigned i = 0; i < NumEls; ++i)
133     Indices.push_back(ConstantInt::get(STy, i));
134 
135   // Add the consecutive indices to the vector value.
136   return ConstantVector::get(Indices);
137 }
138 
139 CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
140                                       MaybeAlign Align, bool isVolatile,
141                                       MDNode *TBAATag, MDNode *ScopeTag,
142                                       MDNode *NoAliasTag) {
143   Ptr = getCastedInt8PtrValue(Ptr);
144   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
145   Type *Tys[] = { Ptr->getType(), Size->getType() };
146   Module *M = BB->getParent()->getParent();
147   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
148 
149   CallInst *CI = createCallHelper(TheFn, Ops, this);
150 
151   if (Align)
152     cast<MemSetInst>(CI)->setDestAlignment(Align->value());
153 
154   // Set the TBAA info if present.
155   if (TBAATag)
156     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
157 
158   if (ScopeTag)
159     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
160 
161   if (NoAliasTag)
162     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
163 
164   return CI;
165 }
166 
167 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
168     Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
169     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
170 
171   Ptr = getCastedInt8PtrValue(Ptr);
172   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
173   Type *Tys[] = {Ptr->getType(), Size->getType()};
174   Module *M = BB->getParent()->getParent();
175   Function *TheFn = Intrinsic::getDeclaration(
176       M, Intrinsic::memset_element_unordered_atomic, Tys);
177 
178   CallInst *CI = createCallHelper(TheFn, Ops, this);
179 
180   cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
181 
182   // Set the TBAA info if present.
183   if (TBAATag)
184     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
185 
186   if (ScopeTag)
187     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
188 
189   if (NoAliasTag)
190     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
191 
192   return CI;
193 }
194 
195 CallInst *IRBuilderBase::CreateMemTransferInst(
196     Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
197     MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag,
198     MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
199   Dst = getCastedInt8PtrValue(Dst);
200   Src = getCastedInt8PtrValue(Src);
201 
202   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
203   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
204   Module *M = BB->getParent()->getParent();
205   Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys);
206 
207   CallInst *CI = createCallHelper(TheFn, Ops, this);
208 
209   auto* MCI = cast<MemTransferInst>(CI);
210   if (DstAlign)
211     MCI->setDestAlignment(*DstAlign);
212   if (SrcAlign)
213     MCI->setSourceAlignment(*SrcAlign);
214 
215   // Set the TBAA info if present.
216   if (TBAATag)
217     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
218 
219   // Set the TBAA Struct info if present.
220   if (TBAAStructTag)
221     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
222 
223   if (ScopeTag)
224     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
225 
226   if (NoAliasTag)
227     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
228 
229   return CI;
230 }
231 
232 CallInst *IRBuilderBase::CreateMemCpyInline(
233     Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign,
234     Value *Size, bool IsVolatile, MDNode *TBAATag, MDNode *TBAAStructTag,
235     MDNode *ScopeTag, MDNode *NoAliasTag) {
236   Dst = getCastedInt8PtrValue(Dst);
237   Src = getCastedInt8PtrValue(Src);
238 
239   Value *Ops[] = {Dst, Src, Size, getInt1(IsVolatile)};
240   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
241   Function *F = BB->getParent();
242   Module *M = F->getParent();
243   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys);
244 
245   CallInst *CI = createCallHelper(TheFn, Ops, this);
246 
247   auto *MCI = cast<MemCpyInlineInst>(CI);
248   if (DstAlign)
249     MCI->setDestAlignment(*DstAlign);
250   if (SrcAlign)
251     MCI->setSourceAlignment(*SrcAlign);
252 
253   // Set the TBAA info if present.
254   if (TBAATag)
255     MCI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
256 
257   // Set the TBAA Struct info if present.
258   if (TBAAStructTag)
259     MCI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
260 
261   if (ScopeTag)
262     MCI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
263 
264   if (NoAliasTag)
265     MCI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
266 
267   return CI;
268 }
269 
270 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
271     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
272     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
273     MDNode *ScopeTag, MDNode *NoAliasTag) {
274   assert(DstAlign >= ElementSize &&
275          "Pointer alignment must be at least element size");
276   assert(SrcAlign >= ElementSize &&
277          "Pointer alignment must be at least element size");
278   Dst = getCastedInt8PtrValue(Dst);
279   Src = getCastedInt8PtrValue(Src);
280 
281   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
282   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
283   Module *M = BB->getParent()->getParent();
284   Function *TheFn = Intrinsic::getDeclaration(
285       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
286 
287   CallInst *CI = createCallHelper(TheFn, Ops, this);
288 
289   // Set the alignment of the pointer args.
290   auto *AMCI = cast<AtomicMemCpyInst>(CI);
291   AMCI->setDestAlignment(DstAlign);
292   AMCI->setSourceAlignment(SrcAlign);
293 
294   // Set the TBAA info if present.
295   if (TBAATag)
296     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
297 
298   // Set the TBAA Struct info if present.
299   if (TBAAStructTag)
300     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
301 
302   if (ScopeTag)
303     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
304 
305   if (NoAliasTag)
306     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
307 
308   return CI;
309 }
310 
311 CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign,
312                                        Value *Src, MaybeAlign SrcAlign,
313                                        Value *Size, bool isVolatile,
314                                        MDNode *TBAATag, MDNode *ScopeTag,
315                                        MDNode *NoAliasTag) {
316   Dst = getCastedInt8PtrValue(Dst);
317   Src = getCastedInt8PtrValue(Src);
318 
319   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
320   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
321   Module *M = BB->getParent()->getParent();
322   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
323 
324   CallInst *CI = createCallHelper(TheFn, Ops, this);
325 
326   auto *MMI = cast<MemMoveInst>(CI);
327   if (DstAlign)
328     MMI->setDestAlignment(*DstAlign);
329   if (SrcAlign)
330     MMI->setSourceAlignment(*SrcAlign);
331 
332   // Set the TBAA info if present.
333   if (TBAATag)
334     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
335 
336   if (ScopeTag)
337     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
338 
339   if (NoAliasTag)
340     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
341 
342   return CI;
343 }
344 
345 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
346     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
347     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
348     MDNode *ScopeTag, MDNode *NoAliasTag) {
349   assert(DstAlign >= ElementSize &&
350          "Pointer alignment must be at least element size");
351   assert(SrcAlign >= ElementSize &&
352          "Pointer alignment must be at least element size");
353   Dst = getCastedInt8PtrValue(Dst);
354   Src = getCastedInt8PtrValue(Src);
355 
356   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
357   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
358   Module *M = BB->getParent()->getParent();
359   Function *TheFn = Intrinsic::getDeclaration(
360       M, Intrinsic::memmove_element_unordered_atomic, Tys);
361 
362   CallInst *CI = createCallHelper(TheFn, Ops, this);
363 
364   // Set the alignment of the pointer args.
365   CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
366   CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
367 
368   // Set the TBAA info if present.
369   if (TBAATag)
370     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
371 
372   // Set the TBAA Struct info if present.
373   if (TBAAStructTag)
374     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
375 
376   if (ScopeTag)
377     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
378 
379   if (NoAliasTag)
380     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
381 
382   return CI;
383 }
384 
385 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
386                                     Value *Src) {
387   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
388   Value *Ops[] = {Src};
389   Type *Tys[] = { Src->getType() };
390   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
391   return createCallHelper(Decl, Ops, Builder);
392 }
393 
394 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
395   Module *M = GetInsertBlock()->getParent()->getParent();
396   Value *Ops[] = {Acc, Src};
397   auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd,
398                                         {Src->getType()});
399   return createCallHelper(Decl, Ops, this);
400 }
401 
402 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
403   Module *M = GetInsertBlock()->getParent()->getParent();
404   Value *Ops[] = {Acc, Src};
405   auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul,
406                                         {Src->getType()});
407   return createCallHelper(Decl, Ops, this);
408 }
409 
410 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
411   return getReductionIntrinsic(this, Intrinsic::vector_reduce_add, Src);
412 }
413 
414 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
415   return getReductionIntrinsic(this, Intrinsic::vector_reduce_mul, Src);
416 }
417 
418 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
419   return getReductionIntrinsic(this, Intrinsic::vector_reduce_and, Src);
420 }
421 
422 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
423   return getReductionIntrinsic(this, Intrinsic::vector_reduce_or, Src);
424 }
425 
426 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
427   return getReductionIntrinsic(this, Intrinsic::vector_reduce_xor, Src);
428 }
429 
430 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
431   auto ID =
432       IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
433   return getReductionIntrinsic(this, ID, Src);
434 }
435 
436 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
437   auto ID =
438       IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
439   return getReductionIntrinsic(this, ID, Src);
440 }
441 
442 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
443   return getReductionIntrinsic(this, Intrinsic::vector_reduce_fmax, Src);
444 }
445 
446 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src) {
447   return getReductionIntrinsic(this, Intrinsic::vector_reduce_fmin, Src);
448 }
449 
450 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
451   assert(isa<PointerType>(Ptr->getType()) &&
452          "lifetime.start only applies to pointers.");
453   Ptr = getCastedInt8PtrValue(Ptr);
454   if (!Size)
455     Size = getInt64(-1);
456   else
457     assert(Size->getType() == getInt64Ty() &&
458            "lifetime.start requires the size to be an i64");
459   Value *Ops[] = { Size, Ptr };
460   Module *M = BB->getParent()->getParent();
461   Function *TheFn =
462       Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
463   return createCallHelper(TheFn, Ops, this);
464 }
465 
466 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
467   assert(isa<PointerType>(Ptr->getType()) &&
468          "lifetime.end only applies to pointers.");
469   Ptr = getCastedInt8PtrValue(Ptr);
470   if (!Size)
471     Size = getInt64(-1);
472   else
473     assert(Size->getType() == getInt64Ty() &&
474            "lifetime.end requires the size to be an i64");
475   Value *Ops[] = { Size, Ptr };
476   Module *M = BB->getParent()->getParent();
477   Function *TheFn =
478       Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
479   return createCallHelper(TheFn, Ops, this);
480 }
481 
482 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
483 
484   assert(isa<PointerType>(Ptr->getType()) &&
485          "invariant.start only applies to pointers.");
486   Ptr = getCastedInt8PtrValue(Ptr);
487   if (!Size)
488     Size = getInt64(-1);
489   else
490     assert(Size->getType() == getInt64Ty() &&
491            "invariant.start requires the size to be an i64");
492 
493   Value *Ops[] = {Size, Ptr};
494   // Fill in the single overloaded type: memory object type.
495   Type *ObjectPtr[1] = {Ptr->getType()};
496   Module *M = BB->getParent()->getParent();
497   Function *TheFn =
498       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
499   return createCallHelper(TheFn, Ops, this);
500 }
501 
502 CallInst *
503 IRBuilderBase::CreateAssumption(Value *Cond,
504                                 ArrayRef<OperandBundleDef> OpBundles) {
505   assert(Cond->getType() == getInt1Ty() &&
506          "an assumption condition must be of type i1");
507 
508   Value *Ops[] = { Cond };
509   Module *M = BB->getParent()->getParent();
510   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
511   return createCallHelper(FnAssume, Ops, this, "", nullptr, OpBundles);
512 }
513 
514 Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
515   Module *M = BB->getModule();
516   auto *FnIntrinsic = Intrinsic::getDeclaration(
517       M, Intrinsic::experimental_noalias_scope_decl, {});
518   return createCallHelper(FnIntrinsic, {Scope}, this);
519 }
520 
521 /// Create a call to a Masked Load intrinsic.
522 /// \p Ty        - vector type to load
523 /// \p Ptr       - base pointer for the load
524 /// \p Alignment - alignment of the source location
525 /// \p Mask      - vector of booleans which indicates what vector lanes should
526 ///                be accessed in memory
527 /// \p PassThru  - pass-through value that is used to fill the masked-off lanes
528 ///                of the result
529 /// \p Name      - name of the result variable
530 CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
531                                           Value *Mask, Value *PassThru,
532                                           const Twine &Name) {
533   auto *PtrTy = cast<PointerType>(Ptr->getType());
534   assert(Ty->isVectorTy() && "Type should be vector");
535   assert(PtrTy->isOpaqueOrPointeeTypeMatches(Ty) && "Wrong element type");
536   assert(Mask && "Mask should not be all-ones (null)");
537   if (!PassThru)
538     PassThru = UndefValue::get(Ty);
539   Type *OverloadedTypes[] = { Ty, PtrTy };
540   Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
541   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
542                                OverloadedTypes, Name);
543 }
544 
545 /// Create a call to a Masked Store intrinsic.
546 /// \p Val       - data to be stored,
547 /// \p Ptr       - base pointer for the store
548 /// \p Alignment - alignment of the destination location
549 /// \p Mask      - vector of booleans which indicates what vector lanes should
550 ///                be accessed in memory
551 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
552                                            Align Alignment, Value *Mask) {
553   auto *PtrTy = cast<PointerType>(Ptr->getType());
554   Type *DataTy = Val->getType();
555   assert(DataTy->isVectorTy() && "Val should be a vector");
556   assert(PtrTy->isOpaqueOrPointeeTypeMatches(DataTy) && "Wrong element type");
557   assert(Mask && "Mask should not be all-ones (null)");
558   Type *OverloadedTypes[] = { DataTy, PtrTy };
559   Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
560   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
561 }
562 
563 /// Create a call to a Masked intrinsic, with given intrinsic Id,
564 /// an array of operands - Ops, and an array of overloaded types -
565 /// OverloadedTypes.
566 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
567                                                ArrayRef<Value *> Ops,
568                                                ArrayRef<Type *> OverloadedTypes,
569                                                const Twine &Name) {
570   Module *M = BB->getParent()->getParent();
571   Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
572   return createCallHelper(TheFn, Ops, this, Name);
573 }
574 
575 /// Create a call to a Masked Gather intrinsic.
576 /// \p Ty       - vector type to gather
577 /// \p Ptrs     - vector of pointers for loading
578 /// \p Align    - alignment for one element
579 /// \p Mask     - vector of booleans which indicates what vector lanes should
580 ///               be accessed in memory
581 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
582 ///               of the result
583 /// \p Name     - name of the result variable
584 CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
585                                             Align Alignment, Value *Mask,
586                                             Value *PassThru,
587                                             const Twine &Name) {
588   auto *VecTy = cast<VectorType>(Ty);
589   ElementCount NumElts = VecTy->getElementCount();
590   auto *PtrsTy = cast<VectorType>(Ptrs->getType());
591   assert(cast<PointerType>(PtrsTy->getElementType())
592              ->isOpaqueOrPointeeTypeMatches(
593                  cast<VectorType>(Ty)->getElementType()) &&
594          "Element type mismatch");
595   assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
596 
597   if (!Mask)
598     Mask = Constant::getAllOnesValue(
599         VectorType::get(Type::getInt1Ty(Context), NumElts));
600 
601   if (!PassThru)
602     PassThru = UndefValue::get(Ty);
603 
604   Type *OverloadedTypes[] = {Ty, PtrsTy};
605   Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
606 
607   // We specify only one type when we create this intrinsic. Types of other
608   // arguments are derived from this type.
609   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
610                                Name);
611 }
612 
613 /// Create a call to a Masked Scatter intrinsic.
614 /// \p Data  - data to be stored,
615 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
616 ///            stored
617 /// \p Align - alignment for one element
618 /// \p Mask  - vector of booleans which indicates what vector lanes should
619 ///            be accessed in memory
620 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
621                                              Align Alignment, Value *Mask) {
622   auto *PtrsTy = cast<VectorType>(Ptrs->getType());
623   auto *DataTy = cast<VectorType>(Data->getType());
624   ElementCount NumElts = PtrsTy->getElementCount();
625 
626 #ifndef NDEBUG
627   auto *PtrTy = cast<PointerType>(PtrsTy->getElementType());
628   assert(NumElts == DataTy->getElementCount() &&
629          PtrTy->isOpaqueOrPointeeTypeMatches(DataTy->getElementType()) &&
630          "Incompatible pointer and data types");
631 #endif
632 
633   if (!Mask)
634     Mask = Constant::getAllOnesValue(
635         VectorType::get(Type::getInt1Ty(Context), NumElts));
636 
637   Type *OverloadedTypes[] = {DataTy, PtrsTy};
638   Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
639 
640   // We specify only one type when we create this intrinsic. Types of other
641   // arguments are derived from this type.
642   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
643 }
644 
645 template <typename T0>
646 static std::vector<Value *>
647 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
648                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
649   std::vector<Value *> Args;
650   Args.push_back(B.getInt64(ID));
651   Args.push_back(B.getInt32(NumPatchBytes));
652   Args.push_back(ActualCallee);
653   Args.push_back(B.getInt32(CallArgs.size()));
654   Args.push_back(B.getInt32(Flags));
655   llvm::append_range(Args, CallArgs);
656   // GC Transition and Deopt args are now always handled via operand bundle.
657   // They will be removed from the signature of gc.statepoint shortly.
658   Args.push_back(B.getInt32(0));
659   Args.push_back(B.getInt32(0));
660   // GC args are now encoded in the gc-live operand bundle
661   return Args;
662 }
663 
664 template<typename T1, typename T2, typename T3>
665 static std::vector<OperandBundleDef>
666 getStatepointBundles(Optional<ArrayRef<T1>> TransitionArgs,
667                      Optional<ArrayRef<T2>> DeoptArgs,
668                      ArrayRef<T3> GCArgs) {
669   std::vector<OperandBundleDef> Rval;
670   if (DeoptArgs) {
671     SmallVector<Value*, 16> DeoptValues;
672     llvm::append_range(DeoptValues, *DeoptArgs);
673     Rval.emplace_back("deopt", DeoptValues);
674   }
675   if (TransitionArgs) {
676     SmallVector<Value*, 16> TransitionValues;
677     llvm::append_range(TransitionValues, *TransitionArgs);
678     Rval.emplace_back("gc-transition", TransitionValues);
679   }
680   if (GCArgs.size()) {
681     SmallVector<Value*, 16> LiveValues;
682     llvm::append_range(LiveValues, GCArgs);
683     Rval.emplace_back("gc-live", LiveValues);
684   }
685   return Rval;
686 }
687 
688 template <typename T0, typename T1, typename T2, typename T3>
689 static CallInst *CreateGCStatepointCallCommon(
690     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
691     FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
692     Optional<ArrayRef<T1>> TransitionArgs, Optional<ArrayRef<T2>> DeoptArgs,
693     ArrayRef<T3> GCArgs, const Twine &Name) {
694   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
695   // Fill in the one generic type'd argument (the function is also vararg)
696   Function *FnStatepoint =
697       Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
698                                 {ActualCallee.getCallee()->getType()});
699 
700   std::vector<Value *> Args = getStatepointArgs(
701       *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);
702 
703   CallInst *CI = Builder->CreateCall(
704       FnStatepoint, Args,
705       getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
706   CI->addParamAttr(2,
707                    Attribute::get(Builder->getContext(), Attribute::ElementType,
708                                   ActualCallee.getFunctionType()));
709   return CI;
710 }
711 
712 CallInst *IRBuilderBase::CreateGCStatepointCall(
713     uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
714     ArrayRef<Value *> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs,
715     ArrayRef<Value *> GCArgs, const Twine &Name) {
716   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
717       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
718       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
719 }
720 
721 CallInst *IRBuilderBase::CreateGCStatepointCall(
722     uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
723     uint32_t Flags, ArrayRef<Value *> CallArgs,
724     Optional<ArrayRef<Use>> TransitionArgs, Optional<ArrayRef<Use>> DeoptArgs,
725     ArrayRef<Value *> GCArgs, const Twine &Name) {
726   return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
727       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
728       DeoptArgs, GCArgs, Name);
729 }
730 
731 CallInst *IRBuilderBase::CreateGCStatepointCall(
732     uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
733     ArrayRef<Use> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs,
734     ArrayRef<Value *> GCArgs, const Twine &Name) {
735   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
736       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
737       CallArgs, None, DeoptArgs, GCArgs, Name);
738 }
739 
740 template <typename T0, typename T1, typename T2, typename T3>
741 static InvokeInst *CreateGCStatepointInvokeCommon(
742     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
743     FunctionCallee ActualInvokee, BasicBlock *NormalDest,
744     BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs,
745     Optional<ArrayRef<T1>> TransitionArgs, Optional<ArrayRef<T2>> DeoptArgs,
746     ArrayRef<T3> GCArgs, const Twine &Name) {
747   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
748   // Fill in the one generic type'd argument (the function is also vararg)
749   Function *FnStatepoint =
750       Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
751                                 {ActualInvokee.getCallee()->getType()});
752 
753   std::vector<Value *> Args =
754       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(),
755                         Flags, InvokeArgs);
756 
757   InvokeInst *II = Builder->CreateInvoke(
758       FnStatepoint, NormalDest, UnwindDest, Args,
759       getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
760   II->addParamAttr(2,
761                    Attribute::get(Builder->getContext(), Attribute::ElementType,
762                                   ActualInvokee.getFunctionType()));
763   return II;
764 }
765 
766 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
767     uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
768     BasicBlock *NormalDest, BasicBlock *UnwindDest,
769     ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Value *>> DeoptArgs,
770     ArrayRef<Value *> GCArgs, const Twine &Name) {
771   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
772       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
773       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
774       DeoptArgs, GCArgs, Name);
775 }
776 
777 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
778     uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
779     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
780     ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
781     Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
782     const Twine &Name) {
783   return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
784       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
785       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
786 }
787 
788 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
789     uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
790     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
791     Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs,
792     const Twine &Name) {
793   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
794       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
795       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
796       Name);
797 }
798 
799 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
800                                         Type *ResultType, const Twine &Name) {
801   Intrinsic::ID ID = Intrinsic::experimental_gc_result;
802   Module *M = BB->getParent()->getParent();
803   Type *Types[] = {ResultType};
804   Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
805 
806   Value *Args[] = {Statepoint};
807   return createCallHelper(FnGCResult, Args, this, Name);
808 }
809 
810 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
811                                           int BaseOffset, int DerivedOffset,
812                                           Type *ResultType, const Twine &Name) {
813   Module *M = BB->getParent()->getParent();
814   Type *Types[] = {ResultType};
815   Function *FnGCRelocate =
816       Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
817 
818   Value *Args[] = {Statepoint, getInt32(BaseOffset), getInt32(DerivedOffset)};
819   return createCallHelper(FnGCRelocate, Args, this, Name);
820 }
821 
822 CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,
823                                                 const Twine &Name) {
824   Module *M = BB->getParent()->getParent();
825   Type *PtrTy = DerivedPtr->getType();
826   Function *FnGCFindBase = Intrinsic::getDeclaration(
827       M, Intrinsic::experimental_gc_get_pointer_base, {PtrTy, PtrTy});
828   return createCallHelper(FnGCFindBase, {DerivedPtr}, this, Name);
829 }
830 
831 CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,
832                                                   const Twine &Name) {
833   Module *M = BB->getParent()->getParent();
834   Type *PtrTy = DerivedPtr->getType();
835   Function *FnGCGetOffset = Intrinsic::getDeclaration(
836       M, Intrinsic::experimental_gc_get_pointer_offset, {PtrTy});
837   return createCallHelper(FnGCGetOffset, {DerivedPtr}, this, Name);
838 }
839 
840 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
841                                               Instruction *FMFSource,
842                                               const Twine &Name) {
843   Module *M = BB->getModule();
844   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
845   return createCallHelper(Fn, {V}, this, Name, FMFSource);
846 }
847 
848 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
849                                                Value *RHS,
850                                                Instruction *FMFSource,
851                                                const Twine &Name) {
852   Module *M = BB->getModule();
853   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
854   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
855 }
856 
857 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
858                                          ArrayRef<Type *> Types,
859                                          ArrayRef<Value *> Args,
860                                          Instruction *FMFSource,
861                                          const Twine &Name) {
862   Module *M = BB->getModule();
863   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
864   return createCallHelper(Fn, Args, this, Name, FMFSource);
865 }
866 
867 CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
868     Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
869     const Twine &Name, MDNode *FPMathTag,
870     Optional<RoundingMode> Rounding,
871     Optional<fp::ExceptionBehavior> Except) {
872   Value *RoundingV = getConstrainedFPRounding(Rounding);
873   Value *ExceptV = getConstrainedFPExcept(Except);
874 
875   FastMathFlags UseFMF = FMF;
876   if (FMFSource)
877     UseFMF = FMFSource->getFastMathFlags();
878 
879   CallInst *C = CreateIntrinsic(ID, {L->getType()},
880                                 {L, R, RoundingV, ExceptV}, nullptr, Name);
881   setConstrainedFPCallAttr(C);
882   setFPAttrs(C, FPMathTag, UseFMF);
883   return C;
884 }
885 
886 Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
887                                    const Twine &Name, MDNode *FPMathTag) {
888   if (Instruction::isBinaryOp(Opc)) {
889     assert(Ops.size() == 2 && "Invalid number of operands!");
890     return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),
891                        Ops[0], Ops[1], Name, FPMathTag);
892   }
893   if (Instruction::isUnaryOp(Opc)) {
894     assert(Ops.size() == 1 && "Invalid number of operands!");
895     return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),
896                       Ops[0], Name, FPMathTag);
897   }
898   llvm_unreachable("Unexpected opcode!");
899 }
900 
901 CallInst *IRBuilderBase::CreateConstrainedFPCast(
902     Intrinsic::ID ID, Value *V, Type *DestTy,
903     Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag,
904     Optional<RoundingMode> Rounding,
905     Optional<fp::ExceptionBehavior> Except) {
906   Value *ExceptV = getConstrainedFPExcept(Except);
907 
908   FastMathFlags UseFMF = FMF;
909   if (FMFSource)
910     UseFMF = FMFSource->getFastMathFlags();
911 
912   CallInst *C;
913   bool HasRoundingMD = false;
914   switch (ID) {
915   default:
916     break;
917 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
918   case Intrinsic::INTRINSIC:                                \
919     HasRoundingMD = ROUND_MODE;                             \
920     break;
921 #include "llvm/IR/ConstrainedOps.def"
922   }
923   if (HasRoundingMD) {
924     Value *RoundingV = getConstrainedFPRounding(Rounding);
925     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
926                         nullptr, Name);
927   } else
928     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
929                         Name);
930 
931   setConstrainedFPCallAttr(C);
932 
933   if (isa<FPMathOperator>(C))
934     setFPAttrs(C, FPMathTag, UseFMF);
935   return C;
936 }
937 
938 Value *IRBuilderBase::CreateFCmpHelper(
939     CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name,
940     MDNode *FPMathTag, bool IsSignaling) {
941   if (IsFPConstrained) {
942     auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
943                           : Intrinsic::experimental_constrained_fcmp;
944     return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);
945   }
946 
947   if (auto *LC = dyn_cast<Constant>(LHS))
948     if (auto *RC = dyn_cast<Constant>(RHS))
949       return Insert(Folder.CreateFCmp(P, LC, RC), Name);
950   return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name);
951 }
952 
953 CallInst *IRBuilderBase::CreateConstrainedFPCmp(
954     Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
955     const Twine &Name, Optional<fp::ExceptionBehavior> Except) {
956   Value *PredicateV = getConstrainedFPPredicate(P);
957   Value *ExceptV = getConstrainedFPExcept(Except);
958 
959   CallInst *C = CreateIntrinsic(ID, {L->getType()},
960                                 {L, R, PredicateV, ExceptV}, nullptr, Name);
961   setConstrainedFPCallAttr(C);
962   return C;
963 }
964 
965 CallInst *IRBuilderBase::CreateConstrainedFPCall(
966     Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
967     Optional<RoundingMode> Rounding,
968     Optional<fp::ExceptionBehavior> Except) {
969   llvm::SmallVector<Value *, 6> UseArgs;
970 
971   append_range(UseArgs, Args);
972   bool HasRoundingMD = false;
973   switch (Callee->getIntrinsicID()) {
974   default:
975     break;
976 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
977   case Intrinsic::INTRINSIC:                                \
978     HasRoundingMD = ROUND_MODE;                             \
979     break;
980 #include "llvm/IR/ConstrainedOps.def"
981   }
982   if (HasRoundingMD)
983     UseArgs.push_back(getConstrainedFPRounding(Rounding));
984   UseArgs.push_back(getConstrainedFPExcept(Except));
985 
986   CallInst *C = CreateCall(Callee, UseArgs, Name);
987   setConstrainedFPCallAttr(C);
988   return C;
989 }
990 
991 Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
992                                    const Twine &Name, Instruction *MDFrom) {
993   if (auto *V = Folder.FoldSelect(C, True, False))
994     return V;
995 
996   SelectInst *Sel = SelectInst::Create(C, True, False);
997   if (MDFrom) {
998     MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);
999     MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);
1000     Sel = addBranchMetadata(Sel, Prof, Unpred);
1001   }
1002   if (isa<FPMathOperator>(Sel))
1003     setFPAttrs(Sel, nullptr /* MDNode* */, FMF);
1004   return Insert(Sel, Name);
1005 }
1006 
1007 Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,
1008                                     const Twine &Name) {
1009   assert(LHS->getType() == RHS->getType() &&
1010          "Pointer subtraction operand types must match!");
1011   assert(cast<PointerType>(LHS->getType())
1012              ->isOpaqueOrPointeeTypeMatches(ElemTy) &&
1013          "Pointer type must match element type");
1014   Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context));
1015   Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context));
1016   Value *Difference = CreateSub(LHS_int, RHS_int);
1017   return CreateExactSDiv(Difference, ConstantExpr::getSizeOf(ElemTy),
1018                          Name);
1019 }
1020 
1021 Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
1022   assert(isa<PointerType>(Ptr->getType()) &&
1023          "launder.invariant.group only applies to pointers.");
1024   // FIXME: we could potentially avoid casts to/from i8*.
1025   auto *PtrType = Ptr->getType();
1026   auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
1027   if (PtrType != Int8PtrTy)
1028     Ptr = CreateBitCast(Ptr, Int8PtrTy);
1029   Module *M = BB->getParent()->getParent();
1030   Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration(
1031       M, Intrinsic::launder_invariant_group, {Int8PtrTy});
1032 
1033   assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy &&
1034          FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
1035              Int8PtrTy &&
1036          "LaunderInvariantGroup should take and return the same type");
1037 
1038   CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr});
1039 
1040   if (PtrType != Int8PtrTy)
1041     return CreateBitCast(Fn, PtrType);
1042   return Fn;
1043 }
1044 
1045 Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
1046   assert(isa<PointerType>(Ptr->getType()) &&
1047          "strip.invariant.group only applies to pointers.");
1048 
1049   // FIXME: we could potentially avoid casts to/from i8*.
1050   auto *PtrType = Ptr->getType();
1051   auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
1052   if (PtrType != Int8PtrTy)
1053     Ptr = CreateBitCast(Ptr, Int8PtrTy);
1054   Module *M = BB->getParent()->getParent();
1055   Function *FnStripInvariantGroup = Intrinsic::getDeclaration(
1056       M, Intrinsic::strip_invariant_group, {Int8PtrTy});
1057 
1058   assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy &&
1059          FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
1060              Int8PtrTy &&
1061          "StripInvariantGroup should take and return the same type");
1062 
1063   CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr});
1064 
1065   if (PtrType != Int8PtrTy)
1066     return CreateBitCast(Fn, PtrType);
1067   return Fn;
1068 }
1069 
1070 Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {
1071   auto *Ty = cast<VectorType>(V->getType());
1072   if (isa<ScalableVectorType>(Ty)) {
1073     Module *M = BB->getParent()->getParent();
1074     Function *F = Intrinsic::getDeclaration(
1075         M, Intrinsic::experimental_vector_reverse, Ty);
1076     return Insert(CallInst::Create(F, V), Name);
1077   }
1078   // Keep the original behaviour for fixed vector
1079   SmallVector<int, 8> ShuffleMask;
1080   int NumElts = Ty->getElementCount().getKnownMinValue();
1081   for (int i = 0; i < NumElts; ++i)
1082     ShuffleMask.push_back(NumElts - i - 1);
1083   return CreateShuffleVector(V, ShuffleMask, Name);
1084 }
1085 
1086 Value *IRBuilderBase::CreateVectorSplice(Value *V1, Value *V2, int64_t Imm,
1087                                          const Twine &Name) {
1088   assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1089   assert(V1->getType() == V2->getType() &&
1090          "Splice expects matching operand types!");
1091 
1092   if (auto *VTy = dyn_cast<ScalableVectorType>(V1->getType())) {
1093     Module *M = BB->getParent()->getParent();
1094     Function *F = Intrinsic::getDeclaration(
1095         M, Intrinsic::experimental_vector_splice, VTy);
1096 
1097     Value *Ops[] = {V1, V2, getInt32(Imm)};
1098     return Insert(CallInst::Create(F, Ops), Name);
1099   }
1100 
1101   unsigned NumElts = cast<FixedVectorType>(V1->getType())->getNumElements();
1102   assert(((-Imm <= NumElts) || (Imm < NumElts)) &&
1103          "Invalid immediate for vector splice!");
1104 
1105   // Keep the original behaviour for fixed vector
1106   unsigned Idx = (NumElts + Imm) % NumElts;
1107   SmallVector<int, 8> Mask;
1108   for (unsigned I = 0; I < NumElts; ++I)
1109     Mask.push_back(Idx + I);
1110 
1111   return CreateShuffleVector(V1, V2, Mask);
1112 }
1113 
1114 Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
1115                                         const Twine &Name) {
1116   auto EC = ElementCount::getFixed(NumElts);
1117   return CreateVectorSplat(EC, V, Name);
1118 }
1119 
1120 Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
1121                                         const Twine &Name) {
1122   assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1123 
1124   // First insert it into a poison vector so we can shuffle it.
1125   Type *I32Ty = getInt32Ty();
1126   Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
1127   V = CreateInsertElement(Poison, V, ConstantInt::get(I32Ty, 0),
1128                           Name + ".splatinsert");
1129 
1130   // Shuffle the value across the desired number of elements.
1131   SmallVector<int, 16> Zeros;
1132   Zeros.resize(EC.getKnownMinValue());
1133   return CreateShuffleVector(V, Zeros, Name + ".splat");
1134 }
1135 
1136 Value *IRBuilderBase::CreateExtractInteger(
1137     const DataLayout &DL, Value *From, IntegerType *ExtractedTy,
1138     uint64_t Offset, const Twine &Name) {
1139   auto *IntTy = cast<IntegerType>(From->getType());
1140   assert(DL.getTypeStoreSize(ExtractedTy) + Offset <=
1141              DL.getTypeStoreSize(IntTy) &&
1142          "Element extends past full value");
1143   uint64_t ShAmt = 8 * Offset;
1144   Value *V = From;
1145   if (DL.isBigEndian())
1146     ShAmt = 8 * (DL.getTypeStoreSize(IntTy) -
1147                  DL.getTypeStoreSize(ExtractedTy) - Offset);
1148   if (ShAmt) {
1149     V = CreateLShr(V, ShAmt, Name + ".shift");
1150   }
1151   assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() &&
1152          "Cannot extract to a larger integer!");
1153   if (ExtractedTy != IntTy) {
1154     V = CreateTrunc(V, ExtractedTy, Name + ".trunc");
1155   }
1156   return V;
1157 }
1158 
1159 Value *IRBuilderBase::CreatePreserveArrayAccessIndex(
1160     Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex,
1161     MDNode *DbgInfo) {
1162   auto *BaseType = Base->getType();
1163   assert(isa<PointerType>(BaseType) &&
1164          "Invalid Base ptr type for preserve.array.access.index.");
1165   assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) &&
1166          "Pointer element type mismatch");
1167 
1168   Value *LastIndexV = getInt32(LastIndex);
1169   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1170   SmallVector<Value *, 4> IdxList(Dimension, Zero);
1171   IdxList.push_back(LastIndexV);
1172 
1173   Type *ResultType =
1174       GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList);
1175 
1176   Module *M = BB->getParent()->getParent();
1177   Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration(
1178       M, Intrinsic::preserve_array_access_index, {ResultType, BaseType});
1179 
1180   Value *DimV = getInt32(Dimension);
1181   CallInst *Fn =
1182       CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV});
1183   Fn->addParamAttr(
1184       0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
1185   if (DbgInfo)
1186     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1187 
1188   return Fn;
1189 }
1190 
1191 Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
1192     Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1193   assert(isa<PointerType>(Base->getType()) &&
1194          "Invalid Base ptr type for preserve.union.access.index.");
1195   auto *BaseType = Base->getType();
1196 
1197   Module *M = BB->getParent()->getParent();
1198   Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration(
1199       M, Intrinsic::preserve_union_access_index, {BaseType, BaseType});
1200 
1201   Value *DIIndex = getInt32(FieldIndex);
1202   CallInst *Fn =
1203       CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex});
1204   if (DbgInfo)
1205     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1206 
1207   return Fn;
1208 }
1209 
1210 Value *IRBuilderBase::CreatePreserveStructAccessIndex(
1211     Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1212     MDNode *DbgInfo) {
1213   auto *BaseType = Base->getType();
1214   assert(isa<PointerType>(BaseType) &&
1215          "Invalid Base ptr type for preserve.struct.access.index.");
1216   assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) &&
1217          "Pointer element type mismatch");
1218 
1219   Value *GEPIndex = getInt32(Index);
1220   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1221   Type *ResultType =
1222       GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex});
1223 
1224   Module *M = BB->getParent()->getParent();
1225   Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration(
1226       M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType});
1227 
1228   Value *DIIndex = getInt32(FieldIndex);
1229   CallInst *Fn = CreateCall(FnPreserveStructAccessIndex,
1230                             {Base, GEPIndex, DIIndex});
1231   Fn->addParamAttr(
1232       0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
1233   if (DbgInfo)
1234     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1235 
1236   return Fn;
1237 }
1238 
1239 CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1240                                                          Value *PtrValue,
1241                                                          Value *AlignValue,
1242                                                          Value *OffsetValue) {
1243   SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1244   if (OffsetValue)
1245     Vals.push_back(OffsetValue);
1246   OperandBundleDefT<Value *> AlignOpB("align", Vals);
1247   return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB});
1248 }
1249 
1250 CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1251                                                    Value *PtrValue,
1252                                                    unsigned Alignment,
1253                                                    Value *OffsetValue) {
1254   assert(isa<PointerType>(PtrValue->getType()) &&
1255          "trying to create an alignment assumption on a non-pointer?");
1256   assert(Alignment != 0 && "Invalid Alignment");
1257   auto *PtrTy = cast<PointerType>(PtrValue->getType());
1258   Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
1259   Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment);
1260   return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1261 }
1262 
1263 CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1264                                                    Value *PtrValue,
1265                                                    Value *Alignment,
1266                                                    Value *OffsetValue) {
1267   assert(isa<PointerType>(PtrValue->getType()) &&
1268          "trying to create an alignment assumption on a non-pointer?");
1269   return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
1270 }
1271 
1272 IRBuilderDefaultInserter::~IRBuilderDefaultInserter() = default;
1273 IRBuilderCallbackInserter::~IRBuilderCallbackInserter() = default;
1274 IRBuilderFolder::~IRBuilderFolder() = default;
1275 void ConstantFolder::anchor() {}
1276 void NoFolder::anchor() {}
1277