1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the auto-upgrade helper functions
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/IR/AutoUpgrade.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/CallSite.h"
17 #include "llvm/IR/Constants.h"
18 #include "llvm/IR/DIBuilder.h"
19 #include "llvm/IR/DebugInfo.h"
20 #include "llvm/IR/DiagnosticInfo.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/IRBuilder.h"
23 #include "llvm/IR/Instruction.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/Support/ErrorHandling.h"
28 #include <cstring>
29 using namespace llvm;
30 
31 // Upgrade the declarations of the SSE4.1 functions whose arguments have
32 // changed their type from v4f32 to v2i64.
33 static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID,
34                                  Function *&NewFn) {
35   // Check whether this is an old version of the function, which received
36   // v4f32 arguments.
37   Type *Arg0Type = F->getFunctionType()->getParamType(0);
38   if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4))
39     return false;
40 
41   // Yes, it's old, replace it with new version.
42   F->setName(F->getName() + ".old");
43   NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
44   return true;
45 }
46 
47 // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
48 // arguments have changed their type from i32 to i8.
49 static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
50                                              Function *&NewFn) {
51   // Check that the last argument is an i32.
52   Type *LastArgType = F->getFunctionType()->getParamType(
53      F->getFunctionType()->getNumParams() - 1);
54   if (!LastArgType->isIntegerTy(32))
55     return false;
56 
57   // Move this function aside and map down.
58   F->setName(F->getName() + ".old");
59   NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
60   return true;
61 }
62 
63 // Upgrade the declarations of AVX-512 cmp intrinsic functions whose 8-bit
64 // immediates have changed their type from i32 to i8.
65 static bool UpgradeAVX512CmpIntrinsic(Function *F, Intrinsic::ID IID,
66                                       Function *&NewFn) {
67   // Check that the last argument is an i32.
68   Type *LastArgType = F->getFunctionType()->getParamType(2);
69   if (!LastArgType->isIntegerTy(32))
70     return false;
71 
72   // Move this function aside and map down.
73   F->setName(F->getName() + ".old");
74   NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
75   return true;
76 }
77 
78 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
79   assert(F && "Illegal to upgrade a non-existent Function.");
80 
81   // Quickly eliminate it, if it's not a candidate.
82   StringRef Name = F->getName();
83   if (Name.size() <= 8 || !Name.startswith("llvm."))
84     return false;
85   Name = Name.substr(5); // Strip off "llvm."
86 
87   switch (Name[0]) {
88   default: break;
89   case 'a': {
90     if (Name.startswith("arm.neon.vclz")) {
91       Type* args[2] = {
92         F->arg_begin()->getType(),
93         Type::getInt1Ty(F->getContext())
94       };
95       // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
96       // the end of the name. Change name from llvm.arm.neon.vclz.* to
97       //  llvm.ctlz.*
98       FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
99       NewFn = Function::Create(fType, F->getLinkage(),
100                                "llvm.ctlz." + Name.substr(14), F->getParent());
101       return true;
102     }
103     if (Name.startswith("arm.neon.vcnt")) {
104       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
105                                         F->arg_begin()->getType());
106       return true;
107     }
108     break;
109   }
110   case 'c': {
111     if (Name.startswith("ctlz.") && F->arg_size() == 1) {
112       F->setName(Name + ".old");
113       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
114                                         F->arg_begin()->getType());
115       return true;
116     }
117     if (Name.startswith("cttz.") && F->arg_size() == 1) {
118       F->setName(Name + ".old");
119       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
120                                         F->arg_begin()->getType());
121       return true;
122     }
123     break;
124   }
125   case 'd': {
126     if (Name.startswith("dbg.declare") && F->arg_size() == 2) {
127       F->setName(Name + ".old");
128       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_declare);
129       return true;
130     }
131     if (Name.startswith("dbg.value") && F->arg_size() == 3) {
132       F->setName(Name + ".old");
133       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
134       return true;
135     }
136     break;
137   }
138 
139   case 'o':
140     // We only need to change the name to match the mangling including the
141     // address space.
142     if (F->arg_size() == 2 && Name.startswith("objectsize.")) {
143       Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
144       if (F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
145         F->setName(Name + ".old");
146         NewFn = Intrinsic::getDeclaration(F->getParent(),
147                                           Intrinsic::objectsize, Tys);
148         return true;
149       }
150     }
151     break;
152 
153   case 'x': {
154     if (Name.startswith("x86.sse2.pcmpeq.") ||
155         Name.startswith("x86.sse2.pcmpgt.") ||
156         Name.startswith("x86.avx2.pcmpeq.") ||
157         Name.startswith("x86.avx2.pcmpgt.") ||
158         Name.startswith("x86.avx.vpermil.") ||
159         Name == "x86.avx.movnt.dq.256" ||
160         Name == "x86.avx.movnt.pd.256" ||
161         Name == "x86.avx.movnt.ps.256" ||
162         Name == "x86.sse42.crc32.64.8" ||
163         Name == "x86.avx.vbroadcast.ss" ||
164         Name == "x86.avx.vbroadcast.ss.256" ||
165         Name == "x86.avx.vbroadcast.sd.256" ||
166         Name == "x86.sse2.psll.dq" ||
167         Name == "x86.sse2.psrl.dq" ||
168         Name == "x86.avx2.psll.dq" ||
169         Name == "x86.avx2.psrl.dq" ||
170         Name == "x86.sse2.psll.dq.bs" ||
171         Name == "x86.sse2.psrl.dq.bs" ||
172         Name == "x86.avx2.psll.dq.bs" ||
173         Name == "x86.avx2.psrl.dq.bs" ||
174         Name == "x86.sse41.pblendw" ||
175         Name == "x86.sse41.blendpd" ||
176         Name == "x86.sse41.blendps" ||
177         Name == "x86.avx.blend.pd.256" ||
178         Name == "x86.avx.blend.ps.256" ||
179         Name == "x86.avx2.pblendw" ||
180         Name == "x86.avx2.pblendd.128" ||
181         Name == "x86.avx2.pblendd.256" ||
182         (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) {
183       NewFn = nullptr;
184       return true;
185     }
186     // SSE4.1 ptest functions may have an old signature.
187     if (Name.startswith("x86.sse41.ptest")) {
188       if (Name == "x86.sse41.ptestc")
189         return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn);
190       if (Name == "x86.sse41.ptestz")
191         return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn);
192       if (Name == "x86.sse41.ptestnzc")
193         return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
194     }
195     // Several blend and other instructions with masks used the wrong number of
196     // bits.
197     if (Name == "x86.sse41.insertps")
198       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
199                                               NewFn);
200     if (Name == "x86.sse41.dppd")
201       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
202                                               NewFn);
203     if (Name == "x86.sse41.dpps")
204       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
205                                               NewFn);
206     if (Name == "x86.sse41.mpsadbw")
207       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
208                                               NewFn);
209     if (Name == "x86.avx.dp.ps.256")
210       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
211                                               NewFn);
212     if (Name == "x86.avx2.mpsadbw")
213       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
214                                               NewFn);
215 
216     if (Name == "x86.avx512.mask.cmp.ps.512")
217       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
218                                        NewFn);
219     if (Name == "x86.avx512.mask.cmp.pd.512")
220       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
221                                        NewFn);
222 
223     if (Name == "x86.avx512.mask.cmp.b.512")
224       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_512,
225                                        NewFn);
226     if (Name == "x86.avx512.mask.cmp.w.512")
227       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_512,
228                                        NewFn);
229     if (Name == "x86.avx512.mask.cmp.d.512")
230       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_512,
231                                        NewFn);
232     if (Name == "x86.avx512.mask.cmp.q.512")
233       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_512,
234                                        NewFn);
235     if (Name == "x86.avx512.mask.ucmp.b.512")
236       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_512,
237                                        NewFn);
238     if (Name == "x86.avx512.mask.ucmp.w.512")
239       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_512,
240                                        NewFn);
241     if (Name == "x86.avx512.mask.ucmp.d.512")
242       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_512,
243                                        NewFn);
244     if (Name == "x86.avx512.mask.ucmp.q.512")
245       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_512,
246                                        NewFn);
247 
248     if (Name == "x86.avx512.mask.cmp.b.256")
249       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_256,
250                                        NewFn);
251     if (Name == "x86.avx512.mask.cmp.w.256")
252       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_256,
253                                        NewFn);
254     if (Name == "x86.avx512.mask.cmp.d.256")
255       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_256,
256                                        NewFn);
257     if (Name == "x86.avx512.mask.cmp.q.256")
258       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_256,
259                                        NewFn);
260     if (Name == "x86.avx512.mask.ucmp.b.256")
261       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_256,
262                                        NewFn);
263     if (Name == "x86.avx512.mask.ucmp.w.256")
264       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_256,
265                                        NewFn);
266     if (Name == "x86.avx512.mask.ucmp.d.256")
267       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_256,
268                                        NewFn);
269     if (Name == "x86.avx512.mask.ucmp.q.256")
270       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_256,
271                                        NewFn);
272 
273     if (Name == "x86.avx512.mask.cmp.b.128")
274       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_128,
275                                        NewFn);
276     if (Name == "x86.avx512.mask.cmp.w.128")
277       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_128,
278                                        NewFn);
279     if (Name == "x86.avx512.mask.cmp.d.128")
280       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_128,
281                                        NewFn);
282     if (Name == "x86.avx512.mask.cmp.q.128")
283       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_128,
284                                        NewFn);
285     if (Name == "x86.avx512.mask.ucmp.b.128")
286       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_128,
287                                        NewFn);
288     if (Name == "x86.avx512.mask.ucmp.w.128")
289       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_128,
290                                        NewFn);
291     if (Name == "x86.avx512.mask.ucmp.d.128")
292       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_128,
293                                        NewFn);
294     if (Name == "x86.avx512.mask.ucmp.q.128")
295       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_128,
296                                        NewFn);
297 
298     // frcz.ss/sd may need to have an argument dropped
299     if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) {
300       F->setName(Name + ".old");
301       NewFn = Intrinsic::getDeclaration(F->getParent(),
302                                         Intrinsic::x86_xop_vfrcz_ss);
303       return true;
304     }
305     if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) {
306       F->setName(Name + ".old");
307       NewFn = Intrinsic::getDeclaration(F->getParent(),
308                                         Intrinsic::x86_xop_vfrcz_sd);
309       return true;
310     }
311     // Fix the FMA4 intrinsics to remove the 4
312     if (Name.startswith("x86.fma4.")) {
313       F->setName("llvm.x86.fma" + Name.substr(8));
314       NewFn = F;
315       return true;
316     }
317     break;
318   }
319   }
320 
321   //  This may not belong here. This function is effectively being overloaded
322   //  to both detect an intrinsic which needs upgrading, and to provide the
323   //  upgraded form of the intrinsic. We should perhaps have two separate
324   //  functions for this.
325   return false;
326 }
327 
328 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
329   NewFn = nullptr;
330   bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
331 
332   // Upgrade intrinsic attributes.  This does not change the function.
333   if (NewFn)
334     F = NewFn;
335   if (unsigned id = F->getIntrinsicID())
336     F->setAttributes(Intrinsic::getAttributes(F->getContext(),
337                                               (Intrinsic::ID)id));
338   return Upgraded;
339 }
340 
341 bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
342   // Nothing to do yet.
343   return false;
344 }
345 
346 static MDNode *getNodeField(const MDNode *DbgNode, unsigned Elt) {
347   if (!DbgNode || Elt >= DbgNode->getNumOperands())
348     return nullptr;
349   return dyn_cast_or_null<MDNode>(DbgNode->getOperand(Elt));
350 }
351 
352 static MetadataAsValue *getExpression(Value *VarOperand, Function *F) {
353   // Old-style DIVariables have an optional expression as the 8th element.
354   DIExpression Expr(getNodeField(
355       cast<MDNode>(cast<MetadataAsValue>(VarOperand)->getMetadata()), 8));
356   if (!Expr) {
357     DIBuilder DIB(*F->getParent(), /*AllowUnresolved*/ false);
358     Expr = DIB.createExpression();
359   }
360   return MetadataAsValue::get(F->getContext(), Expr);
361 }
362 
363 // Handles upgrading SSE2 and AVX2 PSLLDQ intrinsics by converting them
364 // to byte shuffles.
365 static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C,
366                                          Value *Op, unsigned NumLanes,
367                                          unsigned Shift) {
368   // Each lane is 16 bytes.
369   unsigned NumElts = NumLanes * 16;
370 
371   // Bitcast from a 64-bit element type to a byte element type.
372   Op = Builder.CreateBitCast(Op,
373                              VectorType::get(Type::getInt8Ty(C), NumElts),
374                              "cast");
375   // We'll be shuffling in zeroes.
376   Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0));
377 
378   // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
379   // we'll just return the zero vector.
380   if (Shift < 16) {
381     SmallVector<Constant*, 32> Idxs;
382     // 256-bit version is split into two 16-byte lanes.
383     for (unsigned l = 0; l != NumElts; l += 16)
384       for (unsigned i = 0; i != 16; ++i) {
385         unsigned Idx = NumElts + i - Shift;
386         if (Idx < NumElts)
387           Idx -= NumElts - 16; // end of lane, switch operand.
388         Idxs.push_back(Builder.getInt32(Idx + l));
389       }
390 
391     Res = Builder.CreateShuffleVector(Res, Op, ConstantVector::get(Idxs));
392   }
393 
394   // Bitcast back to a 64-bit element type.
395   return Builder.CreateBitCast(Res,
396                                VectorType::get(Type::getInt64Ty(C), 2*NumLanes),
397                                "cast");
398 }
399 
400 // Handles upgrading SSE2 and AVX2 PSRLDQ intrinsics by converting them
401 // to byte shuffles.
402 static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C,
403                                          Value *Op, unsigned NumLanes,
404                                          unsigned Shift) {
405   // Each lane is 16 bytes.
406   unsigned NumElts = NumLanes * 16;
407 
408   // Bitcast from a 64-bit element type to a byte element type.
409   Op = Builder.CreateBitCast(Op,
410                              VectorType::get(Type::getInt8Ty(C), NumElts),
411                              "cast");
412   // We'll be shuffling in zeroes.
413   Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0));
414 
415   // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
416   // we'll just return the zero vector.
417   if (Shift < 16) {
418     SmallVector<Constant*, 32> Idxs;
419     // 256-bit version is split into two 16-byte lanes.
420     for (unsigned l = 0; l != NumElts; l += 16)
421       for (unsigned i = 0; i != 16; ++i) {
422         unsigned Idx = i + Shift;
423         if (Idx >= 16)
424           Idx += NumElts - 16; // end of lane, switch operand.
425         Idxs.push_back(Builder.getInt32(Idx + l));
426       }
427 
428     Res = Builder.CreateShuffleVector(Op, Res, ConstantVector::get(Idxs));
429   }
430 
431   // Bitcast back to a 64-bit element type.
432   return Builder.CreateBitCast(Res,
433                                VectorType::get(Type::getInt64Ty(C), 2*NumLanes),
434                                "cast");
435 }
436 
437 // UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the
438 // upgraded intrinsic. All argument and return casting must be provided in
439 // order to seamlessly integrate with existing context.
440 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
441   Function *F = CI->getCalledFunction();
442   LLVMContext &C = CI->getContext();
443   IRBuilder<> Builder(C);
444   Builder.SetInsertPoint(CI->getParent(), CI);
445 
446   assert(F && "Intrinsic call is not direct?");
447 
448   if (!NewFn) {
449     // Get the Function's name.
450     StringRef Name = F->getName();
451 
452     Value *Rep;
453     // Upgrade packed integer vector compares intrinsics to compare instructions
454     if (Name.startswith("llvm.x86.sse2.pcmpeq.") ||
455         Name.startswith("llvm.x86.avx2.pcmpeq.")) {
456       Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1),
457                                  "pcmpeq");
458       // need to sign extend since icmp returns vector of i1
459       Rep = Builder.CreateSExt(Rep, CI->getType(), "");
460     } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") ||
461                Name.startswith("llvm.x86.avx2.pcmpgt.")) {
462       Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1),
463                                   "pcmpgt");
464       // need to sign extend since icmp returns vector of i1
465       Rep = Builder.CreateSExt(Rep, CI->getType(), "");
466     } else if (Name == "llvm.x86.avx.movnt.dq.256" ||
467                Name == "llvm.x86.avx.movnt.ps.256" ||
468                Name == "llvm.x86.avx.movnt.pd.256") {
469       IRBuilder<> Builder(C);
470       Builder.SetInsertPoint(CI->getParent(), CI);
471 
472       Module *M = F->getParent();
473       SmallVector<Metadata *, 1> Elts;
474       Elts.push_back(
475           ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
476       MDNode *Node = MDNode::get(C, Elts);
477 
478       Value *Arg0 = CI->getArgOperand(0);
479       Value *Arg1 = CI->getArgOperand(1);
480 
481       // Convert the type of the pointer to a pointer to the stored type.
482       Value *BC = Builder.CreateBitCast(Arg0,
483                                         PointerType::getUnqual(Arg1->getType()),
484                                         "cast");
485       StoreInst *SI = Builder.CreateStore(Arg1, BC);
486       SI->setMetadata(M->getMDKindID("nontemporal"), Node);
487       SI->setAlignment(16);
488 
489       // Remove intrinsic.
490       CI->eraseFromParent();
491       return;
492     } else if (Name.startswith("llvm.x86.xop.vpcom")) {
493       Intrinsic::ID intID;
494       if (Name.endswith("ub"))
495         intID = Intrinsic::x86_xop_vpcomub;
496       else if (Name.endswith("uw"))
497         intID = Intrinsic::x86_xop_vpcomuw;
498       else if (Name.endswith("ud"))
499         intID = Intrinsic::x86_xop_vpcomud;
500       else if (Name.endswith("uq"))
501         intID = Intrinsic::x86_xop_vpcomuq;
502       else if (Name.endswith("b"))
503         intID = Intrinsic::x86_xop_vpcomb;
504       else if (Name.endswith("w"))
505         intID = Intrinsic::x86_xop_vpcomw;
506       else if (Name.endswith("d"))
507         intID = Intrinsic::x86_xop_vpcomd;
508       else if (Name.endswith("q"))
509         intID = Intrinsic::x86_xop_vpcomq;
510       else
511         llvm_unreachable("Unknown suffix");
512 
513       Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom"
514       unsigned Imm;
515       if (Name.startswith("lt"))
516         Imm = 0;
517       else if (Name.startswith("le"))
518         Imm = 1;
519       else if (Name.startswith("gt"))
520         Imm = 2;
521       else if (Name.startswith("ge"))
522         Imm = 3;
523       else if (Name.startswith("eq"))
524         Imm = 4;
525       else if (Name.startswith("ne"))
526         Imm = 5;
527       else if (Name.startswith("false"))
528         Imm = 6;
529       else if (Name.startswith("true"))
530         Imm = 7;
531       else
532         llvm_unreachable("Unknown condition");
533 
534       Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
535       Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0),
536                                 CI->getArgOperand(1), Builder.getInt8(Imm));
537     } else if (Name == "llvm.x86.sse42.crc32.64.8") {
538       Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
539                                                Intrinsic::x86_sse42_crc32_32_8);
540       Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
541       Rep = Builder.CreateCall2(CRC32, Trunc0, CI->getArgOperand(1));
542       Rep = Builder.CreateZExt(Rep, CI->getType(), "");
543     } else if (Name.startswith("llvm.x86.avx.vbroadcast")) {
544       // Replace broadcasts with a series of insertelements.
545       Type *VecTy = CI->getType();
546       Type *EltTy = VecTy->getVectorElementType();
547       unsigned EltNum = VecTy->getVectorNumElements();
548       Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
549                                           EltTy->getPointerTo());
550       Value *Load = Builder.CreateLoad(Cast);
551       Type *I32Ty = Type::getInt32Ty(C);
552       Rep = UndefValue::get(VecTy);
553       for (unsigned I = 0; I < EltNum; ++I)
554         Rep = Builder.CreateInsertElement(Rep, Load,
555                                           ConstantInt::get(I32Ty, I));
556     } else if (Name == "llvm.x86.sse2.psll.dq") {
557       // 128-bit shift left specified in bits.
558       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
559       Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
560                                        Shift / 8); // Shift is in bits.
561     } else if (Name == "llvm.x86.sse2.psrl.dq") {
562       // 128-bit shift right specified in bits.
563       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
564       Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
565                                        Shift / 8); // Shift is in bits.
566     } else if (Name == "llvm.x86.avx2.psll.dq") {
567       // 256-bit shift left specified in bits.
568       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
569       Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
570                                        Shift / 8); // Shift is in bits.
571     } else if (Name == "llvm.x86.avx2.psrl.dq") {
572       // 256-bit shift right specified in bits.
573       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
574       Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
575                                        Shift / 8); // Shift is in bits.
576     } else if (Name == "llvm.x86.sse2.psll.dq.bs") {
577       // 128-bit shift left specified in bytes.
578       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
579       Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
580                                        Shift);
581     } else if (Name == "llvm.x86.sse2.psrl.dq.bs") {
582       // 128-bit shift right specified in bytes.
583       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
584       Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
585                                        Shift);
586     } else if (Name == "llvm.x86.avx2.psll.dq.bs") {
587       // 256-bit shift left specified in bytes.
588       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
589       Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
590                                        Shift);
591     } else if (Name == "llvm.x86.avx2.psrl.dq.bs") {
592       // 256-bit shift right specified in bytes.
593       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
594       Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
595                                        Shift);
596     } else if (Name == "llvm.x86.sse41.pblendw" ||
597                Name == "llvm.x86.sse41.blendpd" ||
598                Name == "llvm.x86.sse41.blendps" ||
599                Name == "llvm.x86.avx.blend.pd.256" ||
600                Name == "llvm.x86.avx.blend.ps.256" ||
601                Name == "llvm.x86.avx2.pblendw" ||
602                Name == "llvm.x86.avx2.pblendd.128" ||
603                Name == "llvm.x86.avx2.pblendd.256") {
604       Value *Op0 = CI->getArgOperand(0);
605       Value *Op1 = CI->getArgOperand(1);
606       unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();
607       VectorType *VecTy = cast<VectorType>(CI->getType());
608       unsigned NumElts = VecTy->getNumElements();
609 
610       SmallVector<Constant*, 16> Idxs;
611       for (unsigned i = 0; i != NumElts; ++i) {
612         unsigned Idx = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
613         Idxs.push_back(Builder.getInt32(Idx));
614       }
615 
616       Rep = Builder.CreateShuffleVector(Op0, Op1, ConstantVector::get(Idxs));
617     } else {
618       bool PD128 = false, PD256 = false, PS128 = false, PS256 = false;
619       if (Name == "llvm.x86.avx.vpermil.pd.256")
620         PD256 = true;
621       else if (Name == "llvm.x86.avx.vpermil.pd")
622         PD128 = true;
623       else if (Name == "llvm.x86.avx.vpermil.ps.256")
624         PS256 = true;
625       else if (Name == "llvm.x86.avx.vpermil.ps")
626         PS128 = true;
627 
628       if (PD256 || PD128 || PS256 || PS128) {
629         Value *Op0 = CI->getArgOperand(0);
630         unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
631         SmallVector<Constant*, 8> Idxs;
632 
633         if (PD128)
634           for (unsigned i = 0; i != 2; ++i)
635             Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1));
636         else if (PD256)
637           for (unsigned l = 0; l != 4; l+=2)
638             for (unsigned i = 0; i != 2; ++i)
639               Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l));
640         else if (PS128)
641           for (unsigned i = 0; i != 4; ++i)
642             Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3));
643         else if (PS256)
644           for (unsigned l = 0; l != 8; l+=4)
645             for (unsigned i = 0; i != 4; ++i)
646               Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l));
647         else
648           llvm_unreachable("Unexpected function");
649 
650         Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs));
651       } else {
652         llvm_unreachable("Unknown function for CallInst upgrade.");
653       }
654     }
655 
656     CI->replaceAllUsesWith(Rep);
657     CI->eraseFromParent();
658     return;
659   }
660 
661   std::string Name = CI->getName().str();
662   if (!Name.empty())
663     CI->setName(Name + ".old");
664 
665   switch (NewFn->getIntrinsicID()) {
666   default:
667     llvm_unreachable("Unknown function for CallInst upgrade.");
668 
669   // Upgrade debug intrinsics to use an additional DIExpression argument.
670   case Intrinsic::dbg_declare: {
671     auto NewCI =
672         Builder.CreateCall3(NewFn, CI->getArgOperand(0), CI->getArgOperand(1),
673                             getExpression(CI->getArgOperand(1), F), Name);
674     NewCI->setDebugLoc(CI->getDebugLoc());
675     CI->replaceAllUsesWith(NewCI);
676     CI->eraseFromParent();
677     return;
678   }
679   case Intrinsic::dbg_value: {
680     auto NewCI = Builder.CreateCall4(
681         NewFn, CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
682         getExpression(CI->getArgOperand(2), F), Name);
683     NewCI->setDebugLoc(CI->getDebugLoc());
684     CI->replaceAllUsesWith(NewCI);
685     CI->eraseFromParent();
686     return;
687   }
688   case Intrinsic::ctlz:
689   case Intrinsic::cttz:
690     assert(CI->getNumArgOperands() == 1 &&
691            "Mismatch between function args and call args");
692     CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0),
693                                                Builder.getFalse(), Name));
694     CI->eraseFromParent();
695     return;
696 
697   case Intrinsic::objectsize:
698     CI->replaceAllUsesWith(Builder.CreateCall2(NewFn,
699                                                CI->getArgOperand(0),
700                                                CI->getArgOperand(1),
701                                                Name));
702     CI->eraseFromParent();
703     return;
704 
705   case Intrinsic::ctpop: {
706     CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0)));
707     CI->eraseFromParent();
708     return;
709   }
710 
711   case Intrinsic::x86_xop_vfrcz_ss:
712   case Intrinsic::x86_xop_vfrcz_sd:
713     CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1),
714                                               Name));
715     CI->eraseFromParent();
716     return;
717 
718   case Intrinsic::x86_sse41_ptestc:
719   case Intrinsic::x86_sse41_ptestz:
720   case Intrinsic::x86_sse41_ptestnzc: {
721     // The arguments for these intrinsics used to be v4f32, and changed
722     // to v2i64. This is purely a nop, since those are bitwise intrinsics.
723     // So, the only thing required is a bitcast for both arguments.
724     // First, check the arguments have the old type.
725     Value *Arg0 = CI->getArgOperand(0);
726     if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4))
727       return;
728 
729     // Old intrinsic, add bitcasts
730     Value *Arg1 = CI->getArgOperand(1);
731 
732     Value *BC0 =
733       Builder.CreateBitCast(Arg0,
734                             VectorType::get(Type::getInt64Ty(C), 2),
735                             "cast");
736     Value *BC1 =
737       Builder.CreateBitCast(Arg1,
738                             VectorType::get(Type::getInt64Ty(C), 2),
739                             "cast");
740 
741     CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name);
742     CI->replaceAllUsesWith(NewCall);
743     CI->eraseFromParent();
744     return;
745   }
746 
747   case Intrinsic::x86_sse41_insertps:
748   case Intrinsic::x86_sse41_dppd:
749   case Intrinsic::x86_sse41_dpps:
750   case Intrinsic::x86_sse41_mpsadbw:
751   case Intrinsic::x86_avx_dp_ps_256:
752   case Intrinsic::x86_avx2_mpsadbw: {
753     // Need to truncate the last argument from i32 to i8 -- this argument models
754     // an inherently 8-bit immediate operand to these x86 instructions.
755     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
756                                  CI->arg_operands().end());
757 
758     // Replace the last argument with a trunc.
759     Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
760 
761     CallInst *NewCall = Builder.CreateCall(NewFn, Args);
762     CI->replaceAllUsesWith(NewCall);
763     CI->eraseFromParent();
764     return;
765   }
766   case Intrinsic::x86_avx512_mask_cmp_ps_512:
767   case Intrinsic::x86_avx512_mask_cmp_pd_512: {
768     // Need to truncate the last argument from i32 to i8 -- this argument models
769     // an inherently 8-bit immediate operand to these x86 instructions.
770     SmallVector<Value *, 5> Args(CI->arg_operands().begin(),
771                                  CI->arg_operands().end());
772 
773     // Replace the last argument with a trunc.
774     Args[2] = Builder.CreateTrunc(Args[2], Type::getInt8Ty(C), "trunc");
775 
776     CallInst *NewCall = Builder.CreateCall(NewFn, Args);
777     CI->replaceAllUsesWith(NewCall);
778     CI->eraseFromParent();
779     return;
780   }
781   }
782 }
783 
784 // This tests each Function to determine if it needs upgrading. When we find
785 // one we are interested in, we then upgrade all calls to reflect the new
786 // function.
787 void llvm::UpgradeCallsToIntrinsic(Function* F) {
788   assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
789 
790   // Upgrade the function and check if it is a totaly new function.
791   Function *NewFn;
792   if (UpgradeIntrinsicFunction(F, NewFn)) {
793     if (NewFn != F) {
794       // Replace all uses to the old function with the new one if necessary.
795       for (Value::user_iterator UI = F->user_begin(), UE = F->user_end();
796            UI != UE; ) {
797         if (CallInst *CI = dyn_cast<CallInst>(*UI++))
798           UpgradeIntrinsicCall(CI, NewFn);
799       }
800       // Remove old function, no longer used, from the module.
801       F->eraseFromParent();
802     }
803   }
804 }
805 
806 void llvm::UpgradeInstWithTBAATag(Instruction *I) {
807   MDNode *MD = I->getMetadata(LLVMContext::MD_tbaa);
808   assert(MD && "UpgradeInstWithTBAATag should have a TBAA tag");
809   // Check if the tag uses struct-path aware TBAA format.
810   if (isa<MDNode>(MD->getOperand(0)) && MD->getNumOperands() >= 3)
811     return;
812 
813   if (MD->getNumOperands() == 3) {
814     Metadata *Elts[] = {MD->getOperand(0), MD->getOperand(1)};
815     MDNode *ScalarType = MDNode::get(I->getContext(), Elts);
816     // Create a MDNode <ScalarType, ScalarType, offset 0, const>
817     Metadata *Elts2[] = {ScalarType, ScalarType,
818                          ConstantAsMetadata::get(Constant::getNullValue(
819                              Type::getInt64Ty(I->getContext()))),
820                          MD->getOperand(2)};
821     I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts2));
822   } else {
823     // Create a MDNode <MD, MD, offset 0>
824     Metadata *Elts[] = {MD, MD, ConstantAsMetadata::get(Constant::getNullValue(
825                                     Type::getInt64Ty(I->getContext())))};
826     I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts));
827   }
828 }
829 
830 Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
831                                       Instruction *&Temp) {
832   if (Opc != Instruction::BitCast)
833     return nullptr;
834 
835   Temp = nullptr;
836   Type *SrcTy = V->getType();
837   if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
838       SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
839     LLVMContext &Context = V->getContext();
840 
841     // We have no information about target data layout, so we assume that
842     // the maximum pointer size is 64bit.
843     Type *MidTy = Type::getInt64Ty(Context);
844     Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
845 
846     return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
847   }
848 
849   return nullptr;
850 }
851 
852 Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
853   if (Opc != Instruction::BitCast)
854     return nullptr;
855 
856   Type *SrcTy = C->getType();
857   if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
858       SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
859     LLVMContext &Context = C->getContext();
860 
861     // We have no information about target data layout, so we assume that
862     // the maximum pointer size is 64bit.
863     Type *MidTy = Type::getInt64Ty(Context);
864 
865     return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
866                                      DestTy);
867   }
868 
869   return nullptr;
870 }
871 
872 /// Check the debug info version number, if it is out-dated, drop the debug
873 /// info. Return true if module is modified.
874 bool llvm::UpgradeDebugInfo(Module &M) {
875   unsigned Version = getDebugMetadataVersionFromModule(M);
876   if (Version == DEBUG_METADATA_VERSION)
877     return false;
878 
879   bool RetCode = StripDebugInfo(M);
880   if (RetCode) {
881     DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
882     M.getContext().diagnose(DiagVersion);
883   }
884   return RetCode;
885 }
886 
887 void llvm::UpgradeMDStringConstant(std::string &String) {
888   const std::string OldPrefix = "llvm.vectorizer.";
889   if (String == "llvm.vectorizer.unroll") {
890     String = "llvm.loop.interleave.count";
891   } else if (String.find(OldPrefix) == 0) {
892     String.replace(0, OldPrefix.size(), "llvm.loop.vectorize.");
893   }
894 }
895