1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements the auto-upgrade helper functions. 11 // This is where deprecated IR intrinsics and other IR features are updated to 12 // current specifications. 13 // 14 //===----------------------------------------------------------------------===// 15 16 #include "llvm/IR/AutoUpgrade.h" 17 #include "llvm/ADT/StringSwitch.h" 18 #include "llvm/IR/CFG.h" 19 #include "llvm/IR/CallSite.h" 20 #include "llvm/IR/Constants.h" 21 #include "llvm/IR/DIBuilder.h" 22 #include "llvm/IR/DebugInfo.h" 23 #include "llvm/IR/DiagnosticInfo.h" 24 #include "llvm/IR/Function.h" 25 #include "llvm/IR/IRBuilder.h" 26 #include "llvm/IR/Instruction.h" 27 #include "llvm/IR/IntrinsicInst.h" 28 #include "llvm/IR/LLVMContext.h" 29 #include "llvm/IR/Module.h" 30 #include "llvm/Support/ErrorHandling.h" 31 #include "llvm/Support/Regex.h" 32 #include <cstring> 33 using namespace llvm; 34 35 static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); } 36 37 // Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have 38 // changed their type from v4f32 to v2i64. 39 static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID, 40 Function *&NewFn) { 41 // Check whether this is an old version of the function, which received 42 // v4f32 arguments. 43 Type *Arg0Type = F->getFunctionType()->getParamType(0); 44 if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4)) 45 return false; 46 47 // Yes, it's old, replace it with new version. 48 rename(F); 49 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); 50 return true; 51 } 52 53 // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask 54 // arguments have changed their type from i32 to i8. 55 static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, 56 Function *&NewFn) { 57 // Check that the last argument is an i32. 58 Type *LastArgType = F->getFunctionType()->getParamType( 59 F->getFunctionType()->getNumParams() - 1); 60 if (!LastArgType->isIntegerTy(32)) 61 return false; 62 63 // Move this function aside and map down. 64 rename(F); 65 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); 66 return true; 67 } 68 69 static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) { 70 // All of the intrinsics matches below should be marked with which llvm 71 // version started autoupgrading them. At some point in the future we would 72 // like to use this information to remove upgrade code for some older 73 // intrinsics. It is currently undecided how we will determine that future 74 // point. 75 if (Name=="ssse3.pabs.b.128" || // Added in 6.0 76 Name=="ssse3.pabs.w.128" || // Added in 6.0 77 Name=="ssse3.pabs.d.128" || // Added in 6.0 78 Name.startswith("avx2.pabs.") || // Added in 6.0 79 Name.startswith("avx512.mask.pabs.") || // Added in 6.0 80 Name.startswith("avx512.mask.pbroadcast") || // Added in 6.0 81 Name.startswith("sse2.pcmpeq.") || // Added in 3.1 82 Name.startswith("sse2.pcmpgt.") || // Added in 3.1 83 Name.startswith("avx2.pcmpeq.") || // Added in 3.1 84 Name.startswith("avx2.pcmpgt.") || // Added in 3.1 85 Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9 86 Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9 87 Name.startswith("avx.vperm2f128.") || // Added in 6.0 88 Name == "avx2.vperm2i128" || // Added in 6.0 89 Name == "sse.add.ss" || // Added in 4.0 90 Name == "sse2.add.sd" || // Added in 4.0 91 Name == "sse.sub.ss" || // Added in 4.0 92 Name == "sse2.sub.sd" || // Added in 4.0 93 Name == "sse.mul.ss" || // Added in 4.0 94 Name == "sse2.mul.sd" || // Added in 4.0 95 Name == "sse.div.ss" || // Added in 4.0 96 Name == "sse2.div.sd" || // Added in 4.0 97 Name == "sse41.pmaxsb" || // Added in 3.9 98 Name == "sse2.pmaxs.w" || // Added in 3.9 99 Name == "sse41.pmaxsd" || // Added in 3.9 100 Name == "sse2.pmaxu.b" || // Added in 3.9 101 Name == "sse41.pmaxuw" || // Added in 3.9 102 Name == "sse41.pmaxud" || // Added in 3.9 103 Name == "sse41.pminsb" || // Added in 3.9 104 Name == "sse2.pmins.w" || // Added in 3.9 105 Name == "sse41.pminsd" || // Added in 3.9 106 Name == "sse2.pminu.b" || // Added in 3.9 107 Name == "sse41.pminuw" || // Added in 3.9 108 Name == "sse41.pminud" || // Added in 3.9 109 Name.startswith("avx512.mask.pshuf.b.") || // Added in 4.0 110 Name.startswith("avx2.pmax") || // Added in 3.9 111 Name.startswith("avx2.pmin") || // Added in 3.9 112 Name.startswith("avx512.mask.pmax") || // Added in 4.0 113 Name.startswith("avx512.mask.pmin") || // Added in 4.0 114 Name.startswith("avx2.vbroadcast") || // Added in 3.8 115 Name.startswith("avx2.pbroadcast") || // Added in 3.8 116 Name.startswith("avx.vpermil.") || // Added in 3.1 117 Name.startswith("sse2.pshuf") || // Added in 3.9 118 Name.startswith("avx512.pbroadcast") || // Added in 3.9 119 Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9 120 Name.startswith("avx512.mask.movddup") || // Added in 3.9 121 Name.startswith("avx512.mask.movshdup") || // Added in 3.9 122 Name.startswith("avx512.mask.movsldup") || // Added in 3.9 123 Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9 124 Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9 125 Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9 126 Name.startswith("avx512.mask.shuf.p") || // Added in 4.0 127 Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9 128 Name.startswith("avx512.mask.perm.df.") || // Added in 3.9 129 Name.startswith("avx512.mask.perm.di.") || // Added in 3.9 130 Name.startswith("avx512.mask.punpckl") || // Added in 3.9 131 Name.startswith("avx512.mask.punpckh") || // Added in 3.9 132 Name.startswith("avx512.mask.unpckl.") || // Added in 3.9 133 Name.startswith("avx512.mask.unpckh.") || // Added in 3.9 134 Name.startswith("avx512.mask.pand.") || // Added in 3.9 135 Name.startswith("avx512.mask.pandn.") || // Added in 3.9 136 Name.startswith("avx512.mask.por.") || // Added in 3.9 137 Name.startswith("avx512.mask.pxor.") || // Added in 3.9 138 Name.startswith("avx512.mask.and.") || // Added in 3.9 139 Name.startswith("avx512.mask.andn.") || // Added in 3.9 140 Name.startswith("avx512.mask.or.") || // Added in 3.9 141 Name.startswith("avx512.mask.xor.") || // Added in 3.9 142 Name.startswith("avx512.mask.padd.") || // Added in 4.0 143 Name.startswith("avx512.mask.psub.") || // Added in 4.0 144 Name.startswith("avx512.mask.pmull.") || // Added in 4.0 145 Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0 146 Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0 147 Name.startswith("avx512.mask.pmul.dq.") || // Added in 4.0 148 Name.startswith("avx512.mask.pmulu.dq.") || // Added in 4.0 149 Name.startswith("avx512.mask.packsswb.") || // Added in 5.0 150 Name.startswith("avx512.mask.packssdw.") || // Added in 5.0 151 Name.startswith("avx512.mask.packuswb.") || // Added in 5.0 152 Name.startswith("avx512.mask.packusdw.") || // Added in 5.0 153 Name.startswith("avx512.mask.cmp.b") || // Added in 5.0 154 Name.startswith("avx512.mask.cmp.d") || // Added in 5.0 155 Name.startswith("avx512.mask.cmp.q") || // Added in 5.0 156 Name.startswith("avx512.mask.cmp.w") || // Added in 5.0 157 Name.startswith("avx512.mask.ucmp.") || // Added in 5.0 158 Name == "avx512.mask.add.pd.128" || // Added in 4.0 159 Name == "avx512.mask.add.pd.256" || // Added in 4.0 160 Name == "avx512.mask.add.ps.128" || // Added in 4.0 161 Name == "avx512.mask.add.ps.256" || // Added in 4.0 162 Name == "avx512.mask.div.pd.128" || // Added in 4.0 163 Name == "avx512.mask.div.pd.256" || // Added in 4.0 164 Name == "avx512.mask.div.ps.128" || // Added in 4.0 165 Name == "avx512.mask.div.ps.256" || // Added in 4.0 166 Name == "avx512.mask.mul.pd.128" || // Added in 4.0 167 Name == "avx512.mask.mul.pd.256" || // Added in 4.0 168 Name == "avx512.mask.mul.ps.128" || // Added in 4.0 169 Name == "avx512.mask.mul.ps.256" || // Added in 4.0 170 Name == "avx512.mask.sub.pd.128" || // Added in 4.0 171 Name == "avx512.mask.sub.pd.256" || // Added in 4.0 172 Name == "avx512.mask.sub.ps.128" || // Added in 4.0 173 Name == "avx512.mask.sub.ps.256" || // Added in 4.0 174 Name == "avx512.mask.max.pd.128" || // Added in 5.0 175 Name == "avx512.mask.max.pd.256" || // Added in 5.0 176 Name == "avx512.mask.max.ps.128" || // Added in 5.0 177 Name == "avx512.mask.max.ps.256" || // Added in 5.0 178 Name == "avx512.mask.min.pd.128" || // Added in 5.0 179 Name == "avx512.mask.min.pd.256" || // Added in 5.0 180 Name == "avx512.mask.min.ps.128" || // Added in 5.0 181 Name == "avx512.mask.min.ps.256" || // Added in 5.0 182 Name.startswith("avx512.mask.vpermilvar.") || // Added in 4.0 183 Name.startswith("avx512.mask.psll.d") || // Added in 4.0 184 Name.startswith("avx512.mask.psll.q") || // Added in 4.0 185 Name.startswith("avx512.mask.psll.w") || // Added in 4.0 186 Name.startswith("avx512.mask.psra.d") || // Added in 4.0 187 Name.startswith("avx512.mask.psra.q") || // Added in 4.0 188 Name.startswith("avx512.mask.psra.w") || // Added in 4.0 189 Name.startswith("avx512.mask.psrl.d") || // Added in 4.0 190 Name.startswith("avx512.mask.psrl.q") || // Added in 4.0 191 Name.startswith("avx512.mask.psrl.w") || // Added in 4.0 192 Name.startswith("avx512.mask.pslli") || // Added in 4.0 193 Name.startswith("avx512.mask.psrai") || // Added in 4.0 194 Name.startswith("avx512.mask.psrli") || // Added in 4.0 195 Name.startswith("avx512.mask.psllv") || // Added in 4.0 196 Name.startswith("avx512.mask.psrav") || // Added in 4.0 197 Name.startswith("avx512.mask.psrlv") || // Added in 4.0 198 Name.startswith("sse41.pmovsx") || // Added in 3.8 199 Name.startswith("sse41.pmovzx") || // Added in 3.9 200 Name.startswith("avx2.pmovsx") || // Added in 3.9 201 Name.startswith("avx2.pmovzx") || // Added in 3.9 202 Name.startswith("avx512.mask.pmovsx") || // Added in 4.0 203 Name.startswith("avx512.mask.pmovzx") || // Added in 4.0 204 Name.startswith("avx512.mask.lzcnt.") || // Added in 5.0 205 Name == "sse2.cvtdq2pd" || // Added in 3.9 206 Name == "sse2.cvtps2pd" || // Added in 3.9 207 Name == "avx.cvtdq2.pd.256" || // Added in 3.9 208 Name == "avx.cvt.ps2.pd.256" || // Added in 3.9 209 Name.startswith("avx.vinsertf128.") || // Added in 3.7 210 Name == "avx2.vinserti128" || // Added in 3.7 211 Name.startswith("avx512.mask.insert") || // Added in 4.0 212 Name.startswith("avx.vextractf128.") || // Added in 3.7 213 Name == "avx2.vextracti128" || // Added in 3.7 214 Name.startswith("avx512.mask.vextract") || // Added in 4.0 215 Name.startswith("sse4a.movnt.") || // Added in 3.9 216 Name.startswith("avx.movnt.") || // Added in 3.2 217 Name.startswith("avx512.storent.") || // Added in 3.9 218 Name == "sse41.movntdqa" || // Added in 5.0 219 Name == "avx2.movntdqa" || // Added in 5.0 220 Name == "avx512.movntdqa" || // Added in 5.0 221 Name == "sse2.storel.dq" || // Added in 3.9 222 Name.startswith("sse.storeu.") || // Added in 3.9 223 Name.startswith("sse2.storeu.") || // Added in 3.9 224 Name.startswith("avx.storeu.") || // Added in 3.9 225 Name.startswith("avx512.mask.storeu.") || // Added in 3.9 226 Name.startswith("avx512.mask.store.p") || // Added in 3.9 227 Name.startswith("avx512.mask.store.b.") || // Added in 3.9 228 Name.startswith("avx512.mask.store.w.") || // Added in 3.9 229 Name.startswith("avx512.mask.store.d.") || // Added in 3.9 230 Name.startswith("avx512.mask.store.q.") || // Added in 3.9 231 Name.startswith("avx512.mask.loadu.") || // Added in 3.9 232 Name.startswith("avx512.mask.load.") || // Added in 3.9 233 Name == "sse42.crc32.64.8" || // Added in 3.4 234 Name.startswith("avx.vbroadcast.s") || // Added in 3.5 235 Name.startswith("avx512.mask.palignr.") || // Added in 3.9 236 Name.startswith("avx512.mask.valign.") || // Added in 4.0 237 Name.startswith("sse2.psll.dq") || // Added in 3.7 238 Name.startswith("sse2.psrl.dq") || // Added in 3.7 239 Name.startswith("avx2.psll.dq") || // Added in 3.7 240 Name.startswith("avx2.psrl.dq") || // Added in 3.7 241 Name.startswith("avx512.psll.dq") || // Added in 3.9 242 Name.startswith("avx512.psrl.dq") || // Added in 3.9 243 Name == "sse41.pblendw" || // Added in 3.7 244 Name.startswith("sse41.blendp") || // Added in 3.7 245 Name.startswith("avx.blend.p") || // Added in 3.7 246 Name == "avx2.pblendw" || // Added in 3.7 247 Name.startswith("avx2.pblendd.") || // Added in 3.7 248 Name.startswith("avx.vbroadcastf128") || // Added in 4.0 249 Name == "avx2.vbroadcasti128" || // Added in 3.7 250 Name.startswith("avx512.mask.broadcastf") || // Added in 6.0 251 Name.startswith("avx512.mask.broadcasti") || // Added in 6.0 252 Name == "xop.vpcmov" || // Added in 3.8 253 Name == "xop.vpcmov.256" || // Added in 5.0 254 Name.startswith("avx512.mask.move.s") || // Added in 4.0 255 Name.startswith("avx512.cvtmask2") || // Added in 5.0 256 (Name.startswith("xop.vpcom") && // Added in 3.2 257 F->arg_size() == 2) || 258 Name.startswith("sse2.pavg") || // Added in 6.0 259 Name.startswith("avx2.pavg") || // Added in 6.0 260 Name.startswith("avx512.mask.pavg")) // Added in 6.0 261 return true; 262 263 return false; 264 } 265 266 static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name, 267 Function *&NewFn) { 268 // Only handle intrinsics that start with "x86.". 269 if (!Name.startswith("x86.")) 270 return false; 271 // Remove "x86." prefix. 272 Name = Name.substr(4); 273 274 if (ShouldUpgradeX86Intrinsic(F, Name)) { 275 NewFn = nullptr; 276 return true; 277 } 278 279 // SSE4.1 ptest functions may have an old signature. 280 if (Name.startswith("sse41.ptest")) { // Added in 3.2 281 if (Name.substr(11) == "c") 282 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestc, NewFn); 283 if (Name.substr(11) == "z") 284 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestz, NewFn); 285 if (Name.substr(11) == "nzc") 286 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestnzc, NewFn); 287 } 288 // Several blend and other instructions with masks used the wrong number of 289 // bits. 290 if (Name == "sse41.insertps") // Added in 3.6 291 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps, 292 NewFn); 293 if (Name == "sse41.dppd") // Added in 3.6 294 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd, 295 NewFn); 296 if (Name == "sse41.dpps") // Added in 3.6 297 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps, 298 NewFn); 299 if (Name == "sse41.mpsadbw") // Added in 3.6 300 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw, 301 NewFn); 302 if (Name == "avx.dp.ps.256") // Added in 3.6 303 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256, 304 NewFn); 305 if (Name == "avx2.mpsadbw") // Added in 3.6 306 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw, 307 NewFn); 308 309 // frcz.ss/sd may need to have an argument dropped. Added in 3.2 310 if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) { 311 rename(F); 312 NewFn = Intrinsic::getDeclaration(F->getParent(), 313 Intrinsic::x86_xop_vfrcz_ss); 314 return true; 315 } 316 if (Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) { 317 rename(F); 318 NewFn = Intrinsic::getDeclaration(F->getParent(), 319 Intrinsic::x86_xop_vfrcz_sd); 320 return true; 321 } 322 // Upgrade any XOP PERMIL2 index operand still using a float/double vector. 323 if (Name.startswith("xop.vpermil2")) { // Added in 3.9 324 auto Idx = F->getFunctionType()->getParamType(2); 325 if (Idx->isFPOrFPVectorTy()) { 326 rename(F); 327 unsigned IdxSize = Idx->getPrimitiveSizeInBits(); 328 unsigned EltSize = Idx->getScalarSizeInBits(); 329 Intrinsic::ID Permil2ID; 330 if (EltSize == 64 && IdxSize == 128) 331 Permil2ID = Intrinsic::x86_xop_vpermil2pd; 332 else if (EltSize == 32 && IdxSize == 128) 333 Permil2ID = Intrinsic::x86_xop_vpermil2ps; 334 else if (EltSize == 64 && IdxSize == 256) 335 Permil2ID = Intrinsic::x86_xop_vpermil2pd_256; 336 else 337 Permil2ID = Intrinsic::x86_xop_vpermil2ps_256; 338 NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID); 339 return true; 340 } 341 } 342 343 return false; 344 } 345 346 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { 347 assert(F && "Illegal to upgrade a non-existent Function."); 348 349 // Quickly eliminate it, if it's not a candidate. 350 StringRef Name = F->getName(); 351 if (Name.size() <= 8 || !Name.startswith("llvm.")) 352 return false; 353 Name = Name.substr(5); // Strip off "llvm." 354 355 switch (Name[0]) { 356 default: break; 357 case 'a': { 358 if (Name.startswith("arm.rbit") || Name.startswith("aarch64.rbit")) { 359 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse, 360 F->arg_begin()->getType()); 361 return true; 362 } 363 if (Name.startswith("arm.neon.vclz")) { 364 Type* args[2] = { 365 F->arg_begin()->getType(), 366 Type::getInt1Ty(F->getContext()) 367 }; 368 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to 369 // the end of the name. Change name from llvm.arm.neon.vclz.* to 370 // llvm.ctlz.* 371 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false); 372 NewFn = Function::Create(fType, F->getLinkage(), 373 "llvm.ctlz." + Name.substr(14), F->getParent()); 374 return true; 375 } 376 if (Name.startswith("arm.neon.vcnt")) { 377 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, 378 F->arg_begin()->getType()); 379 return true; 380 } 381 Regex vldRegex("^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$"); 382 if (vldRegex.match(Name)) { 383 auto fArgs = F->getFunctionType()->params(); 384 SmallVector<Type *, 4> Tys(fArgs.begin(), fArgs.end()); 385 // Can't use Intrinsic::getDeclaration here as the return types might 386 // then only be structurally equal. 387 FunctionType* fType = FunctionType::get(F->getReturnType(), Tys, false); 388 NewFn = Function::Create(fType, F->getLinkage(), 389 "llvm." + Name + ".p0i8", F->getParent()); 390 return true; 391 } 392 Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$"); 393 if (vstRegex.match(Name)) { 394 static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1, 395 Intrinsic::arm_neon_vst2, 396 Intrinsic::arm_neon_vst3, 397 Intrinsic::arm_neon_vst4}; 398 399 static const Intrinsic::ID StoreLaneInts[] = { 400 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane, 401 Intrinsic::arm_neon_vst4lane 402 }; 403 404 auto fArgs = F->getFunctionType()->params(); 405 Type *Tys[] = {fArgs[0], fArgs[1]}; 406 if (Name.find("lane") == StringRef::npos) 407 NewFn = Intrinsic::getDeclaration(F->getParent(), 408 StoreInts[fArgs.size() - 3], Tys); 409 else 410 NewFn = Intrinsic::getDeclaration(F->getParent(), 411 StoreLaneInts[fArgs.size() - 5], Tys); 412 return true; 413 } 414 if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") { 415 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer); 416 return true; 417 } 418 break; 419 } 420 421 case 'c': { 422 if (Name.startswith("ctlz.") && F->arg_size() == 1) { 423 rename(F); 424 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, 425 F->arg_begin()->getType()); 426 return true; 427 } 428 if (Name.startswith("cttz.") && F->arg_size() == 1) { 429 rename(F); 430 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz, 431 F->arg_begin()->getType()); 432 return true; 433 } 434 break; 435 } 436 case 'd': { 437 if (Name == "dbg.value" && F->arg_size() == 4) { 438 rename(F); 439 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value); 440 return true; 441 } 442 break; 443 } 444 case 'i': 445 case 'l': { 446 bool IsLifetimeStart = Name.startswith("lifetime.start"); 447 if (IsLifetimeStart || Name.startswith("invariant.start")) { 448 Intrinsic::ID ID = IsLifetimeStart ? 449 Intrinsic::lifetime_start : Intrinsic::invariant_start; 450 auto Args = F->getFunctionType()->params(); 451 Type* ObjectPtr[1] = {Args[1]}; 452 if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) { 453 rename(F); 454 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr); 455 return true; 456 } 457 } 458 459 bool IsLifetimeEnd = Name.startswith("lifetime.end"); 460 if (IsLifetimeEnd || Name.startswith("invariant.end")) { 461 Intrinsic::ID ID = IsLifetimeEnd ? 462 Intrinsic::lifetime_end : Intrinsic::invariant_end; 463 464 auto Args = F->getFunctionType()->params(); 465 Type* ObjectPtr[1] = {Args[IsLifetimeEnd ? 1 : 2]}; 466 if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) { 467 rename(F); 468 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr); 469 return true; 470 } 471 } 472 break; 473 } 474 case 'm': { 475 if (Name.startswith("masked.load.")) { 476 Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() }; 477 if (F->getName() != Intrinsic::getName(Intrinsic::masked_load, Tys)) { 478 rename(F); 479 NewFn = Intrinsic::getDeclaration(F->getParent(), 480 Intrinsic::masked_load, 481 Tys); 482 return true; 483 } 484 } 485 if (Name.startswith("masked.store.")) { 486 auto Args = F->getFunctionType()->params(); 487 Type *Tys[] = { Args[0], Args[1] }; 488 if (F->getName() != Intrinsic::getName(Intrinsic::masked_store, Tys)) { 489 rename(F); 490 NewFn = Intrinsic::getDeclaration(F->getParent(), 491 Intrinsic::masked_store, 492 Tys); 493 return true; 494 } 495 } 496 // Renaming gather/scatter intrinsics with no address space overloading 497 // to the new overload which includes an address space 498 if (Name.startswith("masked.gather.")) { 499 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()}; 500 if (F->getName() != Intrinsic::getName(Intrinsic::masked_gather, Tys)) { 501 rename(F); 502 NewFn = Intrinsic::getDeclaration(F->getParent(), 503 Intrinsic::masked_gather, Tys); 504 return true; 505 } 506 } 507 if (Name.startswith("masked.scatter.")) { 508 auto Args = F->getFunctionType()->params(); 509 Type *Tys[] = {Args[0], Args[1]}; 510 if (F->getName() != Intrinsic::getName(Intrinsic::masked_scatter, Tys)) { 511 rename(F); 512 NewFn = Intrinsic::getDeclaration(F->getParent(), 513 Intrinsic::masked_scatter, Tys); 514 return true; 515 } 516 } 517 break; 518 } 519 case 'n': { 520 if (Name.startswith("nvvm.")) { 521 Name = Name.substr(5); 522 523 // The following nvvm intrinsics correspond exactly to an LLVM intrinsic. 524 Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name) 525 .Cases("brev32", "brev64", Intrinsic::bitreverse) 526 .Case("clz.i", Intrinsic::ctlz) 527 .Case("popc.i", Intrinsic::ctpop) 528 .Default(Intrinsic::not_intrinsic); 529 if (IID != Intrinsic::not_intrinsic && F->arg_size() == 1) { 530 NewFn = Intrinsic::getDeclaration(F->getParent(), IID, 531 {F->getReturnType()}); 532 return true; 533 } 534 535 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but 536 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall. 537 // 538 // TODO: We could add lohi.i2d. 539 bool Expand = StringSwitch<bool>(Name) 540 .Cases("abs.i", "abs.ll", true) 541 .Cases("clz.ll", "popc.ll", "h2f", true) 542 .Cases("max.i", "max.ll", "max.ui", "max.ull", true) 543 .Cases("min.i", "min.ll", "min.ui", "min.ull", true) 544 .Default(false); 545 if (Expand) { 546 NewFn = nullptr; 547 return true; 548 } 549 } 550 break; 551 } 552 case 'o': 553 // We only need to change the name to match the mangling including the 554 // address space. 555 if (Name.startswith("objectsize.")) { 556 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() }; 557 if (F->arg_size() == 2 || 558 F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) { 559 rename(F); 560 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize, 561 Tys); 562 return true; 563 } 564 } 565 break; 566 567 case 's': 568 if (Name == "stackprotectorcheck") { 569 NewFn = nullptr; 570 return true; 571 } 572 break; 573 574 case 'x': 575 if (UpgradeX86IntrinsicFunction(F, Name, NewFn)) 576 return true; 577 } 578 // Remangle our intrinsic since we upgrade the mangling 579 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F); 580 if (Result != None) { 581 NewFn = Result.getValue(); 582 return true; 583 } 584 585 // This may not belong here. This function is effectively being overloaded 586 // to both detect an intrinsic which needs upgrading, and to provide the 587 // upgraded form of the intrinsic. We should perhaps have two separate 588 // functions for this. 589 return false; 590 } 591 592 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) { 593 NewFn = nullptr; 594 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn); 595 assert(F != NewFn && "Intrinsic function upgraded to the same function"); 596 597 // Upgrade intrinsic attributes. This does not change the function. 598 if (NewFn) 599 F = NewFn; 600 if (Intrinsic::ID id = F->getIntrinsicID()) 601 F->setAttributes(Intrinsic::getAttributes(F->getContext(), id)); 602 return Upgraded; 603 } 604 605 bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) { 606 // Nothing to do yet. 607 return false; 608 } 609 610 // Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them 611 // to byte shuffles. 612 static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, 613 Value *Op, unsigned Shift) { 614 Type *ResultTy = Op->getType(); 615 unsigned NumElts = ResultTy->getVectorNumElements() * 8; 616 617 // Bitcast from a 64-bit element type to a byte element type. 618 Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts); 619 Op = Builder.CreateBitCast(Op, VecTy, "cast"); 620 621 // We'll be shuffling in zeroes. 622 Value *Res = Constant::getNullValue(VecTy); 623 624 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise, 625 // we'll just return the zero vector. 626 if (Shift < 16) { 627 uint32_t Idxs[64]; 628 // 256/512-bit version is split into 2/4 16-byte lanes. 629 for (unsigned l = 0; l != NumElts; l += 16) 630 for (unsigned i = 0; i != 16; ++i) { 631 unsigned Idx = NumElts + i - Shift; 632 if (Idx < NumElts) 633 Idx -= NumElts - 16; // end of lane, switch operand. 634 Idxs[l + i] = Idx + l; 635 } 636 637 Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts)); 638 } 639 640 // Bitcast back to a 64-bit element type. 641 return Builder.CreateBitCast(Res, ResultTy, "cast"); 642 } 643 644 // Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them 645 // to byte shuffles. 646 static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, 647 unsigned Shift) { 648 Type *ResultTy = Op->getType(); 649 unsigned NumElts = ResultTy->getVectorNumElements() * 8; 650 651 // Bitcast from a 64-bit element type to a byte element type. 652 Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts); 653 Op = Builder.CreateBitCast(Op, VecTy, "cast"); 654 655 // We'll be shuffling in zeroes. 656 Value *Res = Constant::getNullValue(VecTy); 657 658 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise, 659 // we'll just return the zero vector. 660 if (Shift < 16) { 661 uint32_t Idxs[64]; 662 // 256/512-bit version is split into 2/4 16-byte lanes. 663 for (unsigned l = 0; l != NumElts; l += 16) 664 for (unsigned i = 0; i != 16; ++i) { 665 unsigned Idx = i + Shift; 666 if (Idx >= 16) 667 Idx += NumElts - 16; // end of lane, switch operand. 668 Idxs[l + i] = Idx + l; 669 } 670 671 Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts)); 672 } 673 674 // Bitcast back to a 64-bit element type. 675 return Builder.CreateBitCast(Res, ResultTy, "cast"); 676 } 677 678 static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask, 679 unsigned NumElts) { 680 llvm::VectorType *MaskTy = llvm::VectorType::get(Builder.getInt1Ty(), 681 cast<IntegerType>(Mask->getType())->getBitWidth()); 682 Mask = Builder.CreateBitCast(Mask, MaskTy); 683 684 // If we have less than 8 elements, then the starting mask was an i8 and 685 // we need to extract down to the right number of elements. 686 if (NumElts < 8) { 687 uint32_t Indices[4]; 688 for (unsigned i = 0; i != NumElts; ++i) 689 Indices[i] = i; 690 Mask = Builder.CreateShuffleVector(Mask, Mask, 691 makeArrayRef(Indices, NumElts), 692 "extract"); 693 } 694 695 return Mask; 696 } 697 698 static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask, 699 Value *Op0, Value *Op1) { 700 // If the mask is all ones just emit the align operation. 701 if (const auto *C = dyn_cast<Constant>(Mask)) 702 if (C->isAllOnesValue()) 703 return Op0; 704 705 Mask = getX86MaskVec(Builder, Mask, Op0->getType()->getVectorNumElements()); 706 return Builder.CreateSelect(Mask, Op0, Op1); 707 } 708 709 // Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics. 710 // PALIGNR handles large immediates by shifting while VALIGN masks the immediate 711 // so we need to handle both cases. VALIGN also doesn't have 128-bit lanes. 712 static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, 713 Value *Op1, Value *Shift, 714 Value *Passthru, Value *Mask, 715 bool IsVALIGN) { 716 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue(); 717 718 unsigned NumElts = Op0->getType()->getVectorNumElements(); 719 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!"); 720 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!"); 721 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!"); 722 723 // Mask the immediate for VALIGN. 724 if (IsVALIGN) 725 ShiftVal &= (NumElts - 1); 726 727 // If palignr is shifting the pair of vectors more than the size of two 728 // lanes, emit zero. 729 if (ShiftVal >= 32) 730 return llvm::Constant::getNullValue(Op0->getType()); 731 732 // If palignr is shifting the pair of input vectors more than one lane, 733 // but less than two lanes, convert to shifting in zeroes. 734 if (ShiftVal > 16) { 735 ShiftVal -= 16; 736 Op1 = Op0; 737 Op0 = llvm::Constant::getNullValue(Op0->getType()); 738 } 739 740 uint32_t Indices[64]; 741 // 256-bit palignr operates on 128-bit lanes so we need to handle that 742 for (unsigned l = 0; l < NumElts; l += 16) { 743 for (unsigned i = 0; i != 16; ++i) { 744 unsigned Idx = ShiftVal + i; 745 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN. 746 Idx += NumElts - 16; // End of lane, switch operand. 747 Indices[l + i] = Idx + l; 748 } 749 } 750 751 Value *Align = Builder.CreateShuffleVector(Op1, Op0, 752 makeArrayRef(Indices, NumElts), 753 "palignr"); 754 755 return EmitX86Select(Builder, Mask, Align, Passthru); 756 } 757 758 static Value *UpgradeMaskedStore(IRBuilder<> &Builder, 759 Value *Ptr, Value *Data, Value *Mask, 760 bool Aligned) { 761 // Cast the pointer to the right type. 762 Ptr = Builder.CreateBitCast(Ptr, 763 llvm::PointerType::getUnqual(Data->getType())); 764 unsigned Align = 765 Aligned ? cast<VectorType>(Data->getType())->getBitWidth() / 8 : 1; 766 767 // If the mask is all ones just emit a regular store. 768 if (const auto *C = dyn_cast<Constant>(Mask)) 769 if (C->isAllOnesValue()) 770 return Builder.CreateAlignedStore(Data, Ptr, Align); 771 772 // Convert the mask from an integer type to a vector of i1. 773 unsigned NumElts = Data->getType()->getVectorNumElements(); 774 Mask = getX86MaskVec(Builder, Mask, NumElts); 775 return Builder.CreateMaskedStore(Data, Ptr, Align, Mask); 776 } 777 778 static Value *UpgradeMaskedLoad(IRBuilder<> &Builder, 779 Value *Ptr, Value *Passthru, Value *Mask, 780 bool Aligned) { 781 // Cast the pointer to the right type. 782 Ptr = Builder.CreateBitCast(Ptr, 783 llvm::PointerType::getUnqual(Passthru->getType())); 784 unsigned Align = 785 Aligned ? cast<VectorType>(Passthru->getType())->getBitWidth() / 8 : 1; 786 787 // If the mask is all ones just emit a regular store. 788 if (const auto *C = dyn_cast<Constant>(Mask)) 789 if (C->isAllOnesValue()) 790 return Builder.CreateAlignedLoad(Ptr, Align); 791 792 // Convert the mask from an integer type to a vector of i1. 793 unsigned NumElts = Passthru->getType()->getVectorNumElements(); 794 Mask = getX86MaskVec(Builder, Mask, NumElts); 795 return Builder.CreateMaskedLoad(Ptr, Align, Mask, Passthru); 796 } 797 798 static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) { 799 Value *Op0 = CI.getArgOperand(0); 800 llvm::Type *Ty = Op0->getType(); 801 Value *Zero = llvm::Constant::getNullValue(Ty); 802 Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_SGT, Op0, Zero); 803 Value *Neg = Builder.CreateNeg(Op0); 804 Value *Res = Builder.CreateSelect(Cmp, Op0, Neg); 805 806 if (CI.getNumArgOperands() == 3) 807 Res = EmitX86Select(Builder,CI.getArgOperand(2), Res, CI.getArgOperand(1)); 808 809 return Res; 810 } 811 812 static Value *upgradeIntMinMax(IRBuilder<> &Builder, CallInst &CI, 813 ICmpInst::Predicate Pred) { 814 Value *Op0 = CI.getArgOperand(0); 815 Value *Op1 = CI.getArgOperand(1); 816 Value *Cmp = Builder.CreateICmp(Pred, Op0, Op1); 817 Value *Res = Builder.CreateSelect(Cmp, Op0, Op1); 818 819 if (CI.getNumArgOperands() == 4) 820 Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2)); 821 822 return Res; 823 } 824 825 static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI, 826 unsigned CC, bool Signed) { 827 Value *Op0 = CI.getArgOperand(0); 828 unsigned NumElts = Op0->getType()->getVectorNumElements(); 829 830 Value *Cmp; 831 if (CC == 3) { 832 Cmp = Constant::getNullValue(llvm::VectorType::get(Builder.getInt1Ty(), NumElts)); 833 } else if (CC == 7) { 834 Cmp = Constant::getAllOnesValue(llvm::VectorType::get(Builder.getInt1Ty(), NumElts)); 835 } else { 836 ICmpInst::Predicate Pred; 837 switch (CC) { 838 default: llvm_unreachable("Unknown condition code"); 839 case 0: Pred = ICmpInst::ICMP_EQ; break; 840 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break; 841 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break; 842 case 4: Pred = ICmpInst::ICMP_NE; break; 843 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break; 844 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break; 845 } 846 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1)); 847 } 848 849 Value *Mask = CI.getArgOperand(CI.getNumArgOperands() - 1); 850 const auto *C = dyn_cast<Constant>(Mask); 851 if (!C || !C->isAllOnesValue()) 852 Cmp = Builder.CreateAnd(Cmp, getX86MaskVec(Builder, Mask, NumElts)); 853 854 if (NumElts < 8) { 855 uint32_t Indices[8]; 856 for (unsigned i = 0; i != NumElts; ++i) 857 Indices[i] = i; 858 for (unsigned i = NumElts; i != 8; ++i) 859 Indices[i] = NumElts + i % NumElts; 860 Cmp = Builder.CreateShuffleVector(Cmp, 861 Constant::getNullValue(Cmp->getType()), 862 Indices); 863 } 864 return Builder.CreateBitCast(Cmp, IntegerType::get(CI.getContext(), 865 std::max(NumElts, 8U))); 866 } 867 868 // Replace a masked intrinsic with an older unmasked intrinsic. 869 static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI, 870 Intrinsic::ID IID) { 871 Function *F = CI.getCalledFunction(); 872 Function *Intrin = Intrinsic::getDeclaration(F->getParent(), IID); 873 Value *Rep = Builder.CreateCall(Intrin, 874 { CI.getArgOperand(0), CI.getArgOperand(1) }); 875 return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2)); 876 } 877 878 static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) { 879 Value* A = CI.getArgOperand(0); 880 Value* B = CI.getArgOperand(1); 881 Value* Src = CI.getArgOperand(2); 882 Value* Mask = CI.getArgOperand(3); 883 884 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1)); 885 Value* Cmp = Builder.CreateIsNotNull(AndNode); 886 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0); 887 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0); 888 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2); 889 return Builder.CreateInsertElement(A, Select, (uint64_t)0); 890 } 891 892 893 static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallInst &CI) { 894 Value* Op = CI.getArgOperand(0); 895 Type* ReturnOp = CI.getType(); 896 unsigned NumElts = CI.getType()->getVectorNumElements(); 897 Value *Mask = getX86MaskVec(Builder, Op, NumElts); 898 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2"); 899 } 900 901 /// Upgrade a call to an old intrinsic. All argument and return casting must be 902 /// provided to seamlessly integrate with existing context. 903 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { 904 Function *F = CI->getCalledFunction(); 905 LLVMContext &C = CI->getContext(); 906 IRBuilder<> Builder(C); 907 Builder.SetInsertPoint(CI->getParent(), CI->getIterator()); 908 909 assert(F && "Intrinsic call is not direct?"); 910 911 if (!NewFn) { 912 // Get the Function's name. 913 StringRef Name = F->getName(); 914 915 assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'"); 916 Name = Name.substr(5); 917 918 bool IsX86 = Name.startswith("x86."); 919 if (IsX86) 920 Name = Name.substr(4); 921 bool IsNVVM = Name.startswith("nvvm."); 922 if (IsNVVM) 923 Name = Name.substr(5); 924 925 if (IsX86 && Name.startswith("sse4a.movnt.")) { 926 Module *M = F->getParent(); 927 SmallVector<Metadata *, 1> Elts; 928 Elts.push_back( 929 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); 930 MDNode *Node = MDNode::get(C, Elts); 931 932 Value *Arg0 = CI->getArgOperand(0); 933 Value *Arg1 = CI->getArgOperand(1); 934 935 // Nontemporal (unaligned) store of the 0'th element of the float/double 936 // vector. 937 Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType(); 938 PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy); 939 Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast"); 940 Value *Extract = 941 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement"); 942 943 StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, 1); 944 SI->setMetadata(M->getMDKindID("nontemporal"), Node); 945 946 // Remove intrinsic. 947 CI->eraseFromParent(); 948 return; 949 } 950 951 if (IsX86 && (Name.startswith("avx.movnt.") || 952 Name.startswith("avx512.storent."))) { 953 Module *M = F->getParent(); 954 SmallVector<Metadata *, 1> Elts; 955 Elts.push_back( 956 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); 957 MDNode *Node = MDNode::get(C, Elts); 958 959 Value *Arg0 = CI->getArgOperand(0); 960 Value *Arg1 = CI->getArgOperand(1); 961 962 // Convert the type of the pointer to a pointer to the stored type. 963 Value *BC = Builder.CreateBitCast(Arg0, 964 PointerType::getUnqual(Arg1->getType()), 965 "cast"); 966 VectorType *VTy = cast<VectorType>(Arg1->getType()); 967 StoreInst *SI = Builder.CreateAlignedStore(Arg1, BC, 968 VTy->getBitWidth() / 8); 969 SI->setMetadata(M->getMDKindID("nontemporal"), Node); 970 971 // Remove intrinsic. 972 CI->eraseFromParent(); 973 return; 974 } 975 976 if (IsX86 && Name == "sse2.storel.dq") { 977 Value *Arg0 = CI->getArgOperand(0); 978 Value *Arg1 = CI->getArgOperand(1); 979 980 Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2); 981 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast"); 982 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0); 983 Value *BC = Builder.CreateBitCast(Arg0, 984 PointerType::getUnqual(Elt->getType()), 985 "cast"); 986 Builder.CreateAlignedStore(Elt, BC, 1); 987 988 // Remove intrinsic. 989 CI->eraseFromParent(); 990 return; 991 } 992 993 if (IsX86 && (Name.startswith("sse.storeu.") || 994 Name.startswith("sse2.storeu.") || 995 Name.startswith("avx.storeu."))) { 996 Value *Arg0 = CI->getArgOperand(0); 997 Value *Arg1 = CI->getArgOperand(1); 998 999 Arg0 = Builder.CreateBitCast(Arg0, 1000 PointerType::getUnqual(Arg1->getType()), 1001 "cast"); 1002 Builder.CreateAlignedStore(Arg1, Arg0, 1); 1003 1004 // Remove intrinsic. 1005 CI->eraseFromParent(); 1006 return; 1007 } 1008 1009 if (IsX86 && (Name.startswith("avx512.mask.store"))) { 1010 // "avx512.mask.storeu." or "avx512.mask.store." 1011 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu". 1012 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), 1013 CI->getArgOperand(2), Aligned); 1014 1015 // Remove intrinsic. 1016 CI->eraseFromParent(); 1017 return; 1018 } 1019 1020 Value *Rep; 1021 // Upgrade packed integer vector compare intrinsics to compare instructions. 1022 if (IsX86 && (Name.startswith("sse2.pcmp") || 1023 Name.startswith("avx2.pcmp"))) { 1024 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt." 1025 bool CmpEq = Name[9] == 'e'; 1026 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT, 1027 CI->getArgOperand(0), CI->getArgOperand(1)); 1028 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); 1029 } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){ 1030 unsigned NumElts = 1031 CI->getArgOperand(1)->getType()->getVectorNumElements(); 1032 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0)); 1033 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 1034 CI->getArgOperand(1)); 1035 } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd")) { 1036 Type *I32Ty = Type::getInt32Ty(C); 1037 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), 1038 ConstantInt::get(I32Ty, 0)); 1039 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), 1040 ConstantInt::get(I32Ty, 0)); 1041 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), 1042 Builder.CreateFAdd(Elt0, Elt1), 1043 ConstantInt::get(I32Ty, 0)); 1044 } else if (IsX86 && (Name == "sse.sub.ss" || Name == "sse2.sub.sd")) { 1045 Type *I32Ty = Type::getInt32Ty(C); 1046 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), 1047 ConstantInt::get(I32Ty, 0)); 1048 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), 1049 ConstantInt::get(I32Ty, 0)); 1050 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), 1051 Builder.CreateFSub(Elt0, Elt1), 1052 ConstantInt::get(I32Ty, 0)); 1053 } else if (IsX86 && (Name == "sse.mul.ss" || Name == "sse2.mul.sd")) { 1054 Type *I32Ty = Type::getInt32Ty(C); 1055 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), 1056 ConstantInt::get(I32Ty, 0)); 1057 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), 1058 ConstantInt::get(I32Ty, 0)); 1059 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), 1060 Builder.CreateFMul(Elt0, Elt1), 1061 ConstantInt::get(I32Ty, 0)); 1062 } else if (IsX86 && (Name == "sse.div.ss" || Name == "sse2.div.sd")) { 1063 Type *I32Ty = Type::getInt32Ty(C); 1064 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), 1065 ConstantInt::get(I32Ty, 0)); 1066 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), 1067 ConstantInt::get(I32Ty, 0)); 1068 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), 1069 Builder.CreateFDiv(Elt0, Elt1), 1070 ConstantInt::get(I32Ty, 0)); 1071 } else if (IsX86 && Name.startswith("avx512.mask.pcmp")) { 1072 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt." 1073 bool CmpEq = Name[16] == 'e'; 1074 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true); 1075 } else if (IsX86 && Name.startswith("avx512.mask.cmp")) { 1076 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 1077 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true); 1078 } else if (IsX86 && Name.startswith("avx512.mask.ucmp")) { 1079 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 1080 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false); 1081 } else if(IsX86 && (Name == "ssse3.pabs.b.128" || 1082 Name == "ssse3.pabs.w.128" || 1083 Name == "ssse3.pabs.d.128" || 1084 Name.startswith("avx2.pabs") || 1085 Name.startswith("avx512.mask.pabs"))) { 1086 Rep = upgradeAbs(Builder, *CI); 1087 } else if (IsX86 && (Name == "sse41.pmaxsb" || 1088 Name == "sse2.pmaxs.w" || 1089 Name == "sse41.pmaxsd" || 1090 Name.startswith("avx2.pmaxs") || 1091 Name.startswith("avx512.mask.pmaxs"))) { 1092 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SGT); 1093 } else if (IsX86 && (Name == "sse2.pmaxu.b" || 1094 Name == "sse41.pmaxuw" || 1095 Name == "sse41.pmaxud" || 1096 Name.startswith("avx2.pmaxu") || 1097 Name.startswith("avx512.mask.pmaxu"))) { 1098 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_UGT); 1099 } else if (IsX86 && (Name == "sse41.pminsb" || 1100 Name == "sse2.pmins.w" || 1101 Name == "sse41.pminsd" || 1102 Name.startswith("avx2.pmins") || 1103 Name.startswith("avx512.mask.pmins"))) { 1104 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SLT); 1105 } else if (IsX86 && (Name == "sse2.pminu.b" || 1106 Name == "sse41.pminuw" || 1107 Name == "sse41.pminud" || 1108 Name.startswith("avx2.pminu") || 1109 Name.startswith("avx512.mask.pminu"))) { 1110 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT); 1111 } else if (IsX86 && (Name == "sse2.cvtdq2pd" || 1112 Name == "sse2.cvtps2pd" || 1113 Name == "avx.cvtdq2.pd.256" || 1114 Name == "avx.cvt.ps2.pd.256" || 1115 Name.startswith("avx512.mask.cvtdq2pd.") || 1116 Name.startswith("avx512.mask.cvtudq2pd."))) { 1117 // Lossless i32/float to double conversion. 1118 // Extract the bottom elements if necessary and convert to double vector. 1119 Value *Src = CI->getArgOperand(0); 1120 VectorType *SrcTy = cast<VectorType>(Src->getType()); 1121 VectorType *DstTy = cast<VectorType>(CI->getType()); 1122 Rep = CI->getArgOperand(0); 1123 1124 unsigned NumDstElts = DstTy->getNumElements(); 1125 if (NumDstElts < SrcTy->getNumElements()) { 1126 assert(NumDstElts == 2 && "Unexpected vector size"); 1127 uint32_t ShuffleMask[2] = { 0, 1 }; 1128 Rep = Builder.CreateShuffleVector(Rep, UndefValue::get(SrcTy), 1129 ShuffleMask); 1130 } 1131 1132 bool SInt2Double = (StringRef::npos != Name.find("cvtdq2")); 1133 bool UInt2Double = (StringRef::npos != Name.find("cvtudq2")); 1134 if (SInt2Double) 1135 Rep = Builder.CreateSIToFP(Rep, DstTy, "cvtdq2pd"); 1136 else if (UInt2Double) 1137 Rep = Builder.CreateUIToFP(Rep, DstTy, "cvtudq2pd"); 1138 else 1139 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd"); 1140 1141 if (CI->getNumArgOperands() == 3) 1142 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 1143 CI->getArgOperand(1)); 1144 } else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) { 1145 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0), 1146 CI->getArgOperand(1), CI->getArgOperand(2), 1147 /*Aligned*/false); 1148 } else if (IsX86 && (Name.startswith("avx512.mask.load."))) { 1149 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0), 1150 CI->getArgOperand(1),CI->getArgOperand(2), 1151 /*Aligned*/true); 1152 } else if (IsX86 && Name.startswith("xop.vpcom")) { 1153 Intrinsic::ID intID; 1154 if (Name.endswith("ub")) 1155 intID = Intrinsic::x86_xop_vpcomub; 1156 else if (Name.endswith("uw")) 1157 intID = Intrinsic::x86_xop_vpcomuw; 1158 else if (Name.endswith("ud")) 1159 intID = Intrinsic::x86_xop_vpcomud; 1160 else if (Name.endswith("uq")) 1161 intID = Intrinsic::x86_xop_vpcomuq; 1162 else if (Name.endswith("b")) 1163 intID = Intrinsic::x86_xop_vpcomb; 1164 else if (Name.endswith("w")) 1165 intID = Intrinsic::x86_xop_vpcomw; 1166 else if (Name.endswith("d")) 1167 intID = Intrinsic::x86_xop_vpcomd; 1168 else if (Name.endswith("q")) 1169 intID = Intrinsic::x86_xop_vpcomq; 1170 else 1171 llvm_unreachable("Unknown suffix"); 1172 1173 Name = Name.substr(9); // strip off "xop.vpcom" 1174 unsigned Imm; 1175 if (Name.startswith("lt")) 1176 Imm = 0; 1177 else if (Name.startswith("le")) 1178 Imm = 1; 1179 else if (Name.startswith("gt")) 1180 Imm = 2; 1181 else if (Name.startswith("ge")) 1182 Imm = 3; 1183 else if (Name.startswith("eq")) 1184 Imm = 4; 1185 else if (Name.startswith("ne")) 1186 Imm = 5; 1187 else if (Name.startswith("false")) 1188 Imm = 6; 1189 else if (Name.startswith("true")) 1190 Imm = 7; 1191 else 1192 llvm_unreachable("Unknown condition"); 1193 1194 Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID); 1195 Rep = 1196 Builder.CreateCall(VPCOM, {CI->getArgOperand(0), CI->getArgOperand(1), 1197 Builder.getInt8(Imm)}); 1198 } else if (IsX86 && Name.startswith("xop.vpcmov")) { 1199 Value *Sel = CI->getArgOperand(2); 1200 Value *NotSel = Builder.CreateNot(Sel); 1201 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel); 1202 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel); 1203 Rep = Builder.CreateOr(Sel0, Sel1); 1204 } else if (IsX86 && Name == "sse42.crc32.64.8") { 1205 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(), 1206 Intrinsic::x86_sse42_crc32_32_8); 1207 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C)); 1208 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)}); 1209 Rep = Builder.CreateZExt(Rep, CI->getType(), ""); 1210 } else if (IsX86 && Name.startswith("avx.vbroadcast.s")) { 1211 // Replace broadcasts with a series of insertelements. 1212 Type *VecTy = CI->getType(); 1213 Type *EltTy = VecTy->getVectorElementType(); 1214 unsigned EltNum = VecTy->getVectorNumElements(); 1215 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0), 1216 EltTy->getPointerTo()); 1217 Value *Load = Builder.CreateLoad(EltTy, Cast); 1218 Type *I32Ty = Type::getInt32Ty(C); 1219 Rep = UndefValue::get(VecTy); 1220 for (unsigned I = 0; I < EltNum; ++I) 1221 Rep = Builder.CreateInsertElement(Rep, Load, 1222 ConstantInt::get(I32Ty, I)); 1223 } else if (IsX86 && (Name.startswith("sse41.pmovsx") || 1224 Name.startswith("sse41.pmovzx") || 1225 Name.startswith("avx2.pmovsx") || 1226 Name.startswith("avx2.pmovzx") || 1227 Name.startswith("avx512.mask.pmovsx") || 1228 Name.startswith("avx512.mask.pmovzx"))) { 1229 VectorType *SrcTy = cast<VectorType>(CI->getArgOperand(0)->getType()); 1230 VectorType *DstTy = cast<VectorType>(CI->getType()); 1231 unsigned NumDstElts = DstTy->getNumElements(); 1232 1233 // Extract a subvector of the first NumDstElts lanes and sign/zero extend. 1234 SmallVector<uint32_t, 8> ShuffleMask(NumDstElts); 1235 for (unsigned i = 0; i != NumDstElts; ++i) 1236 ShuffleMask[i] = i; 1237 1238 Value *SV = Builder.CreateShuffleVector( 1239 CI->getArgOperand(0), UndefValue::get(SrcTy), ShuffleMask); 1240 1241 bool DoSext = (StringRef::npos != Name.find("pmovsx")); 1242 Rep = DoSext ? Builder.CreateSExt(SV, DstTy) 1243 : Builder.CreateZExt(SV, DstTy); 1244 // If there are 3 arguments, it's a masked intrinsic so we need a select. 1245 if (CI->getNumArgOperands() == 3) 1246 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 1247 CI->getArgOperand(1)); 1248 } else if (IsX86 && (Name.startswith("avx.vbroadcastf128") || 1249 Name == "avx2.vbroadcasti128")) { 1250 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle. 1251 Type *EltTy = CI->getType()->getVectorElementType(); 1252 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits(); 1253 Type *VT = VectorType::get(EltTy, NumSrcElts); 1254 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0), 1255 PointerType::getUnqual(VT)); 1256 Value *Load = Builder.CreateAlignedLoad(Op, 1); 1257 if (NumSrcElts == 2) 1258 Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), 1259 { 0, 1, 0, 1 }); 1260 else 1261 Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), 1262 { 0, 1, 2, 3, 0, 1, 2, 3 }); 1263 } else if (IsX86 && (Name.startswith("avx512.mask.broadcastf") || 1264 Name.startswith("avx512.mask.broadcasti"))) { 1265 unsigned NumSrcElts = 1266 CI->getArgOperand(0)->getType()->getVectorNumElements(); 1267 unsigned NumDstElts = CI->getType()->getVectorNumElements(); 1268 1269 SmallVector<uint32_t, 8> ShuffleMask(NumDstElts); 1270 for (unsigned i = 0; i != NumDstElts; ++i) 1271 ShuffleMask[i] = i % NumSrcElts; 1272 1273 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0), 1274 CI->getArgOperand(0), 1275 ShuffleMask); 1276 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 1277 CI->getArgOperand(1)); 1278 } else if (IsX86 && (Name.startswith("avx2.pbroadcast") || 1279 Name.startswith("avx2.vbroadcast") || 1280 Name.startswith("avx512.pbroadcast") || 1281 Name.startswith("avx512.mask.broadcast.s"))) { 1282 // Replace vp?broadcasts with a vector shuffle. 1283 Value *Op = CI->getArgOperand(0); 1284 unsigned NumElts = CI->getType()->getVectorNumElements(); 1285 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), NumElts); 1286 Rep = Builder.CreateShuffleVector(Op, UndefValue::get(Op->getType()), 1287 Constant::getNullValue(MaskTy)); 1288 1289 if (CI->getNumArgOperands() == 3) 1290 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 1291 CI->getArgOperand(1)); 1292 } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) { 1293 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0), 1294 CI->getArgOperand(1), 1295 CI->getArgOperand(2), 1296 CI->getArgOperand(3), 1297 CI->getArgOperand(4), 1298 false); 1299 } else if (IsX86 && Name.startswith("avx512.mask.valign.")) { 1300 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0), 1301 CI->getArgOperand(1), 1302 CI->getArgOperand(2), 1303 CI->getArgOperand(3), 1304 CI->getArgOperand(4), 1305 true); 1306 } else if (IsX86 && (Name == "sse2.psll.dq" || 1307 Name == "avx2.psll.dq")) { 1308 // 128/256-bit shift left specified in bits. 1309 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 1310 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), 1311 Shift / 8); // Shift is in bits. 1312 } else if (IsX86 && (Name == "sse2.psrl.dq" || 1313 Name == "avx2.psrl.dq")) { 1314 // 128/256-bit shift right specified in bits. 1315 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 1316 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), 1317 Shift / 8); // Shift is in bits. 1318 } else if (IsX86 && (Name == "sse2.psll.dq.bs" || 1319 Name == "avx2.psll.dq.bs" || 1320 Name == "avx512.psll.dq.512")) { 1321 // 128/256/512-bit shift left specified in bytes. 1322 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 1323 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); 1324 } else if (IsX86 && (Name == "sse2.psrl.dq.bs" || 1325 Name == "avx2.psrl.dq.bs" || 1326 Name == "avx512.psrl.dq.512")) { 1327 // 128/256/512-bit shift right specified in bytes. 1328 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 1329 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); 1330 } else if (IsX86 && (Name == "sse41.pblendw" || 1331 Name.startswith("sse41.blendp") || 1332 Name.startswith("avx.blend.p") || 1333 Name == "avx2.pblendw" || 1334 Name.startswith("avx2.pblendd."))) { 1335 Value *Op0 = CI->getArgOperand(0); 1336 Value *Op1 = CI->getArgOperand(1); 1337 unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 1338 VectorType *VecTy = cast<VectorType>(CI->getType()); 1339 unsigned NumElts = VecTy->getNumElements(); 1340 1341 SmallVector<uint32_t, 16> Idxs(NumElts); 1342 for (unsigned i = 0; i != NumElts; ++i) 1343 Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i; 1344 1345 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs); 1346 } else if (IsX86 && (Name.startswith("avx.vinsertf128.") || 1347 Name == "avx2.vinserti128" || 1348 Name.startswith("avx512.mask.insert"))) { 1349 Value *Op0 = CI->getArgOperand(0); 1350 Value *Op1 = CI->getArgOperand(1); 1351 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 1352 unsigned DstNumElts = CI->getType()->getVectorNumElements(); 1353 unsigned SrcNumElts = Op1->getType()->getVectorNumElements(); 1354 unsigned Scale = DstNumElts / SrcNumElts; 1355 1356 // Mask off the high bits of the immediate value; hardware ignores those. 1357 Imm = Imm % Scale; 1358 1359 // Extend the second operand into a vector the size of the destination. 1360 Value *UndefV = UndefValue::get(Op1->getType()); 1361 SmallVector<uint32_t, 8> Idxs(DstNumElts); 1362 for (unsigned i = 0; i != SrcNumElts; ++i) 1363 Idxs[i] = i; 1364 for (unsigned i = SrcNumElts; i != DstNumElts; ++i) 1365 Idxs[i] = SrcNumElts; 1366 Rep = Builder.CreateShuffleVector(Op1, UndefV, Idxs); 1367 1368 // Insert the second operand into the first operand. 1369 1370 // Note that there is no guarantee that instruction lowering will actually 1371 // produce a vinsertf128 instruction for the created shuffles. In 1372 // particular, the 0 immediate case involves no lane changes, so it can 1373 // be handled as a blend. 1374 1375 // Example of shuffle mask for 32-bit elements: 1376 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11> 1377 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 > 1378 1379 // First fill with identify mask. 1380 for (unsigned i = 0; i != DstNumElts; ++i) 1381 Idxs[i] = i; 1382 // Then replace the elements where we need to insert. 1383 for (unsigned i = 0; i != SrcNumElts; ++i) 1384 Idxs[i + Imm * SrcNumElts] = i + DstNumElts; 1385 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs); 1386 1387 // If the intrinsic has a mask operand, handle that. 1388 if (CI->getNumArgOperands() == 5) 1389 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, 1390 CI->getArgOperand(3)); 1391 } else if (IsX86 && (Name.startswith("avx.vextractf128.") || 1392 Name == "avx2.vextracti128" || 1393 Name.startswith("avx512.mask.vextract"))) { 1394 Value *Op0 = CI->getArgOperand(0); 1395 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 1396 unsigned DstNumElts = CI->getType()->getVectorNumElements(); 1397 unsigned SrcNumElts = Op0->getType()->getVectorNumElements(); 1398 unsigned Scale = SrcNumElts / DstNumElts; 1399 1400 // Mask off the high bits of the immediate value; hardware ignores those. 1401 Imm = Imm % Scale; 1402 1403 // Get indexes for the subvector of the input vector. 1404 SmallVector<uint32_t, 8> Idxs(DstNumElts); 1405 for (unsigned i = 0; i != DstNumElts; ++i) { 1406 Idxs[i] = i + (Imm * DstNumElts); 1407 } 1408 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); 1409 1410 // If the intrinsic has a mask operand, handle that. 1411 if (CI->getNumArgOperands() == 4) 1412 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1413 CI->getArgOperand(2)); 1414 } else if (!IsX86 && Name == "stackprotectorcheck") { 1415 Rep = nullptr; 1416 } else if (IsX86 && (Name.startswith("avx512.mask.perm.df.") || 1417 Name.startswith("avx512.mask.perm.di."))) { 1418 Value *Op0 = CI->getArgOperand(0); 1419 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 1420 VectorType *VecTy = cast<VectorType>(CI->getType()); 1421 unsigned NumElts = VecTy->getNumElements(); 1422 1423 SmallVector<uint32_t, 8> Idxs(NumElts); 1424 for (unsigned i = 0; i != NumElts; ++i) 1425 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3); 1426 1427 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); 1428 1429 if (CI->getNumArgOperands() == 4) 1430 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1431 CI->getArgOperand(2)); 1432 } else if (IsX86 && (Name.startswith("avx.vperm2f128.") || 1433 Name == "avx2.vperm2i128")) { 1434 // The immediate permute control byte looks like this: 1435 // [1:0] - select 128 bits from sources for low half of destination 1436 // [2] - ignore 1437 // [3] - zero low half of destination 1438 // [5:4] - select 128 bits from sources for high half of destination 1439 // [6] - ignore 1440 // [7] - zero high half of destination 1441 1442 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 1443 1444 unsigned NumElts = CI->getType()->getVectorNumElements(); 1445 unsigned HalfSize = NumElts / 2; 1446 SmallVector<uint32_t, 8> ShuffleMask(NumElts); 1447 1448 // Determine which operand(s) are actually in use for this instruction. 1449 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0); 1450 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0); 1451 1452 // If needed, replace operands based on zero mask. 1453 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0; 1454 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1; 1455 1456 // Permute low half of result. 1457 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0; 1458 for (unsigned i = 0; i < HalfSize; ++i) 1459 ShuffleMask[i] = StartIndex + i; 1460 1461 // Permute high half of result. 1462 StartIndex = (Imm & 0x10) ? HalfSize : 0; 1463 for (unsigned i = 0; i < HalfSize; ++i) 1464 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i; 1465 1466 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask); 1467 1468 } else if (IsX86 && (Name.startswith("avx.vpermil.") || 1469 Name == "sse2.pshuf.d" || 1470 Name.startswith("avx512.mask.vpermil.p") || 1471 Name.startswith("avx512.mask.pshuf.d."))) { 1472 Value *Op0 = CI->getArgOperand(0); 1473 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 1474 VectorType *VecTy = cast<VectorType>(CI->getType()); 1475 unsigned NumElts = VecTy->getNumElements(); 1476 // Calculate the size of each index in the immediate. 1477 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits(); 1478 unsigned IdxMask = ((1 << IdxSize) - 1); 1479 1480 SmallVector<uint32_t, 8> Idxs(NumElts); 1481 // Lookup the bits for this element, wrapping around the immediate every 1482 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need 1483 // to offset by the first index of each group. 1484 for (unsigned i = 0; i != NumElts; ++i) 1485 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask); 1486 1487 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); 1488 1489 if (CI->getNumArgOperands() == 4) 1490 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1491 CI->getArgOperand(2)); 1492 } else if (IsX86 && (Name == "sse2.pshufl.w" || 1493 Name.startswith("avx512.mask.pshufl.w."))) { 1494 Value *Op0 = CI->getArgOperand(0); 1495 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 1496 unsigned NumElts = CI->getType()->getVectorNumElements(); 1497 1498 SmallVector<uint32_t, 16> Idxs(NumElts); 1499 for (unsigned l = 0; l != NumElts; l += 8) { 1500 for (unsigned i = 0; i != 4; ++i) 1501 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l; 1502 for (unsigned i = 4; i != 8; ++i) 1503 Idxs[i + l] = i + l; 1504 } 1505 1506 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); 1507 1508 if (CI->getNumArgOperands() == 4) 1509 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1510 CI->getArgOperand(2)); 1511 } else if (IsX86 && (Name == "sse2.pshufh.w" || 1512 Name.startswith("avx512.mask.pshufh.w."))) { 1513 Value *Op0 = CI->getArgOperand(0); 1514 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 1515 unsigned NumElts = CI->getType()->getVectorNumElements(); 1516 1517 SmallVector<uint32_t, 16> Idxs(NumElts); 1518 for (unsigned l = 0; l != NumElts; l += 8) { 1519 for (unsigned i = 0; i != 4; ++i) 1520 Idxs[i + l] = i + l; 1521 for (unsigned i = 0; i != 4; ++i) 1522 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l; 1523 } 1524 1525 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); 1526 1527 if (CI->getNumArgOperands() == 4) 1528 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1529 CI->getArgOperand(2)); 1530 } else if (IsX86 && Name.startswith("avx512.mask.shuf.p")) { 1531 Value *Op0 = CI->getArgOperand(0); 1532 Value *Op1 = CI->getArgOperand(1); 1533 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 1534 unsigned NumElts = CI->getType()->getVectorNumElements(); 1535 1536 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); 1537 unsigned HalfLaneElts = NumLaneElts / 2; 1538 1539 SmallVector<uint32_t, 16> Idxs(NumElts); 1540 for (unsigned i = 0; i != NumElts; ++i) { 1541 // Base index is the starting element of the lane. 1542 Idxs[i] = i - (i % NumLaneElts); 1543 // If we are half way through the lane switch to the other source. 1544 if ((i % NumLaneElts) >= HalfLaneElts) 1545 Idxs[i] += NumElts; 1546 // Now select the specific element. By adding HalfLaneElts bits from 1547 // the immediate. Wrapping around the immediate every 8-bits. 1548 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1); 1549 } 1550 1551 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs); 1552 1553 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, 1554 CI->getArgOperand(3)); 1555 } else if (IsX86 && (Name.startswith("avx512.mask.movddup") || 1556 Name.startswith("avx512.mask.movshdup") || 1557 Name.startswith("avx512.mask.movsldup"))) { 1558 Value *Op0 = CI->getArgOperand(0); 1559 unsigned NumElts = CI->getType()->getVectorNumElements(); 1560 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); 1561 1562 unsigned Offset = 0; 1563 if (Name.startswith("avx512.mask.movshdup.")) 1564 Offset = 1; 1565 1566 SmallVector<uint32_t, 16> Idxs(NumElts); 1567 for (unsigned l = 0; l != NumElts; l += NumLaneElts) 1568 for (unsigned i = 0; i != NumLaneElts; i += 2) { 1569 Idxs[i + l + 0] = i + l + Offset; 1570 Idxs[i + l + 1] = i + l + Offset; 1571 } 1572 1573 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); 1574 1575 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 1576 CI->getArgOperand(1)); 1577 } else if (IsX86 && (Name.startswith("avx512.mask.punpckl") || 1578 Name.startswith("avx512.mask.unpckl."))) { 1579 Value *Op0 = CI->getArgOperand(0); 1580 Value *Op1 = CI->getArgOperand(1); 1581 int NumElts = CI->getType()->getVectorNumElements(); 1582 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); 1583 1584 SmallVector<uint32_t, 64> Idxs(NumElts); 1585 for (int l = 0; l != NumElts; l += NumLaneElts) 1586 for (int i = 0; i != NumLaneElts; ++i) 1587 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2); 1588 1589 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs); 1590 1591 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1592 CI->getArgOperand(2)); 1593 } else if (IsX86 && (Name.startswith("avx512.mask.punpckh") || 1594 Name.startswith("avx512.mask.unpckh."))) { 1595 Value *Op0 = CI->getArgOperand(0); 1596 Value *Op1 = CI->getArgOperand(1); 1597 int NumElts = CI->getType()->getVectorNumElements(); 1598 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); 1599 1600 SmallVector<uint32_t, 64> Idxs(NumElts); 1601 for (int l = 0; l != NumElts; l += NumLaneElts) 1602 for (int i = 0; i != NumLaneElts; ++i) 1603 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2); 1604 1605 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs); 1606 1607 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1608 CI->getArgOperand(2)); 1609 } else if (IsX86 && Name.startswith("avx512.mask.pand.")) { 1610 Rep = Builder.CreateAnd(CI->getArgOperand(0), CI->getArgOperand(1)); 1611 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1612 CI->getArgOperand(2)); 1613 } else if (IsX86 && Name.startswith("avx512.mask.pandn.")) { 1614 Rep = Builder.CreateAnd(Builder.CreateNot(CI->getArgOperand(0)), 1615 CI->getArgOperand(1)); 1616 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1617 CI->getArgOperand(2)); 1618 } else if (IsX86 && Name.startswith("avx512.mask.por.")) { 1619 Rep = Builder.CreateOr(CI->getArgOperand(0), CI->getArgOperand(1)); 1620 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1621 CI->getArgOperand(2)); 1622 } else if (IsX86 && Name.startswith("avx512.mask.pxor.")) { 1623 Rep = Builder.CreateXor(CI->getArgOperand(0), CI->getArgOperand(1)); 1624 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1625 CI->getArgOperand(2)); 1626 } else if (IsX86 && Name.startswith("avx512.mask.and.")) { 1627 VectorType *FTy = cast<VectorType>(CI->getType()); 1628 VectorType *ITy = VectorType::getInteger(FTy); 1629 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy), 1630 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); 1631 Rep = Builder.CreateBitCast(Rep, FTy); 1632 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1633 CI->getArgOperand(2)); 1634 } else if (IsX86 && Name.startswith("avx512.mask.andn.")) { 1635 VectorType *FTy = cast<VectorType>(CI->getType()); 1636 VectorType *ITy = VectorType::getInteger(FTy); 1637 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy)); 1638 Rep = Builder.CreateAnd(Rep, 1639 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); 1640 Rep = Builder.CreateBitCast(Rep, FTy); 1641 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1642 CI->getArgOperand(2)); 1643 } else if (IsX86 && Name.startswith("avx512.mask.or.")) { 1644 VectorType *FTy = cast<VectorType>(CI->getType()); 1645 VectorType *ITy = VectorType::getInteger(FTy); 1646 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy), 1647 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); 1648 Rep = Builder.CreateBitCast(Rep, FTy); 1649 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1650 CI->getArgOperand(2)); 1651 } else if (IsX86 && Name.startswith("avx512.mask.xor.")) { 1652 VectorType *FTy = cast<VectorType>(CI->getType()); 1653 VectorType *ITy = VectorType::getInteger(FTy); 1654 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy), 1655 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); 1656 Rep = Builder.CreateBitCast(Rep, FTy); 1657 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1658 CI->getArgOperand(2)); 1659 } else if (IsX86 && Name.startswith("avx512.mask.padd.")) { 1660 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1)); 1661 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1662 CI->getArgOperand(2)); 1663 } else if (IsX86 && Name.startswith("avx512.mask.psub.")) { 1664 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1)); 1665 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1666 CI->getArgOperand(2)); 1667 } else if (IsX86 && Name.startswith("avx512.mask.pmull.")) { 1668 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1)); 1669 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1670 CI->getArgOperand(2)); 1671 } else if (IsX86 && (Name.startswith("avx512.mask.add.p"))) { 1672 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1)); 1673 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1674 CI->getArgOperand(2)); 1675 } else if (IsX86 && Name.startswith("avx512.mask.div.p")) { 1676 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1)); 1677 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1678 CI->getArgOperand(2)); 1679 } else if (IsX86 && Name.startswith("avx512.mask.mul.p")) { 1680 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1)); 1681 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1682 CI->getArgOperand(2)); 1683 } else if (IsX86 && Name.startswith("avx512.mask.sub.p")) { 1684 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1)); 1685 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1686 CI->getArgOperand(2)); 1687 } else if (IsX86 && Name.startswith("avx512.mask.lzcnt.")) { 1688 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), 1689 Intrinsic::ctlz, 1690 CI->getType()), 1691 { CI->getArgOperand(0), Builder.getInt1(false) }); 1692 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 1693 CI->getArgOperand(1)); 1694 } else if (IsX86 && (Name.startswith("avx512.mask.max.p") || 1695 Name.startswith("avx512.mask.min.p"))) { 1696 bool IsMin = Name[13] == 'i'; 1697 VectorType *VecTy = cast<VectorType>(CI->getType()); 1698 unsigned VecWidth = VecTy->getPrimitiveSizeInBits(); 1699 unsigned EltWidth = VecTy->getScalarSizeInBits(); 1700 Intrinsic::ID IID; 1701 if (!IsMin && VecWidth == 128 && EltWidth == 32) 1702 IID = Intrinsic::x86_sse_max_ps; 1703 else if (!IsMin && VecWidth == 128 && EltWidth == 64) 1704 IID = Intrinsic::x86_sse2_max_pd; 1705 else if (!IsMin && VecWidth == 256 && EltWidth == 32) 1706 IID = Intrinsic::x86_avx_max_ps_256; 1707 else if (!IsMin && VecWidth == 256 && EltWidth == 64) 1708 IID = Intrinsic::x86_avx_max_pd_256; 1709 else if (IsMin && VecWidth == 128 && EltWidth == 32) 1710 IID = Intrinsic::x86_sse_min_ps; 1711 else if (IsMin && VecWidth == 128 && EltWidth == 64) 1712 IID = Intrinsic::x86_sse2_min_pd; 1713 else if (IsMin && VecWidth == 256 && EltWidth == 32) 1714 IID = Intrinsic::x86_avx_min_ps_256; 1715 else if (IsMin && VecWidth == 256 && EltWidth == 64) 1716 IID = Intrinsic::x86_avx_min_pd_256; 1717 else 1718 llvm_unreachable("Unexpected intrinsic"); 1719 1720 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 1721 { CI->getArgOperand(0), CI->getArgOperand(1) }); 1722 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1723 CI->getArgOperand(2)); 1724 } else if (IsX86 && Name.startswith("avx512.mask.pshuf.b.")) { 1725 VectorType *VecTy = cast<VectorType>(CI->getType()); 1726 Intrinsic::ID IID; 1727 if (VecTy->getPrimitiveSizeInBits() == 128) 1728 IID = Intrinsic::x86_ssse3_pshuf_b_128; 1729 else if (VecTy->getPrimitiveSizeInBits() == 256) 1730 IID = Intrinsic::x86_avx2_pshuf_b; 1731 else if (VecTy->getPrimitiveSizeInBits() == 512) 1732 IID = Intrinsic::x86_avx512_pshuf_b_512; 1733 else 1734 llvm_unreachable("Unexpected intrinsic"); 1735 1736 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 1737 { CI->getArgOperand(0), CI->getArgOperand(1) }); 1738 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1739 CI->getArgOperand(2)); 1740 } else if (IsX86 && (Name.startswith("avx512.mask.pmul.dq.") || 1741 Name.startswith("avx512.mask.pmulu.dq."))) { 1742 bool IsUnsigned = Name[16] == 'u'; 1743 VectorType *VecTy = cast<VectorType>(CI->getType()); 1744 Intrinsic::ID IID; 1745 if (!IsUnsigned && VecTy->getPrimitiveSizeInBits() == 128) 1746 IID = Intrinsic::x86_sse41_pmuldq; 1747 else if (!IsUnsigned && VecTy->getPrimitiveSizeInBits() == 256) 1748 IID = Intrinsic::x86_avx2_pmul_dq; 1749 else if (!IsUnsigned && VecTy->getPrimitiveSizeInBits() == 512) 1750 IID = Intrinsic::x86_avx512_pmul_dq_512; 1751 else if (IsUnsigned && VecTy->getPrimitiveSizeInBits() == 128) 1752 IID = Intrinsic::x86_sse2_pmulu_dq; 1753 else if (IsUnsigned && VecTy->getPrimitiveSizeInBits() == 256) 1754 IID = Intrinsic::x86_avx2_pmulu_dq; 1755 else if (IsUnsigned && VecTy->getPrimitiveSizeInBits() == 512) 1756 IID = Intrinsic::x86_avx512_pmulu_dq_512; 1757 else 1758 llvm_unreachable("Unexpected intrinsic"); 1759 1760 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 1761 { CI->getArgOperand(0), CI->getArgOperand(1) }); 1762 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1763 CI->getArgOperand(2)); 1764 } else if (IsX86 && Name.startswith("avx512.mask.pack")) { 1765 bool IsUnsigned = Name[16] == 'u'; 1766 bool IsDW = Name[18] == 'd'; 1767 VectorType *VecTy = cast<VectorType>(CI->getType()); 1768 Intrinsic::ID IID; 1769 if (!IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 128) 1770 IID = Intrinsic::x86_sse2_packsswb_128; 1771 else if (!IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 256) 1772 IID = Intrinsic::x86_avx2_packsswb; 1773 else if (!IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 512) 1774 IID = Intrinsic::x86_avx512_packsswb_512; 1775 else if (!IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 128) 1776 IID = Intrinsic::x86_sse2_packssdw_128; 1777 else if (!IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 256) 1778 IID = Intrinsic::x86_avx2_packssdw; 1779 else if (!IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 512) 1780 IID = Intrinsic::x86_avx512_packssdw_512; 1781 else if (IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 128) 1782 IID = Intrinsic::x86_sse2_packuswb_128; 1783 else if (IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 256) 1784 IID = Intrinsic::x86_avx2_packuswb; 1785 else if (IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 512) 1786 IID = Intrinsic::x86_avx512_packuswb_512; 1787 else if (IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 128) 1788 IID = Intrinsic::x86_sse41_packusdw; 1789 else if (IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 256) 1790 IID = Intrinsic::x86_avx2_packusdw; 1791 else if (IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 512) 1792 IID = Intrinsic::x86_avx512_packusdw_512; 1793 else 1794 llvm_unreachable("Unexpected intrinsic"); 1795 1796 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 1797 { CI->getArgOperand(0), CI->getArgOperand(1) }); 1798 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 1799 CI->getArgOperand(2)); 1800 } else if (IsX86 && Name.startswith("avx512.mask.psll")) { 1801 bool IsImmediate = Name[16] == 'i' || 1802 (Name.size() > 18 && Name[18] == 'i'); 1803 bool IsVariable = Name[16] == 'v'; 1804 char Size = Name[16] == '.' ? Name[17] : 1805 Name[17] == '.' ? Name[18] : 1806 Name[18] == '.' ? Name[19] : 1807 Name[20]; 1808 1809 Intrinsic::ID IID; 1810 if (IsVariable && Name[17] != '.') { 1811 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di 1812 IID = Intrinsic::x86_avx2_psllv_q; 1813 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di 1814 IID = Intrinsic::x86_avx2_psllv_q_256; 1815 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si 1816 IID = Intrinsic::x86_avx2_psllv_d; 1817 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si 1818 IID = Intrinsic::x86_avx2_psllv_d_256; 1819 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi 1820 IID = Intrinsic::x86_avx512_psllv_w_128; 1821 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi 1822 IID = Intrinsic::x86_avx512_psllv_w_256; 1823 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi 1824 IID = Intrinsic::x86_avx512_psllv_w_512; 1825 else 1826 llvm_unreachable("Unexpected size"); 1827 } else if (Name.endswith(".128")) { 1828 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128 1829 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d 1830 : Intrinsic::x86_sse2_psll_d; 1831 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128 1832 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q 1833 : Intrinsic::x86_sse2_psll_q; 1834 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128 1835 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w 1836 : Intrinsic::x86_sse2_psll_w; 1837 else 1838 llvm_unreachable("Unexpected size"); 1839 } else if (Name.endswith(".256")) { 1840 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256 1841 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d 1842 : Intrinsic::x86_avx2_psll_d; 1843 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256 1844 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q 1845 : Intrinsic::x86_avx2_psll_q; 1846 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256 1847 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w 1848 : Intrinsic::x86_avx2_psll_w; 1849 else 1850 llvm_unreachable("Unexpected size"); 1851 } else { 1852 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512 1853 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 : 1854 IsVariable ? Intrinsic::x86_avx512_psllv_d_512 : 1855 Intrinsic::x86_avx512_psll_d_512; 1856 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512 1857 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 : 1858 IsVariable ? Intrinsic::x86_avx512_psllv_q_512 : 1859 Intrinsic::x86_avx512_psll_q_512; 1860 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w 1861 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512 1862 : Intrinsic::x86_avx512_psll_w_512; 1863 else 1864 llvm_unreachable("Unexpected size"); 1865 } 1866 1867 Rep = UpgradeX86MaskedShift(Builder, *CI, IID); 1868 } else if (IsX86 && Name.startswith("avx512.mask.psrl")) { 1869 bool IsImmediate = Name[16] == 'i' || 1870 (Name.size() > 18 && Name[18] == 'i'); 1871 bool IsVariable = Name[16] == 'v'; 1872 char Size = Name[16] == '.' ? Name[17] : 1873 Name[17] == '.' ? Name[18] : 1874 Name[18] == '.' ? Name[19] : 1875 Name[20]; 1876 1877 Intrinsic::ID IID; 1878 if (IsVariable && Name[17] != '.') { 1879 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di 1880 IID = Intrinsic::x86_avx2_psrlv_q; 1881 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di 1882 IID = Intrinsic::x86_avx2_psrlv_q_256; 1883 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si 1884 IID = Intrinsic::x86_avx2_psrlv_d; 1885 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si 1886 IID = Intrinsic::x86_avx2_psrlv_d_256; 1887 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi 1888 IID = Intrinsic::x86_avx512_psrlv_w_128; 1889 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi 1890 IID = Intrinsic::x86_avx512_psrlv_w_256; 1891 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi 1892 IID = Intrinsic::x86_avx512_psrlv_w_512; 1893 else 1894 llvm_unreachable("Unexpected size"); 1895 } else if (Name.endswith(".128")) { 1896 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128 1897 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d 1898 : Intrinsic::x86_sse2_psrl_d; 1899 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128 1900 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q 1901 : Intrinsic::x86_sse2_psrl_q; 1902 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128 1903 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w 1904 : Intrinsic::x86_sse2_psrl_w; 1905 else 1906 llvm_unreachable("Unexpected size"); 1907 } else if (Name.endswith(".256")) { 1908 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256 1909 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d 1910 : Intrinsic::x86_avx2_psrl_d; 1911 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256 1912 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q 1913 : Intrinsic::x86_avx2_psrl_q; 1914 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256 1915 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w 1916 : Intrinsic::x86_avx2_psrl_w; 1917 else 1918 llvm_unreachable("Unexpected size"); 1919 } else { 1920 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512 1921 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 : 1922 IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 : 1923 Intrinsic::x86_avx512_psrl_d_512; 1924 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512 1925 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 : 1926 IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 : 1927 Intrinsic::x86_avx512_psrl_q_512; 1928 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w) 1929 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512 1930 : Intrinsic::x86_avx512_psrl_w_512; 1931 else 1932 llvm_unreachable("Unexpected size"); 1933 } 1934 1935 Rep = UpgradeX86MaskedShift(Builder, *CI, IID); 1936 } else if (IsX86 && Name.startswith("avx512.mask.psra")) { 1937 bool IsImmediate = Name[16] == 'i' || 1938 (Name.size() > 18 && Name[18] == 'i'); 1939 bool IsVariable = Name[16] == 'v'; 1940 char Size = Name[16] == '.' ? Name[17] : 1941 Name[17] == '.' ? Name[18] : 1942 Name[18] == '.' ? Name[19] : 1943 Name[20]; 1944 1945 Intrinsic::ID IID; 1946 if (IsVariable && Name[17] != '.') { 1947 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si 1948 IID = Intrinsic::x86_avx2_psrav_d; 1949 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si 1950 IID = Intrinsic::x86_avx2_psrav_d_256; 1951 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi 1952 IID = Intrinsic::x86_avx512_psrav_w_128; 1953 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi 1954 IID = Intrinsic::x86_avx512_psrav_w_256; 1955 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi 1956 IID = Intrinsic::x86_avx512_psrav_w_512; 1957 else 1958 llvm_unreachable("Unexpected size"); 1959 } else if (Name.endswith(".128")) { 1960 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128 1961 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d 1962 : Intrinsic::x86_sse2_psra_d; 1963 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128 1964 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 : 1965 IsVariable ? Intrinsic::x86_avx512_psrav_q_128 : 1966 Intrinsic::x86_avx512_psra_q_128; 1967 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128 1968 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w 1969 : Intrinsic::x86_sse2_psra_w; 1970 else 1971 llvm_unreachable("Unexpected size"); 1972 } else if (Name.endswith(".256")) { 1973 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256 1974 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d 1975 : Intrinsic::x86_avx2_psra_d; 1976 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256 1977 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 : 1978 IsVariable ? Intrinsic::x86_avx512_psrav_q_256 : 1979 Intrinsic::x86_avx512_psra_q_256; 1980 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256 1981 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w 1982 : Intrinsic::x86_avx2_psra_w; 1983 else 1984 llvm_unreachable("Unexpected size"); 1985 } else { 1986 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512 1987 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 : 1988 IsVariable ? Intrinsic::x86_avx512_psrav_d_512 : 1989 Intrinsic::x86_avx512_psra_d_512; 1990 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q 1991 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 : 1992 IsVariable ? Intrinsic::x86_avx512_psrav_q_512 : 1993 Intrinsic::x86_avx512_psra_q_512; 1994 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w 1995 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512 1996 : Intrinsic::x86_avx512_psra_w_512; 1997 else 1998 llvm_unreachable("Unexpected size"); 1999 } 2000 2001 Rep = UpgradeX86MaskedShift(Builder, *CI, IID); 2002 } else if (IsX86 && Name.startswith("avx512.mask.move.s")) { 2003 Rep = upgradeMaskedMove(Builder, *CI); 2004 } else if (IsX86 && Name.startswith("avx512.cvtmask2")) { 2005 Rep = UpgradeMaskToInt(Builder, *CI); 2006 } else if (IsX86 && Name.startswith("avx512.mask.vpermilvar.")) { 2007 Intrinsic::ID IID; 2008 if (Name.endswith("ps.128")) 2009 IID = Intrinsic::x86_avx_vpermilvar_ps; 2010 else if (Name.endswith("pd.128")) 2011 IID = Intrinsic::x86_avx_vpermilvar_pd; 2012 else if (Name.endswith("ps.256")) 2013 IID = Intrinsic::x86_avx_vpermilvar_ps_256; 2014 else if (Name.endswith("pd.256")) 2015 IID = Intrinsic::x86_avx_vpermilvar_pd_256; 2016 else if (Name.endswith("ps.512")) 2017 IID = Intrinsic::x86_avx512_vpermilvar_ps_512; 2018 else if (Name.endswith("pd.512")) 2019 IID = Intrinsic::x86_avx512_vpermilvar_pd_512; 2020 else 2021 llvm_unreachable("Unexpected vpermilvar intrinsic"); 2022 2023 Function *Intrin = Intrinsic::getDeclaration(F->getParent(), IID); 2024 Rep = Builder.CreateCall(Intrin, 2025 { CI->getArgOperand(0), CI->getArgOperand(1) }); 2026 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 2027 CI->getArgOperand(2)); 2028 } else if (IsX86 && Name.endswith(".movntdqa")) { 2029 Module *M = F->getParent(); 2030 MDNode *Node = MDNode::get( 2031 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); 2032 2033 Value *Ptr = CI->getArgOperand(0); 2034 VectorType *VTy = cast<VectorType>(CI->getType()); 2035 2036 // Convert the type of the pointer to a pointer to the stored type. 2037 Value *BC = 2038 Builder.CreateBitCast(Ptr, PointerType::getUnqual(VTy), "cast"); 2039 LoadInst *LI = Builder.CreateAlignedLoad(BC, VTy->getBitWidth() / 8); 2040 LI->setMetadata(M->getMDKindID("nontemporal"), Node); 2041 Rep = LI; 2042 } else if (IsX86 && 2043 (Name.startswith("sse2.pavg") || Name.startswith("avx2.pavg") || 2044 Name.startswith("avx512.mask.pavg"))) { 2045 // llvm.x86.sse2.pavg.b/w, llvm.x86.avx2.pavg.b/w, 2046 // llvm.x86.avx512.mask.pavg.b/w 2047 Value *A = CI->getArgOperand(0); 2048 Value *B = CI->getArgOperand(1); 2049 VectorType *ZextType = VectorType::getExtendedElementVectorType( 2050 cast<VectorType>(A->getType())); 2051 Value *ExtendedA = Builder.CreateZExt(A, ZextType); 2052 Value *ExtendedB = Builder.CreateZExt(B, ZextType); 2053 Value *Sum = Builder.CreateAdd(ExtendedA, ExtendedB); 2054 Value *AddOne = Builder.CreateAdd(Sum, ConstantInt::get(ZextType, 1)); 2055 Value *ShiftR = Builder.CreateLShr(AddOne, ConstantInt::get(ZextType, 1)); 2056 Rep = Builder.CreateTrunc(ShiftR, A->getType()); 2057 if (CI->getNumArgOperands() > 2) { 2058 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 2059 CI->getArgOperand(2)); 2060 } 2061 } else if (IsNVVM && (Name == "abs.i" || Name == "abs.ll")) { 2062 Value *Arg = CI->getArgOperand(0); 2063 Value *Neg = Builder.CreateNeg(Arg, "neg"); 2064 Value *Cmp = Builder.CreateICmpSGE( 2065 Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond"); 2066 Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs"); 2067 } else if (IsNVVM && (Name == "max.i" || Name == "max.ll" || 2068 Name == "max.ui" || Name == "max.ull")) { 2069 Value *Arg0 = CI->getArgOperand(0); 2070 Value *Arg1 = CI->getArgOperand(1); 2071 Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull") 2072 ? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond") 2073 : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond"); 2074 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max"); 2075 } else if (IsNVVM && (Name == "min.i" || Name == "min.ll" || 2076 Name == "min.ui" || Name == "min.ull")) { 2077 Value *Arg0 = CI->getArgOperand(0); 2078 Value *Arg1 = CI->getArgOperand(1); 2079 Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull") 2080 ? Builder.CreateICmpULE(Arg0, Arg1, "min.cond") 2081 : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond"); 2082 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min"); 2083 } else if (IsNVVM && Name == "clz.ll") { 2084 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 and returns an i64. 2085 Value *Arg = CI->getArgOperand(0); 2086 Value *Ctlz = Builder.CreateCall( 2087 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, 2088 {Arg->getType()}), 2089 {Arg, Builder.getFalse()}, "ctlz"); 2090 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc"); 2091 } else if (IsNVVM && Name == "popc.ll") { 2092 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 and returns an 2093 // i64. 2094 Value *Arg = CI->getArgOperand(0); 2095 Value *Popc = Builder.CreateCall( 2096 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, 2097 {Arg->getType()}), 2098 Arg, "ctpop"); 2099 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc"); 2100 } else if (IsNVVM && Name == "h2f") { 2101 Rep = Builder.CreateCall(Intrinsic::getDeclaration( 2102 F->getParent(), Intrinsic::convert_from_fp16, 2103 {Builder.getFloatTy()}), 2104 CI->getArgOperand(0), "h2f"); 2105 } else { 2106 llvm_unreachable("Unknown function for CallInst upgrade."); 2107 } 2108 2109 if (Rep) 2110 CI->replaceAllUsesWith(Rep); 2111 CI->eraseFromParent(); 2112 return; 2113 } 2114 2115 CallInst *NewCall = nullptr; 2116 switch (NewFn->getIntrinsicID()) { 2117 default: { 2118 // Handle generic mangling change, but nothing else 2119 assert( 2120 (CI->getCalledFunction()->getName() != NewFn->getName()) && 2121 "Unknown function for CallInst upgrade and isn't just a name change"); 2122 CI->setCalledFunction(NewFn); 2123 return; 2124 } 2125 2126 case Intrinsic::arm_neon_vld1: 2127 case Intrinsic::arm_neon_vld2: 2128 case Intrinsic::arm_neon_vld3: 2129 case Intrinsic::arm_neon_vld4: 2130 case Intrinsic::arm_neon_vld2lane: 2131 case Intrinsic::arm_neon_vld3lane: 2132 case Intrinsic::arm_neon_vld4lane: 2133 case Intrinsic::arm_neon_vst1: 2134 case Intrinsic::arm_neon_vst2: 2135 case Intrinsic::arm_neon_vst3: 2136 case Intrinsic::arm_neon_vst4: 2137 case Intrinsic::arm_neon_vst2lane: 2138 case Intrinsic::arm_neon_vst3lane: 2139 case Intrinsic::arm_neon_vst4lane: { 2140 SmallVector<Value *, 4> Args(CI->arg_operands().begin(), 2141 CI->arg_operands().end()); 2142 NewCall = Builder.CreateCall(NewFn, Args); 2143 break; 2144 } 2145 2146 case Intrinsic::bitreverse: 2147 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); 2148 break; 2149 2150 case Intrinsic::ctlz: 2151 case Intrinsic::cttz: 2152 assert(CI->getNumArgOperands() == 1 && 2153 "Mismatch between function args and call args"); 2154 NewCall = 2155 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()}); 2156 break; 2157 2158 case Intrinsic::objectsize: { 2159 Value *NullIsUnknownSize = CI->getNumArgOperands() == 2 2160 ? Builder.getFalse() 2161 : CI->getArgOperand(2); 2162 NewCall = Builder.CreateCall( 2163 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize}); 2164 break; 2165 } 2166 2167 case Intrinsic::ctpop: 2168 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); 2169 break; 2170 2171 case Intrinsic::convert_from_fp16: 2172 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); 2173 break; 2174 2175 case Intrinsic::dbg_value: 2176 // Upgrade from the old version that had an extra offset argument. 2177 assert(CI->getNumArgOperands() == 4); 2178 // Drop nonzero offsets instead of attempting to upgrade them. 2179 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1))) 2180 if (Offset->isZeroValue()) { 2181 NewCall = Builder.CreateCall( 2182 NewFn, 2183 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)}); 2184 break; 2185 } 2186 CI->eraseFromParent(); 2187 return; 2188 2189 case Intrinsic::x86_xop_vfrcz_ss: 2190 case Intrinsic::x86_xop_vfrcz_sd: 2191 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)}); 2192 break; 2193 2194 case Intrinsic::x86_xop_vpermil2pd: 2195 case Intrinsic::x86_xop_vpermil2ps: 2196 case Intrinsic::x86_xop_vpermil2pd_256: 2197 case Intrinsic::x86_xop_vpermil2ps_256: { 2198 SmallVector<Value *, 4> Args(CI->arg_operands().begin(), 2199 CI->arg_operands().end()); 2200 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType()); 2201 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy); 2202 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy); 2203 NewCall = Builder.CreateCall(NewFn, Args); 2204 break; 2205 } 2206 2207 case Intrinsic::x86_sse41_ptestc: 2208 case Intrinsic::x86_sse41_ptestz: 2209 case Intrinsic::x86_sse41_ptestnzc: { 2210 // The arguments for these intrinsics used to be v4f32, and changed 2211 // to v2i64. This is purely a nop, since those are bitwise intrinsics. 2212 // So, the only thing required is a bitcast for both arguments. 2213 // First, check the arguments have the old type. 2214 Value *Arg0 = CI->getArgOperand(0); 2215 if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4)) 2216 return; 2217 2218 // Old intrinsic, add bitcasts 2219 Value *Arg1 = CI->getArgOperand(1); 2220 2221 Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2); 2222 2223 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast"); 2224 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast"); 2225 2226 NewCall = Builder.CreateCall(NewFn, {BC0, BC1}); 2227 break; 2228 } 2229 2230 case Intrinsic::x86_sse41_insertps: 2231 case Intrinsic::x86_sse41_dppd: 2232 case Intrinsic::x86_sse41_dpps: 2233 case Intrinsic::x86_sse41_mpsadbw: 2234 case Intrinsic::x86_avx_dp_ps_256: 2235 case Intrinsic::x86_avx2_mpsadbw: { 2236 // Need to truncate the last argument from i32 to i8 -- this argument models 2237 // an inherently 8-bit immediate operand to these x86 instructions. 2238 SmallVector<Value *, 4> Args(CI->arg_operands().begin(), 2239 CI->arg_operands().end()); 2240 2241 // Replace the last argument with a trunc. 2242 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc"); 2243 NewCall = Builder.CreateCall(NewFn, Args); 2244 break; 2245 } 2246 2247 case Intrinsic::thread_pointer: { 2248 NewCall = Builder.CreateCall(NewFn, {}); 2249 break; 2250 } 2251 2252 case Intrinsic::invariant_start: 2253 case Intrinsic::invariant_end: 2254 case Intrinsic::masked_load: 2255 case Intrinsic::masked_store: 2256 case Intrinsic::masked_gather: 2257 case Intrinsic::masked_scatter: { 2258 SmallVector<Value *, 4> Args(CI->arg_operands().begin(), 2259 CI->arg_operands().end()); 2260 NewCall = Builder.CreateCall(NewFn, Args); 2261 break; 2262 } 2263 } 2264 assert(NewCall && "Should have either set this variable or returned through " 2265 "the default case"); 2266 std::string Name = CI->getName(); 2267 if (!Name.empty()) { 2268 CI->setName(Name + ".old"); 2269 NewCall->setName(Name); 2270 } 2271 CI->replaceAllUsesWith(NewCall); 2272 CI->eraseFromParent(); 2273 } 2274 2275 void llvm::UpgradeCallsToIntrinsic(Function *F) { 2276 assert(F && "Illegal attempt to upgrade a non-existent intrinsic."); 2277 2278 // Check if this function should be upgraded and get the replacement function 2279 // if there is one. 2280 Function *NewFn; 2281 if (UpgradeIntrinsicFunction(F, NewFn)) { 2282 // Replace all users of the old function with the new function or new 2283 // instructions. This is not a range loop because the call is deleted. 2284 for (auto UI = F->user_begin(), UE = F->user_end(); UI != UE; ) 2285 if (CallInst *CI = dyn_cast<CallInst>(*UI++)) 2286 UpgradeIntrinsicCall(CI, NewFn); 2287 2288 // Remove old function, no longer used, from the module. 2289 F->eraseFromParent(); 2290 } 2291 } 2292 2293 MDNode *llvm::UpgradeTBAANode(MDNode &MD) { 2294 // Check if the tag uses struct-path aware TBAA format. 2295 if (isa<MDNode>(MD.getOperand(0)) && MD.getNumOperands() >= 3) 2296 return &MD; 2297 2298 auto &Context = MD.getContext(); 2299 if (MD.getNumOperands() == 3) { 2300 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)}; 2301 MDNode *ScalarType = MDNode::get(Context, Elts); 2302 // Create a MDNode <ScalarType, ScalarType, offset 0, const> 2303 Metadata *Elts2[] = {ScalarType, ScalarType, 2304 ConstantAsMetadata::get( 2305 Constant::getNullValue(Type::getInt64Ty(Context))), 2306 MD.getOperand(2)}; 2307 return MDNode::get(Context, Elts2); 2308 } 2309 // Create a MDNode <MD, MD, offset 0> 2310 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue( 2311 Type::getInt64Ty(Context)))}; 2312 return MDNode::get(Context, Elts); 2313 } 2314 2315 Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, 2316 Instruction *&Temp) { 2317 if (Opc != Instruction::BitCast) 2318 return nullptr; 2319 2320 Temp = nullptr; 2321 Type *SrcTy = V->getType(); 2322 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() && 2323 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) { 2324 LLVMContext &Context = V->getContext(); 2325 2326 // We have no information about target data layout, so we assume that 2327 // the maximum pointer size is 64bit. 2328 Type *MidTy = Type::getInt64Ty(Context); 2329 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy); 2330 2331 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy); 2332 } 2333 2334 return nullptr; 2335 } 2336 2337 Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) { 2338 if (Opc != Instruction::BitCast) 2339 return nullptr; 2340 2341 Type *SrcTy = C->getType(); 2342 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() && 2343 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) { 2344 LLVMContext &Context = C->getContext(); 2345 2346 // We have no information about target data layout, so we assume that 2347 // the maximum pointer size is 64bit. 2348 Type *MidTy = Type::getInt64Ty(Context); 2349 2350 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy), 2351 DestTy); 2352 } 2353 2354 return nullptr; 2355 } 2356 2357 /// Check the debug info version number, if it is out-dated, drop the debug 2358 /// info. Return true if module is modified. 2359 bool llvm::UpgradeDebugInfo(Module &M) { 2360 unsigned Version = getDebugMetadataVersionFromModule(M); 2361 if (Version == DEBUG_METADATA_VERSION) 2362 return false; 2363 2364 bool RetCode = StripDebugInfo(M); 2365 if (RetCode) { 2366 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version); 2367 M.getContext().diagnose(DiagVersion); 2368 } 2369 return RetCode; 2370 } 2371 2372 bool llvm::UpgradeModuleFlags(Module &M) { 2373 NamedMDNode *ModFlags = M.getModuleFlagsMetadata(); 2374 if (!ModFlags) 2375 return false; 2376 2377 bool HasObjCFlag = false, HasClassProperties = false, Changed = false; 2378 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) { 2379 MDNode *Op = ModFlags->getOperand(I); 2380 if (Op->getNumOperands() != 3) 2381 continue; 2382 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1)); 2383 if (!ID) 2384 continue; 2385 if (ID->getString() == "Objective-C Image Info Version") 2386 HasObjCFlag = true; 2387 if (ID->getString() == "Objective-C Class Properties") 2388 HasClassProperties = true; 2389 // Upgrade PIC/PIE Module Flags. The module flag behavior for these two 2390 // field was Error and now they are Max. 2391 if (ID->getString() == "PIC Level" || ID->getString() == "PIE Level") { 2392 if (auto *Behavior = 2393 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) { 2394 if (Behavior->getLimitedValue() == Module::Error) { 2395 Type *Int32Ty = Type::getInt32Ty(M.getContext()); 2396 Metadata *Ops[3] = { 2397 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Max)), 2398 MDString::get(M.getContext(), ID->getString()), 2399 Op->getOperand(2)}; 2400 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops)); 2401 Changed = true; 2402 } 2403 } 2404 } 2405 // Upgrade Objective-C Image Info Section. Removed the whitespce in the 2406 // section name so that llvm-lto will not complain about mismatching 2407 // module flags that is functionally the same. 2408 if (ID->getString() == "Objective-C Image Info Section") { 2409 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) { 2410 SmallVector<StringRef, 4> ValueComp; 2411 Value->getString().split(ValueComp, " "); 2412 if (ValueComp.size() != 1) { 2413 std::string NewValue; 2414 for (auto &S : ValueComp) 2415 NewValue += S.str(); 2416 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1), 2417 MDString::get(M.getContext(), NewValue)}; 2418 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops)); 2419 Changed = true; 2420 } 2421 } 2422 } 2423 } 2424 2425 // "Objective-C Class Properties" is recently added for Objective-C. We 2426 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module 2427 // flag of value 0, so we can correclty downgrade this flag when trying to 2428 // link an ObjC bitcode without this module flag with an ObjC bitcode with 2429 // this module flag. 2430 if (HasObjCFlag && !HasClassProperties) { 2431 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties", 2432 (uint32_t)0); 2433 Changed = true; 2434 } 2435 2436 return Changed; 2437 } 2438 2439 static bool isOldLoopArgument(Metadata *MD) { 2440 auto *T = dyn_cast_or_null<MDTuple>(MD); 2441 if (!T) 2442 return false; 2443 if (T->getNumOperands() < 1) 2444 return false; 2445 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0)); 2446 if (!S) 2447 return false; 2448 return S->getString().startswith("llvm.vectorizer."); 2449 } 2450 2451 static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) { 2452 StringRef OldPrefix = "llvm.vectorizer."; 2453 assert(OldTag.startswith(OldPrefix) && "Expected old prefix"); 2454 2455 if (OldTag == "llvm.vectorizer.unroll") 2456 return MDString::get(C, "llvm.loop.interleave.count"); 2457 2458 return MDString::get( 2459 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size())) 2460 .str()); 2461 } 2462 2463 static Metadata *upgradeLoopArgument(Metadata *MD) { 2464 auto *T = dyn_cast_or_null<MDTuple>(MD); 2465 if (!T) 2466 return MD; 2467 if (T->getNumOperands() < 1) 2468 return MD; 2469 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0)); 2470 if (!OldTag) 2471 return MD; 2472 if (!OldTag->getString().startswith("llvm.vectorizer.")) 2473 return MD; 2474 2475 // This has an old tag. Upgrade it. 2476 SmallVector<Metadata *, 8> Ops; 2477 Ops.reserve(T->getNumOperands()); 2478 Ops.push_back(upgradeLoopTag(T->getContext(), OldTag->getString())); 2479 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I) 2480 Ops.push_back(T->getOperand(I)); 2481 2482 return MDTuple::get(T->getContext(), Ops); 2483 } 2484 2485 MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) { 2486 auto *T = dyn_cast<MDTuple>(&N); 2487 if (!T) 2488 return &N; 2489 2490 if (none_of(T->operands(), isOldLoopArgument)) 2491 return &N; 2492 2493 SmallVector<Metadata *, 8> Ops; 2494 Ops.reserve(T->getNumOperands()); 2495 for (Metadata *MD : T->operands()) 2496 Ops.push_back(upgradeLoopArgument(MD)); 2497 2498 return MDTuple::get(T->getContext(), Ops); 2499 } 2500